大量数据循环对比查找相似的，for循环对比量很大，有没有办法优化

别玩丐帮 2019-04-10 04:50:48

目前有个业务是统计2W道题里有没有相似的题目，有的话加入到list里面，最后导出excel，目前是两个循环对比，完后调用相似的工具类，但是对比下来量很大，时间很非常慢，大家有没有什么优化的方法



    List<AnswerOcr> answerOcrList = service.queryForList();

    Set<AnswerOcr> setList = new HashSet<>();

    for (int i = 0; i < answerOcrList.size(); i++) {

      if (i + 1 >= answerOcrList.size()) {

        break;

      }

      for (int j = i + 1; j < answerOcrList.size(); j++) {

        if (StrUtil.isEmpty(answerOcrList.get(i).getOcr())||StrUtil.isEmpty(answerOcrList.get(j).getOcr())){

          continue;

        }

        setList.addAll(SimilarityUtils.levenshtein(answerOcrList.get(i),answerOcrList.get(j)));

      }

    }



    BigDecimal bigDecimal = new BigDecimal(setList.size()).divide(new BigDecimal(answerOcrList.size()),5,BigDecimal.ROUND_HALF_DOWN);

    System.out.println("重复率"+bigDecimal);

// 通过工具类创建writer

    ExcelWriter writer = ExcelUtil.getWriter("/home/noah/demo/similarity.xls");



 public static Set<AnswerOcr> levenshtein(AnswerOcr answerOcr1, AnswerOcr answerOcr2) {

    Set<AnswerOcr> answerOcrList = new HashSet<>();

    //计算两个字符串的长度。

    int len1 = answerOcr1.getOcr().length();

    int len2 = answerOcr2.getOcr().length();

    //建立上面说的数组，比字符长度大一个空间

    int[][] dif = new int[len1 + 1][len2 + 1];

    //赋初值，步骤B。

    for (int a = 0; a <= len1; a++) {

      dif[a][0] = a;

    }

    for (int a = 0; a <= len2; a++) {

      dif[0][a] = a;

    }

    //计算两个字符是否一样，计算左上的值

    int temp;

    for (int i = 1; i <= len1; i++) {

      for (int j = 1; j <= len2; j++) {

        if (answerOcr1.getOcr().charAt(i - 1) == answerOcr2.getOcr().charAt(j - 1)) {

          temp = 0;

        } else {

          temp = 1;

        }

        //取三个值中最小的

        dif[i][j] = min(dif[i - 1][j - 1] + temp, dif[i][j - 1] + 1,

            dif[i - 1][j] + 1);

      }

    }

//    取数组右下角的值，同样不同位置代表不同字符串的比较

    //计算相似度

    float similarity =1 - (float) dif[len1][len2] / Math.max(answerOcr1.getOcr().length(), answerOcr2.getOcr().length());

    if (similarity > 0.8){

      answerOcrList.add(answerOcr1);

      answerOcr1.setIds(answerOcr1.getIds()+","+answerOcr2.getQuestionid());

//      DownloadPicFromURL.downloadPicture(answerOcr1.getPath(),"d:/ocr/similarity/"+answerOcr1.getXueduanname()+"-"+answerOcr1.getGradename()+"-"+answerOcr1.getSubjectname()+"-"+answerOcr1.getQuestionid()+".jpg");

      answerOcrList.add(answerOcr2);

      answerOcr2.setIds(answerOcr2.getIds()+","+answerOcr1.getQuestionid());

//      DownloadPicFromURL.downloadPicture(answerOcr2.getPath(),"d:/ocr/similarity/"+answerOcr2.getXueduanname()+"-"+answerOcr2.getGradename()+"-"+answerOcr2.getSubjectname()+"-"+answerOcr2.getQuestionid()+".jpg");

    }

    return answerOcrList;