查重bug fix

community-service/src/main/java/com/tanpu/community/service/ThemeTextCheckService.java

@@ -18,6 +18,7 @@ import java.util.List;
 @Slf4j
 public class ThemeTextCheckService {
 
+    public static final int SENTENCE_MIN_LENGTH = 5;
     @Resource
     private ThemeCheckDuplicateMapper themeCheckDuplicateMapper;
 
@@ -37,8 +38,9 @@ public class ThemeTextCheckService {
             String[] split = content.split("。");
             for (int i = 0; i < split.length; i++) {
                 String trim = StringUtils.trim(split[i]);
-                insert(themeId, trim, getHash(trim), i + 1, split.length, userId, themeType);
-
+                if (trim.length() > SENTENCE_MIN_LENGTH) {
+                    insert(themeId, trim, getHash(trim), i + 1, split.length, userId, themeType);
+                }
             }
         } catch (Exception e) {
             log.error("文本查重insert失败,themeId:" + themeId);
@@ -75,8 +77,15 @@ public class ThemeTextCheckService {
         List<Integer> list = new ArrayList<>();
         for (int i = 0; i < split.length; i++) {
             String trim = StringUtils.trim(split[i]);
-            list.add(getHash(trim));
+            if (trim.length() > SENTENCE_MIN_LENGTH) {
+                list.add(getHash(trim));
+            }
         }
+
+        if (list.size() <= 2) {
+            return false;
+        }
+
         // 一年以内
         LambdaQueryWrapper<ThemeCheckDuplicateEntity> w = new LambdaQueryWrapper<ThemeCheckDuplicateEntity>().in(ThemeCheckDuplicateEntity::getPartitionHash, list)
                 .eq(ThemeCheckDuplicateEntity::getDeleteTag, BizStatus.DeleteTag.tag_init)
@@ -84,7 +93,7 @@ public class ThemeTextCheckService {
                 .groupBy(ThemeCheckDuplicateEntity::getThemeId);
         Integer check = themeCheckDuplicateMapper.check(w);
         // 重复率大于80%
-        if (check != null && check * 10 >= split.length * 8) {
+        if (check != null && check >= list.size() * 0.8) {
             return true;
         }
         return false;
@@ -114,19 +123,21 @@ public class ThemeTextCheckService {
         String[] split = content.split("。");
         for (int i = 0; i < split.length; i++) {
             String trim = StringUtils.trim(split[i]);
+            if (trim.length() > SENTENCE_MIN_LENGTH) {
+                ThemeCheckDuplicateEntity build = ThemeCheckDuplicateEntity.builder()
+                        .themeId(themeId)
+                        .partitionText(trim)
+                        .partitionHash(getHash(split[i]))
+                        .partitionNum(i + 1)
+                        .totalParts(split.length)
+                        .userId(userId)
+                        .themeType(themeType)
+                        .createTime(createTime)
+                        .updateTime(createTime)
+                        .build();
+                themeCheckDuplicateMapper.insert(build);
+            }
 
-            ThemeCheckDuplicateEntity build = ThemeCheckDuplicateEntity.builder()
-                    .themeId(themeId)
-                    .partitionText(trim)
-                    .partitionHash(getHash(split[i]))
-                    .partitionNum(i + 1)
-                    .totalParts(split.length)
-                    .userId(userId)
-                    .themeType(themeType)
-                    .createTime(createTime)
-                    .updateTime(createTime)
-                    .build();
-            themeCheckDuplicateMapper.insert(build);
         }