es 搜索主题正文

community-api/src/main/java/com/tanpu/community/api/beans/qo/ESThemeQo.java

@@ -18,23 +18,23 @@ public class ESThemeQo {
     @ApiModelProperty(value = "类型 1：讨论无标题 2：长文有标题 3：转发 4:评论")
     public Integer themeType;
 
-    @ApiModelProperty(value = "内容")
-    public String content;
-
-    @ApiModelProperty(value = "话题名称")
-    public String topicTitle;
-
-    @ApiModelProperty(value = "昵称")
-    public String nickName;
-
-    @ApiModelProperty(value = "作者认证")
-    public String authLabel;
-
-    @ApiModelProperty(value = "转发的主题")
-    public FormerThemeQo formerTheme;
-
-    @ApiModelProperty(value = "评论Id(我的评论列表使用)")
-    public String commentId;
+    @ApiModelProperty(value = "文本内容")
+    public String textContent;
+
+//    @ApiModelProperty(value = "话题名称")
+//    public String topicTitle;
+//
+//    @ApiModelProperty(value = "昵称")
+//    public String nickName;
+//
+//    @ApiModelProperty(value = "作者认证")
+//    public String authLabel;
+//
+//    @ApiModelProperty(value = "转发的主题")
+//    public FormerThemeQo formerTheme;
+//
+//    @ApiModelProperty(value = "评论Id(我的评论列表使用)")
+//    public String commentId;
 
     public Long createTime;
 

community-api/src/main/java/com/tanpu/community/api/beans/qo/ThemeContentQo.java

@@ -23,7 +23,6 @@ public class ThemeContentQo {
     @ApiModelProperty(value = "文本的值是内容，附件的值为id")
     private String value;
 
-
     @ApiModelProperty(value = "产品类型,0 公募，1 私募，2 白名单，3 私有")
     private Integer productType;
 

community-api/src/main/java/com/tanpu/community/api/beans/req/page/Pageable.java

@@ -15,9 +15,9 @@ public class Pageable {
 
     public static final Integer DEFAULT_PAGE_NUMBER = 0;
 
-    private Integer pageNumber = DEFAULT_PAGE_NUMBER;
+    public Integer pageNumber = DEFAULT_PAGE_NUMBER;
 
-    private Integer pageSize = DEFAULT_PAGE_SIZE;
+    public Integer pageSize = DEFAULT_PAGE_SIZE;
 
     public Pageable() {
     }

community-api/src/main/java/com/tanpu/community/api/beans/req/search/ThemeFullSearchReq.java

@@ -3,7 +3,11 @@ package com.tanpu.community.api.beans.req.search;
 import com.tanpu.community.api.beans.req.page.Pageable;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
-public class ThemeFullSearchReq extends Pageable {
+public class ThemeFullSearchReq {
+    public Pageable page;
     public String keyword;
+    public List<String> excludeIds;
 }

community-api/src/main/java/com/tanpu/community/api/beans/resp/ThemeFullSearchResp.java

+package com.tanpu.community.api.beans.resp;
+
+import com.tanpu.community.api.beans.qo.ThemeQo;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class ThemeFullSearchResp {
+    public List<String> excludeIds;
+    public List<ThemeQo> themes;
+
+    public ThemeFullSearchResp() {
+        this.excludeIds = new ArrayList<>();
+        this.themes = new ArrayList<>();
+    }
+}

community-service/src/main/java/com/tanpu/community/CommunityApplication.java

@@ -14,6 +14,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 @EnableCaching
 @EnableScheduling
 @EnableFeignClients
+@EnableConfigurationProperties({ESConfig.class})
 public class CommunityApplication {
 
     public static void main(String[] args) {

community-service/src/main/java/com/tanpu/community/config/ESConfig.java

 package com.tanpu.community.config;
 
+import lombok.Data;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
@@ -16,16 +17,14 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.stereotype.Component;
 
-@Configuration
+@Data
+@Component
+@ConfigurationProperties(prefix = "es")
 public class ESConfig {
 
-    @Value("${es.userName}")
     private String userName;
-    @Value("${es.userPasswd}")
     private String userPasswd;
-    @Value("${es.host}")
     private String host;
-    @Value("${es.port}")
     private Integer port;
 
     @Bean

community-service/src/main/java/com/tanpu/community/controller/SearchController.java

@@ -6,6 +6,7 @@ import com.tanpu.common.auth.UserInfoHelper;
 import com.tanpu.community.api.beans.qo.ThemeQo;
 import com.tanpu.community.api.beans.qo.TopicDetailQo;
 import com.tanpu.community.api.beans.req.search.ThemeFullSearchReq;
+import com.tanpu.community.api.beans.resp.ThemeFullSearchResp;
 import com.tanpu.community.manager.ThemeManager;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
@@ -27,9 +28,12 @@ public class SearchController {
     private ThemeManager themeManager;
 
     // 内容全文搜索
-    public CommonResp<List<ThemeQo>> themeSearch(ThemeFullSearchReq req) {
-        List<ThemeQo> list = themeManager.themeFullSearch(req.keyword, req.getPageNumber(), req.getPageSize(), userHolder.getUserId());
-        return CommonResp.success(list);
+    @ApiOperation("全文搜索主题")
+    @PostMapping(value = "/themeFullText")
+    @ResponseBody
+    public CommonResp<ThemeFullSearchResp> themeFullText(@RequestBody ThemeFullSearchReq req) {
+        ThemeFullSearchResp resp = themeManager.themeFullSearch(req.keyword, req.page.pageNumber, req.page.pageSize, req.excludeIds, userHolder.getUserId());
+        return CommonResp.success(resp);
     }
 
 }

community-service/src/main/java/com/tanpu/community/controller/ThemeController.java

@@ -34,7 +34,8 @@ public class ThemeController {
     @PostMapping(value = "/publish")
     @ResponseBody
     public CommonResp<CreateThemeResp> publishTheme(@Validated @RequestBody CreateThemeReq req) {
-        String userId = userHolder.getUserId();
+//        String userId = userHolder.getUserId();
+        String userId = "4444";
         return CommonResp.success(themeManager.publishTheme(req, userId));
     }
 

community-service/src/main/java/com/tanpu/community/manager/ThemeManager.java

@@ -9,6 +9,7 @@ import com.tanpu.community.api.beans.qo.ThemeQo;
 import com.tanpu.community.api.beans.req.homepage.QueryRecordThemeReq;
 import com.tanpu.community.api.beans.req.theme.*;
 import com.tanpu.community.api.beans.resp.CreateThemeResp;
+import com.tanpu.community.api.beans.resp.ThemeFullSearchResp;
 import com.tanpu.community.api.enums.*;
 import com.tanpu.community.api.enums.BlockTypeEnum;
 import com.tanpu.community.api.enums.CollectionTypeEnum;
@@ -21,9 +22,11 @@ import com.tanpu.community.service.*;
 import com.tanpu.community.service.BlackListService;
 import com.tanpu.community.service.base.ESService;
 import com.tanpu.community.util.ConvertUtil;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -34,6 +37,7 @@ import java.time.LocalDateTime;
 import java.util.*;
 import java.util.stream.Collectors;
 
+@Slf4j
 @Service
 public class ThemeManager {
     @Resource
@@ -66,17 +70,26 @@ public class ThemeManager {
     @Autowired
     private ESService esService;
 
-    public List<ThemeQo> themeFullSearch(String keyword, Integer pageNo, Integer pageSize, String userId) {
+    public ThemeFullSearchResp themeFullSearch(String keyword, Integer pageNo, Integer pageSize, List<String> excludeIds, String userId) {
         Integer from = (pageNo - 1) * pageSize;
+        ThemeFullSearchResp resp = new ThemeFullSearchResp();
 
         // 按时间倒叙查询
-        List<ESThemeQo> themes = esService.queryThemeIdByContentAndTitle(keyword, from, pageSize);
-        if (themes.isEmpty()) {
-            return new ArrayList<>();
+        // todo redis
+        List<ESThemeQo> esIds = esService.queryThemeIdByContentAndTitle(keyword, from, pageSize * 5);
+        if (esIds.isEmpty()) {
+            return resp;
         }
 
-        List<ThemeEntity> themeEntities = themeService.queryByThemeIds(themes.stream().map(ESThemeQo::getThemeId).collect(Collectors.toList()));
-        return convertEntityToQo(themeEntities, userId);
+        // 排除已经展示过的id
+        List<String> filterEsIds = esIds.stream().map(ESThemeQo::getThemeId).filter(tId -> {
+            return !excludeIds.contains(tId);
+        }).limit(pageSize).collect(Collectors.toList());
+
+        resp.themes = convertEntityToQo(themeService.queryByThemeIds(filterEsIds), userId);
+        resp.excludeIds.addAll(filterEsIds);
+
+        return resp;
     }
 
     @Transactional
@@ -90,8 +103,6 @@ public class ThemeManager {
         themeEntity.setAuthorId(userId);
         themeEntity.setContent(JsonUtil.toJson(req.getContent()));
 
-
-
         if (StringUtils.isEmpty(req.getEditThemeId())) {
             //新建
             themeService.insertTheme(themeEntity);
@@ -108,7 +119,11 @@ public class ThemeManager {
         }
         themeAttachmentService.insertList(themeAttachments);
 
-
+        try {
+            esService.insertOrUpdateTheme(ConvertUtil.convert(themeEntity));
+        } catch (Exception e) {
+            log.error("error in save theme to ES. themeId:{}, error:{}", themeEntity.getThemeId(), ExceptionUtils.getStackTrace(e));
+        }
 
         return CreateThemeResp.builder().themeId(themeEntity.getThemeId()).build();
     }

community-service/src/main/java/com/tanpu/community/model/ESWrapper.java

+package com.tanpu.community.model;
+
+import com.alibaba.fastjson.JSONArray;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+public class ESWrapper<T> {
+    @JsonProperty("_index")
+    public String index;
+    @JsonProperty("_type")
+    public String type;
+    @JsonProperty("_id")
+    public String id;
+    @JsonProperty("_score")
+    public String score;
+    @JsonProperty("_source")
+    public T source;
+    @JsonProperty("sort")
+    public JSONArray sort;
+}

community-service/src/main/java/com/tanpu/community/service/base/ESHelper.java

@@ -49,7 +49,7 @@ public class ESHelper {
 
             req.source(json, XContentType.JSON);
             IndexResponse resp = client.index(req, RequestOptions.DEFAULT);
-            validStatus(resp.status(), RestStatus.CREATED);
+            validStatus(resp.status(), RestStatus.CREATED, RestStatus.OK);
         } catch (IOException e) {
             log.error("ES Helper error:{}", ExceptionUtils.getStackTrace(e));
         }
@@ -64,7 +64,7 @@ public class ESHelper {
 
             req.source(data);
             IndexResponse resp = client.index(req, RequestOptions.DEFAULT);
-            validStatus(resp.status(), RestStatus.CREATED);
+            validStatus(resp.status(), RestStatus.OK);
         } catch (IOException e) {
             log.error("ES Helper error:{}", ExceptionUtils.getStackTrace(e));
         }
@@ -86,12 +86,13 @@ public class ESHelper {
     }
 
 
-    private void validStatus(RestStatus status, RestStatus expect) {
-        if (status != expect) {
+    private void validStatus(RestStatus status, RestStatus... expect) {
+        for (RestStatus ex : expect) {
+            if (ex == status) return;
+        }
         log.error("ES Helper fail! status:{}", status.toString());
         throw new RuntimeException("ES fail");
     }
-    }
 
 
 
@@ -111,11 +112,14 @@ public class ESHelper {
 
 
     public static void main(String[] args) {
+        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("1", "2"));
+
         RestClientBuilder builder = RestClient.builder(new HttpHost("42.194.224.208", 9200))
                 .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                     @Override
                     public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
-                        return httpClientBuilder;
+                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                     }
                 });
 
@@ -128,29 +132,30 @@ public class ESHelper {
         System.out.println("insert");
 
         Map<String, Object> map = new HashMap<>();
-        map.put("name", "小行星2");
-        map.put("context", "这里有一个小行星2");
-//        helper.insert("test_index", "2", map);
+        map.put("name", "太阳44444444444444");
+        map.put("context", "这里有一个小太阳444444444");
+//        helper.insert("test_index", "", "2", map);
 
 
 
         SearchSourceBuilder search = new SearchSourceBuilder();
 
         BoolQueryBuilder boolQb = QueryBuilders.boolQuery();
-        MatchQueryBuilder matchQb = QueryBuilders.matchQuery("context", "星");
+        MatchQueryBuilder matchQb = QueryBuilders.matchQuery("textContent", "小星星");
         boolQb.must(matchQb);
 
-        String[] includes = new String[]{"id"};
-        String[] excludes = new String[]{};
-        search.query(boolQb).fetchSource(includes, excludes).from(0).size(50);
+//        String[] includes = new String[]{"id"};
+//        String[] excludes = new String[]{};
+        search.query(boolQb).from(0).size(50);
 
 
-        SearchHit[] hits = helper.selectLike("test_index", search);
+        SearchHit[] hits = helper.selectLike("theme", search);
         System.out.println(hits.length);
         for (SearchHit hit : hits) {
             System.out.println(hit.toString());
             System.out.println(hit.getFields());
         }
+        System.out.println("done");
     }
 
 

community-service/src/main/java/com/tanpu/community/service/base/ESService.java

 package com.tanpu.community.service.base;
 
 import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.tanpu.common.util.JsonUtil;
 import com.tanpu.community.api.beans.qo.ESThemeQo;
 import com.tanpu.community.api.beans.qo.ThemeQo;
 import com.tanpu.community.dao.entity.community.ThemeEntity;
+import com.tanpu.community.model.ESWrapper;
+import lombok.extern.slf4j.Slf4j;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -18,6 +22,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 
+@Slf4j
 @Service
 public class ESService {
 
@@ -26,7 +31,13 @@ public class ESService {
 
     private static final String INDEX_THEME = "theme";
 
-    public void insertTheme(ESThemeQo qo) {
+    public void insertOrUpdateThemes(List<ESThemeQo> qos) {
+        for (ESThemeQo qo : qos) {
+            insertOrUpdateTheme(qo);
+        }
+    }
+    // 只要设置了_id，则直接覆盖
+    public void insertOrUpdateTheme(ESThemeQo qo) {
         helper.insert(INDEX_THEME, String.valueOf(qo.themeType), qo.themeId, JSON.toJSONString(qo));
     }
 
@@ -34,7 +45,7 @@ public class ESService {
         SearchSourceBuilder search = new SearchSourceBuilder();
 
         BoolQueryBuilder boolQb = QueryBuilders.boolQuery();
-        MatchQueryBuilder contentQb = QueryBuilders.matchQuery("content", keyword);
+        MatchQueryBuilder contentQb = QueryBuilders.matchQuery("textContent", keyword);
         MatchQueryBuilder titleQb = QueryBuilders.matchQuery("title", keyword);
         boolQb.should(contentQb);
         boolQb.should(titleQb);
@@ -42,10 +53,11 @@ public class ESService {
         String[] includes = new String[]{"id", "themeId", "createTime"};
         String[] excludes = new String[]{};
         search.query(boolQb).fetchSource(includes, excludes).sort("createTime", SortOrder.DESC).from(from).size(size);
+        search.query(boolQb).sort("createTime", SortOrder.DESC).from(from).size(size);
 
         SearchHit[] hits = helper.selectLike(INDEX_THEME, search);
         return Arrays.stream(hits).map(h -> {
-            return JSON.parseObject(h.toString(), ESThemeQo.class);
+            return JsonUtil.toBean(h.getSourceAsString(), ESThemeQo.class);
         }).collect(Collectors.toList());
     }
 }

community-service/src/main/java/com/tanpu/community/util/ConvertUtil.java

@@ -15,6 +15,9 @@ import com.tanpu.community.dao.entity.user.UserInfoEntity;
 import org.springframework.beans.BeanUtils;
 import org.springframework.util.StringUtils;
 
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.temporal.TemporalField;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -74,6 +77,33 @@ public class ConvertUtil {
         return themeEntities.stream().map(ConvertUtil::themeEntityToQo2).collect(Collectors.toList());
     }
 
+    public static ESThemeQo convert(ThemeEntity entity) {
+        ESThemeQo qo = new ESThemeQo();
+        BeanUtils.copyProperties(entity, qo);
+
+        // 抽取文本内容
+        List<ThemeContentQo> themeContentQos = JsonUtil.toBean(entity.getContent(), new TypeReference<List<ThemeContentQo>>() {});
+        StringBuilder sb = new StringBuilder();
+        themeContentQos.stream().filter(q -> {
+            // todo enum
+            return q.getType().equals("108");
+        }).forEach(q -> {
+            sb.append(q.getValue());
+        });
+        qo.textContent = sb.toString();
+
+        Long now = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
+        if (entity.getCreateTime() == null) {
+            qo.setCreateTime(now);
+            qo.setUpdateTime(now);
+        } else {
+            qo.setCreateTime(entity.getCreateTime().toInstant(ZoneOffset.of("+8")).toEpochMilli());
+            qo.setUpdateTime(now);
+        }
+
+        return qo;
+    }
+
 
     public static TopicDTO topicEntityToDTO(TopicEntity topicEntity) {
         TopicDTO topicDTO = new TopicDTO();

community-service/src/main/resources/application.yml

@@ -9,7 +9,7 @@ spring.servlet:
     max-request-size: 5MB
 
 # apollo
-apollo.bootstrap.enabled: true
+apollo.bootstrap.enabled: false
 app.id: tp-community
 
 apollo: