Elasticsearch在springcloud項目中同步數據的運用

一，ES的基本概念

1.什麼是全文搜索引擎：

咱們搜索時按結構化的拼音搜到讀音，而後按其指向的頁數，即可找到咱們的非結構化數據——也即對字的解釋。這種先創建索引，再對索引進行搜索的過程就叫全文檢索(Full-text Search)。

表明就是lucence。Lucene是根據關健字來搜索的文本搜索工具，只能在某個網站內部搜索文本內容，不能跨網站搜索。

對lucence進行簡化能夠採用ES，ES是面向文檔的（document oriented）的，對文檔（注意不是成行成列的數據）進行索引，搜索，排序，過濾。

對比着傳統mysql的概念理解

database對應indexes索引，

table對應types類型，

rows對應doucment文檔，

column對應field字段。

建立，刪除索引，建立映射，增刪改查文檔。

查文檔分爲帶分詞器的query string查詢和不帶索引的term查詢。

二，前期準備

安裝Elasticsearch，頭信息，ik分詞器。

建立索引庫xccourse

建立映射

 

三，logstash將mysql數據庫中的內容同步到ES索引庫中的

課程系統遠程經過feign調用CMS內容管理系統，並在課程系統中完成發佈，並頁面保存到本地，，同時將信息保存在coursepub中，完成課程發佈。

基於logstash同步mysql數據庫中coursepub表的信息到es後臺，能夠在ES後臺「數據瀏覽」看到與coursepub徹底一致的內容。

配置文件mysql.config

 1 input {
 2   stdin {
 3   }
 4   jdbc {
 5   jdbc_connection_string => "jdbc:mysql://localhost:3306/xc_course?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
 6   # the user we wish to excute our statement as
 7   jdbc_user => "root"
 8   jdbc_password => root
 9   # the path to our downloaded jdbc driver  
10   jdbc_driver_library => "E:/soft/apache-maven-3.3.9/repository/mysql/mysql-connector-java/5.1.36/mysql-connector-java-5.1.36.jar"
11   # the name of the driver class for mysql
12   jdbc_driver_class => "com.mysql.jdbc.Driver"
13   jdbc_paging_enabled => "true"
14   jdbc_page_size => "50000"
15   #要執行的sql文件
16   #statement_filepath => "/conf/course.sql"
17   statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
18   #定時配置
19   schedule => "* * * * *"
20   record_last_run => true
21   last_run_metadata_path => "‪E:/xuecheng/es/logstash-6.2.1/config/logstash_metadata"
22   }
23 }
24 
25 
26 output {
27   elasticsearch {
28   #ES的ip地址和端口
29   hosts => "localhost:9200"
30   #hosts => ["localhost:9200","localhost:9202","localhost:9203"]
31   #ES索引庫名稱
32   index => "xc_course"
33   document_id => "%{id}"
34   document_type => "doc"
35   template =>"E:/xuecheng/es/logstash-6.2.1/config/xc_course_template.json"
36   template_name =>"xc_course"
37   template_overwrite =>"true"
38   }
39   stdout {
40  #日誌輸出
41   codec => json_lines
42   }
43 }

21行的logstash_metadata文件

--- 2018-06-30 11:26:00.150000000 Z

啓動logstash

數據瀏覽顯示mysql的數據。

四，搜索微服務普通分級搜索和按照關鍵字查詢，以及按照分類等級查詢，按照分頁查詢

配置類

 ElasticsearchConfig

 1 import org.apache.http.HttpHost;
 2 import org.elasticsearch.client.RestClient;
 3 import org.elasticsearch.client.RestHighLevelClient;
 4 import org.springframework.beans.factory.annotation.Value;
 5 import org.springframework.context.annotation.Bean;
 6 import org.springframework.context.annotation.Configuration;
 7 
 8 /**
 9  * @author Administrator
10  * @version 1.0
11  **/
12 @Configuration
13 public class ElasticsearchConfig {
14 
15     @Value("${xuecheng.elasticsearch.hostlist}")
16     private String hostlist;
17 
18     @Bean
19     public RestHighLevelClient restHighLevelClient(){
20         //解析hostlist配置信息
21         String[] split = hostlist.split(",");
22         //建立HttpHost數組，其中存放es主機和端口的配置信息
23         HttpHost[] httpHostArray = new HttpHost[split.length];
24         for(int i=0;i<split.length;i++){
25             String item = split[i];
26             httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
27         }
28         //建立RestHighLevelClient客戶端
29         return new RestHighLevelClient(RestClient.builder(httpHostArray));
30     }
31 
32     //項目主要使用RestHighLevelClient，對於低級的客戶端暫時不用
33     @Bean
34     public RestClient restClient(){
35         //解析hostlist配置信息
36         String[] split = hostlist.split(",");
37         //建立HttpHost數組，其中存放es主機和端口的配置信息
38         HttpHost[] httpHostArray = new HttpHost[split.length];
39         for(int i=0;i<split.length;i++){
40             String item = split[i];
41             httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
42         }
43         return RestClient.builder(httpHostArray).build();
44     }
45 
46 }

api

 8 @Api(value = "課程搜索",description = "課程搜索",tags = {"課程搜索"})
 9 public interface EsCourseControllerApi {
10 
11     @ApiOperation("課程搜索")
12    QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam);
13 
14 }

controller

 1 @RestController
 2 @RequestMapping("/search/course")
 3 public class EsCourseController implements EsCourseControllerApi {
 4 
 5     @Autowired
 6     private CourseSearchService courseSearchService;
 7 
 8 
 9     @Override
10     @GetMapping("/list/{page}/{size}")
11     public QueryResponseResult<CoursePub> searchCourse(@PathVariable("page") int page, @PathVariable("size")int size, CourseSearchParam courseSearchParam) {
12         return courseSearchService.searchCourse(page,size,courseSearchParam);
13     }
14 }

yml

 1 server:
 2   port: ${port:40100}
 3 spring:
 4   application:
 5     name: xc-search-service
 6 xc:
 7   elasticsearch:
 8     hostlist: ${eshostlist:127.0.0.1:9200} #多個結點中間用逗號分隔
 9     course:
10       index: xc_course
11       type: doc

 

service

  1 import com.xuecheng.filesystem.framework.domain.course.response.CoursePub;
  2 import com.xuecheng.filesystem.framework.domain.search.CourseSearchParam;
  3 import com.xuecheng.filesystem.framework.model.response.CommonCode;
  4 import com.xuecheng.filesystem.framework.model.response.QueryResponseResult;
  5 import com.xuecheng.filesystem.framework.model.response.QueryResult;
  6 import org.apache.commons.lang3.StringUtils;
  7 import org.elasticsearch.action.search.SearchRequest;
  8 import org.elasticsearch.action.search.SearchResponse;
  9 import org.elasticsearch.client.RestHighLevelClient;
 10 import org.elasticsearch.common.text.Text;
 11 import org.elasticsearch.index.query.BoolQueryBuilder;
 12 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 13 import org.elasticsearch.index.query.QueryBuilders;
 14 import org.elasticsearch.search.SearchHit;
 15 import org.elasticsearch.search.SearchHits;
 16 import org.elasticsearch.search.builder.SearchSourceBuilder;
 17 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 18 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 19 import org.springframework.beans.factory.annotation.Autowired;
 20 import org.springframework.beans.factory.annotation.Value;
 21 import org.springframework.stereotype.Service;
 22 
 23 import java.io.IOException;
 24 import java.util.ArrayList;
 25 import java.util.List;
 26 import java.util.Map;
 27 
 28 @Service
 29 public class CourseSearchService {
 30 
 31     @Autowired
 32     private RestHighLevelClient restHighLevelClient;
 33 
 34     @Value("${xuecheng.elasticsearch.course.index}")
 35     private String index;
 36 
 37     @Value("${xuecheng.elasticsearch.course.type}")
 38     private String type;
 39 
 40     /**
 41      * 課程搜索
 42      * 關鍵字查詢
 43      * 一級分類，二級分類，難度等級
 44      * 分頁查詢
 45      * 高亮查詢
 46      * @param page
 47      * @param size
 48      * @param courseSearchParam
 49      * @return
 50      */
 51     public QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam) {
 52 
 53         SearchRequest searchRequest = new SearchRequest(index);
 54         searchRequest.types(type);
 55 
 56         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
 57 
 58         //boolQueryBuilder  must  C1  C2 C1 AND C2        or  C1 OR C2
 59         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
 60 
 61         //按照關鍵字查詢
 62         if (StringUtils.isNotEmpty(courseSearchParam.getKeyword())){
 63 
 64             MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(courseSearchParam.getKeyword(), "name", "description", "teachplan");
 65             //查詢權重
 66             multiMatchQueryBuilder.field("name",10);
 67             multiMatchQueryBuilder.minimumShouldMatch("70%");
 68 
 69             boolQueryBuilder.must(multiMatchQueryBuilder);
 70         }
 71 
 72         //過濾查詢&等值查詢
 73         //一級分類查詢
 74         if (StringUtils.isNotEmpty(courseSearchParam.getMt())){
 75             boolQueryBuilder.filter(QueryBuilders.termQuery("mt",courseSearchParam.getMt()));
 76         }
 77 
 78         //二級分類查詢
 79         if (StringUtils.isNotEmpty(courseSearchParam.getSt())){
 80             boolQueryBuilder.filter(QueryBuilders.termQuery("st",courseSearchParam.getSt()));
 81         }
 82 
 83         //難度等級查詢
 84         if (StringUtils.isNotEmpty(courseSearchParam.getGrade())){
 85             boolQueryBuilder.filter(QueryBuilders.termQuery("grade",courseSearchParam.getGrade()));
 86         }
 87 
 88         //分頁查詢
 89         if (page <=0){ //當前頁
 90             page =1;
 91         }
 92         if (size <= 0 ){
 93             size = 10;
 94         }
 95 
 96         int start = (page-1)*size;
 97         //從哪開始查
 98         sourceBuilder.from(start);
 99         //每頁顯示多少
100         sourceBuilder.size(size);
101 
102         //高亮查詢
103         HighlightBuilder highlightBuilder = new HighlightBuilder();
104         //高亮前綴
105         highlightBuilder.preTags("<font class='eslight'>");
106         //高亮後綴
107         highlightBuilder.postTags("</font>");
108         //高亮域
109         highlightBuilder.fields().add(new HighlightBuilder.Field("name"));
110         sourceBuilder.highlighter(highlightBuilder);
111 
112 
113         sourceBuilder.query(boolQueryBuilder);
114 
115         searchRequest.source(sourceBuilder);
116 
117         SearchResponse searchResponse = null;
118         try {
119             searchResponse = restHighLevelClient.search(searchRequest);
120         } catch (IOException e) {
121             e.printStackTrace();
122         }
123         //獲取查詢結果
124         SearchHits hits = searchResponse.getHits();
125         SearchHit[] searchHits = hits.getHits();
126 
127         List<CoursePub> coursePubList = new ArrayList<>();
128 
129         for (SearchHit searchHit : searchHits) {
130 
131             Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
132             CoursePub coursePub = new CoursePub();
133 
134             String id = (String) sourceAsMap.get("id");
135             coursePub.setId(id);
136 
137             //名稱
138             String name = (String) sourceAsMap.get("name");
139             //獲取高亮
140             Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
141             if (highlightFields != null){
142                 HighlightField highlightField = highlightFields.get("name");
143                 if (highlightField != null){
144                     Text[] fragments = highlightField.fragments();
145                     if (fragments !=null){
146                         StringBuffer stringBuffer = new StringBuffer();
147                         for (Text fragment : fragments) {
148                             stringBuffer.append(fragment);
149                         }
150                         name = stringBuffer.toString();
151                     }
152                 }
153                 coursePub.setName(name);
154             }
155 
156 
157             //圖片
158             String pic = (String) sourceAsMap.get("pic");
159             coursePub.setPic(pic);
160 
161             //價格
162             Float price = null;
163             if (sourceAsMap.get("price") != null){
164                 price = Float.parseFloat(String.valueOf(sourceAsMap.get("price")));
165             }
166             coursePub.setPrice(price);
167 
168             //原價
169             Float price_old = null;
170             if (sourceAsMap.get("price_old")!=null){
171                 price_old = Float.parseFloat(String.valueOf(sourceAsMap.get("price_old")));
172             }
173             coursePub.setPrice_old(price_old);
174 
175             coursePubList.add(coursePub);
176 
177         }
178 
179         //數據封裝
180         QueryResult queryResult = new QueryResult();
181         queryResult.setTotal(hits.getTotalHits());
182         queryResult.setList(coursePubList);
183         return new QueryResponseResult<CoursePub>(CommonCode.SUCCESS,queryResult);
184     }
185 }

nginx代理轉發，用戶請求/course/search的nginx將請求轉發給nuxt。js服務。nginx在轉發時根據每臺nuxt服務器的負載狀況進行轉發，實現負載均衡

最後實現走進‘’‘課程搜索’能夠進行關鍵字查詢並對關鍵字實現高亮顯示，並採用二級聯動實現分級。