Elasticsearch Sliced Scroll分頁檢索案例分享

<div class="content" id="articleContent"> <div class="ad-wrap" xongsyz="" hidden=""> <a data-traceid="blog_detail_above_text_link_1" data-tracepid="blog_detail_above_text_link" style="color:#A00;font-weight:bold;" href="https://my.oschina.net/u/2663968/blog/3041208" target="_blank" xongsyz="" hidden="">面試：你懂什麼是分佈式系統嗎？Redis分佈式鎖都不會？>>> </a>  <img src="https://www.oschina.net/img/hot3.png" align="absmiddle" style="max-height: 32px; max-width: 32px;"> </div> The best elasticsearch highlevel java rest api-----<a href="https://www.oschina.net/p/bboss-elastic" target="_blank" rel="nofollow">bboss</a>  Elasticsearch Sliced Scroll分頁檢索案例分享  咱們在文章《<a href="https://my.oschina.net/bboss/blog/1786493" target="_blank" rel="nofollow">Elasticsearch Scroll分頁檢索案例分享</a>》中介紹了elasticsearch scroll的基本用法，本文介紹Elasticsearch Sliced Scroll分頁檢索功能。 <h1 id="h1_1">1.準備工做</h1> 參考文檔《<a href="https://my.oschina.net/bboss/blog/1556866" target="_blank" rel="nofollow">高性能elasticsearch ORM開發庫使用介紹</a>》導入和配置es客戶端 <h1 id="h1_2">2.定義Sliced Scroll檢索dsl</h1> 建立配置文件-在resources目錄下定義文件scroll.xml <pre>esmapper/scroll.xml</pre> 文件內容包含Sliced Scroll檢索dsl語句-scrollSliceQuery <pre><code class="hljs xml"><property name="scrollSliceQuery"> <![CDATA[ { "slice": { "id": $id, "max": $max }, "size":$size, "query": { "term" : { "gc.jvmGcOldCount" : 3 } } } ]]> </property></code></pre> <h1 id="h1_3">3.串行方式執行slice檢索</h1> <pre><code class="hljs java">/** * 串行方式執行slice scroll操做 */ @Test public void testSliceScroll() { ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml"); List<String> scrollIds = new ArrayList<>(); long starttime = System.currentTimeMillis(); //scroll slice分頁檢索 int max = 6; long realTotalSize = 0; for (int i = 0; i < max; i++) { Map params = new HashMap(); params.put("id", i); params.put("max", max);//最多6個slice，不能大於share數 params.put("size", 100);//每頁100條記錄 ESDatas<Map> sliceResponse = clientUtil.searchList("agentstat-*/_search?scroll=1m", "scrollSliceQuery", params,Map.class); List<Map> sliceDatas = sliceResponse.getDatas(); realTotalSize = realTotalSize + sliceDatas.size(); long totalSize = sliceResponse.getTotalSize(); String scrollId = sliceResponse.getScrollId(); if (scrollId != null) scrollIds.add(scrollId); System.out.println("totalSize:" + totalSize); System.out.println("scrollId:" + scrollId); if (sliceDatas != null && sliceDatas.size() >= 100) {//每頁100條記錄，迭代scrollid，遍歷scroll分頁結果 do { sliceResponse = clientUtil.searchScroll("1m", scrollId, Map.class); String sliceScrollId = sliceResponse.getScrollId(); if (sliceScrollId != null) scrollIds.add(sliceScrollId); sliceDatas = sliceResponse.getDatas(); if (sliceDatas == null || sliceDatas.size() < 100) { break; } realTotalSize = realTotalSize + sliceDatas.size(); } while (true); } } //打印處理耗時和實際檢索到的數據 long endtime = System.currentTimeMillis(); System.out.println("耗時："+(endtime - starttime)+",realTotalSize："+realTotalSize); //查詢存在es服務器上的scroll上下文信息 String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET); System.out.println(scrolls); //處理完畢後清除scroll上下文信息 if(scrollIds.size() > 0) { scrolls = clientUtil.deleteScrolls(scrollIds); System.out.println(scrolls); } //清理完畢後查看scroll上下文信息 scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET); System.out.println(scrolls); }</code></pre> <h1 id="h1_4">4.並行方式執行slice檢索</h1> <pre><code class="hljs java">//用來存放實際slice檢索總記錄數 long realTotalSize ; //輔助方法，用來累計每次scroll獲取到的記錄數 synchronized void incrementSize(int size){ this.realTotalSize = this.realTotalSize + size; } /** * 並行方式執行slice scroll操做 */ @Test public void testParralSliceScroll() { final ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml"); final List<String> scrollIds = new ArrayList<>(); long starttime = System.currentTimeMillis(); //scroll slice分頁檢索 final int max = 6; final CountDownLatch countDownLatch = new CountDownLatch(max);//線程任務完成計數器，每一個線程對應一個sclice,每運行完一個slice任務,countDownLatch計數減去1html

<span class="hljs-keyword">for</span> (<span class="hljs-keyword">int</span> j = <span class="hljs-number">0</span>; j &lt; max; j++) {<span class="hljs-comment">//啓動max個線程，並行處理每一個slice任務</span>
	<span class="hljs-keyword">final</span> <span class="hljs-keyword">int</span> i = j;
	Thread sliceThread = <span class="hljs-keyword">new</span> Thread(<span class="hljs-keyword">new</span> Runnable() {<span class="hljs-comment">//多線程並行執行scroll操做作，每一個線程對應一個sclice</span>

		<span class="hljs-meta">@Override</span>
		<span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title">run</span><span class="hljs-params">()</span> </span>{
			Map params = <span class="hljs-keyword">new</span> HashMap();
			params.put(<span class="hljs-string">"id"</span>, i);
			params.put(<span class="hljs-string">"max"</span>, max);<span class="hljs-comment">//最多6個slice，不能大於share數</span>
			params.put(<span class="hljs-string">"size"</span>, <span class="hljs-number">100</span>);<span class="hljs-comment">//每頁100條記錄</span>
			ESDatas&lt;Map&gt; sliceResponse = clientUtil.searchList(<span class="hljs-string">"agentstat-*/_search?scroll=1m"</span>,
					<span class="hljs-string">"scrollSliceQuery"</span>, params,Map.class);
			List&lt;Map&gt; sliceDatas = sliceResponse.getDatas();
			incrementSize( sliceDatas.size());<span class="hljs-comment">//統計實際處理的文檔數量</span>
			<span class="hljs-keyword">long</span> totalSize = sliceResponse.getTotalSize();
			String scrollId = sliceResponse.getScrollId();
			<span class="hljs-keyword">if</span> (scrollId != <span class="hljs-keyword">null</span>)
				scrollIds.add(scrollId);
			System.out.println(<span class="hljs-string">"totalSize:"</span> + totalSize);
			System.out.println(<span class="hljs-string">"scrollId:"</span> + scrollId);
			<span class="hljs-keyword">if</span> (sliceDatas != <span class="hljs-keyword">null</span> &amp;&amp; sliceDatas.size() &gt;= <span class="hljs-number">100</span>) {<span class="hljs-comment">//每頁100條記錄，迭代scrollid，遍歷scroll分頁結果</span>
				<span class="hljs-keyword">do</span> {
					sliceResponse = clientUtil.searchScroll(<span class="hljs-string">"1m"</span>, scrollId, Map.class);
					String sliceScrollId = sliceResponse.getScrollId();
					<span class="hljs-keyword">if</span> (sliceScrollId != <span class="hljs-keyword">null</span>)
						scrollIds.add(sliceScrollId);
					sliceDatas = sliceResponse.getDatas();
					<span class="hljs-keyword">if</span> (sliceDatas == <span class="hljs-keyword">null</span> || sliceDatas.size() &lt; <span class="hljs-number">100</span>) {
						<span class="hljs-keyword">break</span>;
					}
					incrementSize( sliceDatas.size());<span class="hljs-comment">//統計實際處理的文檔數量</span>
				} <span class="hljs-keyword">while</span> (<span class="hljs-keyword">true</span>);
			}
			countDownLatch.countDown();<span class="hljs-comment">//slice檢索完畢後計數器減1</span>
		}

	});
	sliceThread.start();<span class="hljs-comment">//啓動線程</span>
}
<span class="hljs-keyword">try</span> {
	countDownLatch.await();<span class="hljs-comment">//等待全部的線程執行完畢,計數器變成0</span>
} <span class="hljs-keyword">catch</span> (InterruptedException e) {
	e.printStackTrace();
}
  <span class="hljs-comment">//打印處理耗時和實際檢索到的數據</span>
<span class="hljs-keyword">long</span> endtime = System.currentTimeMillis();
System.out.println(<span class="hljs-string">"耗時："</span>+(endtime - starttime)+<span class="hljs-string">",realTotalSize："</span>+realTotalSize);
<span class="hljs-comment">//查詢存在es服務器上的scroll上下文信息</span>
String scrolls = clientUtil.executeHttp(<span class="hljs-string">"_nodes/stats/indices/search"</span>, ClientUtil.HTTP_GET);

// System.out.println(scrolls); //處理完畢後清除scroll上下文信息 if(scrollIds.size() > 0) { scrolls = clientUtil.deleteScrolls(scrollIds); // System.out.println(scrolls); } //清理完畢後查看scroll上下文信息 scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET); // System.out.println(scrolls); }</code></pre>java

經過串行運行和並行運行結果比較，並行處理的性能要好不少，實際檢索到的文檔數量等價一致。 <h1 id="h1_5">5.參考文檔</h1> <a href="https://www.elastic.co/guide/en/elasticsearch/reference/6.2/search-request-scroll.html" target="_blank" rel="nofollow">https://www.elastic.co/guide/en/elasticsearch/reference/6.2/search-request-scroll.html</a> <h1 id="h1_6">6.開發交流</h1> elasticsearch技術交流羣：166471282 <div class="ad-wrap" style="margin-top: 12px;" xongsyz="" hidden=""> <div id="blog-title-ad" data-traceid="blog_ali" data-tracepid="blogdown" style=""> <script async="" src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>  <ins class="adsbygoogle" style="display:inline-block;width:728px;height:90px" data-ad-client="ca-pub-7090564139599510" data-ad-slot="2321828009" xongsyz="" hidden=""></ins> <script> (adsbygoogle = window.adsbygoogle || []).push({}); </script> </div>node

面試

</div>
                                        </div>