HttpClient4.5 使用http鏈接池發送http請求深度示例

HttpClient 3.x,4.x都提供http鏈接池管理器,當使用了請求鏈接池管理器（好比PoolingHttpClientConnectionManager）後，HttpClient就能夠同時執行多個線程的請求了。

hc3.x和4.x的早期版本,提供了PoolingClientConnectionManager,DefaultHttpClient等類來實現http鏈接池,但這些類在4.3.x版本以後大部分就已通過時,本文使用4.3.x提供的最新的PoolingHttpClientConnectionManager等類進行http鏈接池的實現.
廢話很少說,下面是所有代碼:

public class PoolTest {
    private static void config(HttpRequestBase httpRequestBase) {
        httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");
        httpRequestBase.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5");
        httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7");
        
        // 配置請求的超時設置
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectionRequestTimeout(3000)
                .setConnectTimeout(3000)
                .setSocketTimeout(3000)
                .build();
        httpRequestBase.setConfig(requestConfig);        
    }
    public static void main(String[] args) {
        ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
        LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", plainsf)
                .register("https", sslsf)
                .build();
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
        // 將最大鏈接數增長到200
        cm.setMaxTotal(200);
        // 將每一個路由基礎的鏈接增長到20
        cm.setDefaultMaxPerRoute(20);
        // 將目標主機的最大鏈接數增長到50
        HttpHost localhost = new HttpHost("http://blog.csdn.net/gaolu",80);
        cm.setMaxPerRoute(new HttpRoute(localhost), 50);
        
        //請求重試處理
        HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
            public boolean retryRequest(IOException exception,int executionCount, HttpContext context) {
                if (executionCount >= 5) {// 若是已經重試了5次，就放棄                    
                    return false;
                }
                if (exception instanceof NoHttpResponseException) {// 若是服務器丟掉了鏈接，那麼就重試                    
                    return true;
                }
                if (exception instanceof SSLHandshakeException) {// 不要重試SSL握手異常                    
                    return false;
                }                
                if (exception instanceof InterruptedIOException) {// 超時                    
                    return false;
                }
                if (exception instanceof UnknownHostException) {// 目標服務器不可達                    
                    return false;
                }
                if (exception instanceof ConnectTimeoutException) {// 鏈接被拒絕                    
                    return false;
                }
                if (exception instanceof SSLException) {// ssl握手異常                    
                    return false;
                }
                
                HttpClientContext clientContext = HttpClientContext.adapt(context);
                HttpRequest request = clientContext.getRequest();
                // 若是請求是冪等的，就再次嘗試
                if (!(request instanceof HttpEntityEnclosingRequest)) {                    
                    return true;
                }
                return false;
            }
        };  
        
        CloseableHttpClient httpClient = HttpClients.custom()
                .setConnectionManager(cm)
                .setRetryHandler(httpRequestRetryHandler)
                .build();
        // URL列表數組
        String[] urisToGet = {         
                "http://blog.csdn.net/gaolu/article/details/48466059",
                "http://blog.csdn.net/gaolu/article/details/48243103",
                "http://blog.csdn.net/gaolu/article/details/47656987",
                "http://blog.csdn.net/gaolu/article/details/47055029",
                
                "http://blog.csdn.net/gaolu/article/details/46400883",
                "http://blog.csdn.net/gaolu/article/details/46359127",
                "http://blog.csdn.net/gaolu/article/details/46224821",
                "http://blog.csdn.net/gaolu/article/details/45305769",
                
                "http://blog.csdn.net/gaolu/article/details/43701763",
                "http://blog.csdn.net/gaolu/article/details/43195449",
                "http://blog.csdn.net/gaolu/article/details/42915521",
                "http://blog.csdn.net/gaolu/article/details/41802319",
                
                "http://blog.csdn.net/gaolu/article/details/41045233",
                "http://blog.csdn.net/gaolu/article/details/40395425",
                "http://blog.csdn.net/gaolu/article/details/40047065",
                "http://blog.csdn.net/gaolu/article/details/39891877",
                
                "http://blog.csdn.net/gaolu/article/details/39499073",
                "http://blog.csdn.net/gaolu/article/details/39314327",
                "http://blog.csdn.net/gaolu/article/details/38820809",
                "http://blog.csdn.net/gaolu/article/details/38439375",
        };
        
        long start = System.currentTimeMillis();        
        try {
            int pagecount = urisToGet.length;
            ExecutorService executors = Executors.newFixedThreadPool(pagecount);
            CountDownLatch countDownLatch = new CountDownLatch(pagecount);         
            for(int i = 0; i< pagecount;i++){
                HttpGet httpget = new HttpGet(urisToGet[i]);
                config(httpget);
                //啓動線程抓取
                executors.execute(new GetRunnable(httpClient,httpget,countDownLatch));
            }
            countDownLatch.await();
            executors.shutdown();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            System.out.println("線程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 全部線程已完成，開始進入下一步！");
        }        
        
        long end = System.currentTimeMillis();
        System.out.println("consume -> " + (end - start));
    }
    
    static class GetRunnable implements Runnable {
        private CountDownLatch countDownLatch;
        private final CloseableHttpClient httpClient;
        private final HttpGet httpget;
        
        public GetRunnable(CloseableHttpClient httpClient, HttpGet httpget, CountDownLatch countDownLatch){
            this.httpClient = httpClient;
            this.httpget = httpget;
            
            this.countDownLatch = countDownLatch;
        }
        @Override
        public void run() {
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpget,HttpClientContext.create());
                HttpEntity entity = response.getEntity();
                System.out.println(EntityUtils.toString(entity, "utf-8")) ;
                EntityUtils.consume(entity);
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                countDownLatch.countDown();
                
                try {
                    if(response != null)
                        response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }    
}

主要參考文檔:
http://free0007.iteye.com/blog/2012308