php curl批處理--可控併發異步

一般狀況下 PHP 中的 cURL 是阻塞運行的,就是說建立一個 cURL 請求之後必須等它執行成功或者超時纔會執行下一個請求:API接口訪問通常會首選CURL

在實際項目或者本身編寫小工具(好比新聞聚合,商品價格監控,比價)的過程當中, 一般須要從第3方網站或者API接口獲取數據, 在須要處理1個URL隊列時, 爲了提升性能, 能夠採用cURL提供的curl_multi_*族函數實現簡單的併發.php

<?php
include 'curl.class.php';
function callback($response, $info, $error, $request)
{
    echo 'response:<br>';
    print_r($response);

    echo '<br>' . date("Y-m-d H:i:s") . '&nbsp;&nbsp;&nbsp;<br>';
    echo '<br>' . str_repeat("-", 100) . '<br>';
}

$USER_COOKIE = (!empty($_REQUEST['cookie'])) ? $_REQUEST['cookie'] : file_get_contents("cookie.txt");

$curl = new Curl ("callback");

$data = array(
    array(
        'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=qmr&type=rec_gametime&referfrom=&rt=0.42521539455332336', //秦美人
        'method' => 'POST',
        'post_data' => '',
        'header' => null,
        'options' => array(
            CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=qmr&fenQuNum=3",
            CURLOPT_COOKIE => $USER_COOKIE,
        )
    ),
    array(
        'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=sq&type=rec_gametime&referfrom=&rt=0.42521539455332336', //神曲
        'method' => 'POST',
        'post_data' => '',
        'header' => null,
        'options' => array(
            CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=sq&fenQuNum=41",
            CURLOPT_COOKIE => $USER_COOKIE,
        )
    ),
    array(
        'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=frxz&type=rec_gametime&referfrom=&rt=0.42521539455332336', //凡人修真
        'method' => 'POST',
        'post_data' => '',
        'header' => null,
        'options' => array(
            CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=frxz&fenQuNum=3",
            CURLOPT_COOKIE => $USER_COOKIE,
        )
    ),
    array(
        'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=smxj&type=rec_gametime&referfrom=&rt=0.42521539455332336', //神魔仙界
        'method' => 'POST',
        'post_data' => '',
        'header' => null,
        'options' => array(
            CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=smxj&fenQuNum=2",
            CURLOPT_COOKIE => $USER_COOKIE,
        )
    ),
    array(
        'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=qsqy&type=rec_gametime&referfrom=&rt=0.42521539455332336', //傾世情緣
        'method' => 'POST',
        'post_data' => '',
        'header' => null,
        'options' => array(
            CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=qsqy&fenQuNum=11",
            CURLOPT_COOKIE => $USER_COOKIE,
        )
    ),
);

foreach ($data as $val) {
    $request = new Curl_request ($val ['url'], $val ['method'], $val ['post_data'], $val ['header'], $val ['options']);
    $curl->add($request);
}

$curl->execute();
echo $curl->display_errors();

 

 

使用下來效果很好,沒有反作用,併發數可控,應用之處多多,本身發揮想象吧cookie

<?php
/**
 * cURL批量處理  工具類
 * 
 * @since Version 1.0
 * @author Justmepzy <justmepzy@gmail.com>
 * @link http://t.qq.com/JustPzy
 */


/**
 *單一的請求對象
 */
class Curl_request {
    public $url         = '';
    public $method         = 'GET';
    public $post_data     = null;
    public $headers     = null;
    public $options     = null;
    /**
     * 
     * @param string $url
     * @param string $method
     * @param string $post_data
     * @param string $headers
     * @param array $options
     * @return void
     */
    public function __construct($url, $method = 'GET', $post_data = null, $headers = null, $options = null) {
        $this->url = $url;
        $this->method = strtoupper( $method );
        $this->post_data = $post_data;
        $this->headers = $headers;
        $this->options = $options;
    }
    /**
     * @return void
     */
    public function __destruct() {
        unset ( $this->url, $this->method, $this->post_data, $this->headers, $this->options );
    }
}
/**
 * 包含請求列隊處理
 */
class Curl {
    /**
     * 請求url個數
     * @var int
     */
    private $size             = 5;
    /**
     * 等待全部cURL批處理中的活動鏈接等待響應時間
     * @var int
     */
    private $timeout         = 5;
    /**
     * 完成請求回調函數
     * @var string
     */
    private $callback         = null;
    /**
     * cRUL配置
     * @var array
     */
    private $options         = array (CURLOPT_SSL_VERIFYPEER => 0,CURLOPT_RETURNTRANSFER => 1,CURLOPT_CONNECTTIMEOUT => 30 );
    /**
     * 請求頭
     * @var array
     */
    private $headers         = array ();
    /**
     * 請求列隊
     * @var array
     */
    private $requests         = array ();
    /**
     * 請求列隊索引
     * @var array
     */
    private $request_map     = array ();
    /**
     * 錯誤
     * @var array
     */
    private $errors         = array ();
    /**
     * @access public
     * @param string $callback 回調函數
     * 該函數有4個參數($response,$info,$error,$request)
     * $response    url返回的body
     * $info        cURL鏈接資源句柄的信息
     * $error        錯誤
     * $request        請求對象
     */
    public function __construct($callback = null) {
        $this->callback = $callback;
    }
    /**
     * 添加一個請求對象到列隊
     * @access public
     * @param object $request
     * @return boolean
     */
    public function add($request) {
        $this->requests [] = $request;
        return TRUE;
    }
    /**
     * 建立一個請求對象並添加到列隊
     * @access public
     * @param string $url
     * @param string $method
     * @param string $post_data
     * @param string $headers
     * @param array $options
     * @return boolean
     */
    public function request($url, $method = 'GET', $post_data = null, $headers = null, $options = null) {
        $this->requests [] = new Curl_request ( $url, $method, $post_data, $headers, $options );
        return TRUE;
    }
    /**
     * 建立GET請求對象
     * @access public
     * @param string $url
     * @param string $headers
     * @param array $options
     * @return boolean
     */
    public function get($url, $headers = null, $options = null) {
        return $this->request ( $url, "GET", null, $headers, $options );
    }
    /**
     * 建立一個POST請求對象
     * @access public
     * @param string $url
     * @param string $post_data
     * @param string $headers
     * @param array $options
     * @return boolean
     */
    public function post($url, $post_data = null, $headers = null, $options = null) {
        return $this->request ( $url, "POST", $post_data, $headers, $options );
    }
    /**
     * 執行cURL
     * @access public
     * @param int $size 最大鏈接數
     * @return Ambigous <boolean, mixed>|boolean
     */
    public function execute($size = null) {
        if (sizeof ( $this->requests ) == 1) {
            return $this->single_curl ();
        } else {
            return $this->rolling_curl ( $size );
        }
    }
    /**
     * 單個url請求
     * @access private
     * @return mixed|boolean
     */
    private function single_curl() {
        $ch = curl_init ();
        $request = array_shift ( $this->requests );
        $options = $this->get_options ( $request );
        curl_setopt_array ( $ch, $options );
        $output = curl_exec ( $ch );
        $info = curl_getinfo ( $ch );
        
        // it's not neccesary to set a callback for one-off requests
        if ($this->callback) {
            $callback = $this->callback;
            if (is_callable ( $this->callback )) {
                call_user_func ( $callback, $output, $info, $request );
            }
        } else
            return $output;
        return true;
    }
    /**
     * 多個url請求
     * @access private
     * @param int $size 最大鏈接數
     * @return boolean
     */
    private function rolling_curl($size = null) {
        if ($size)
            $this->size = $size;
        else 
            $this->size = count($this->requests);
        if (sizeof ( $this->requests ) < $this->size)
            $this->size = sizeof ( $this->requests );
        if ($this->size < 2)
            $this->set_error ( 'size must be greater than 1' );
        $master = curl_multi_init ();
        //添加cURL鏈接資源句柄到map索引
        for($i = 0; $i < $this->size; $i ++) {
            $ch = curl_init ();
            $options = $this->get_options ( $this->requests [$i] );
            curl_setopt_array ( $ch, $options );
            curl_multi_add_handle ( $master, $ch );
            
            $key = ( string ) $ch;
            $this->request_map [$key] = $i;
        }
        
        $active = $done = null;
        do {
            while ( ($execrun = curl_multi_exec ( $master, $active )) == CURLM_CALL_MULTI_PERFORM )
                ;
            if ($execrun != CURLM_OK)
                break;
            //有一個請求完成則回調
            while ( $done = curl_multi_info_read ( $master ) ) {
                //$done    完成的請求句柄
                $info = curl_getinfo ( $done ['handle'] );//
                $output = curl_multi_getcontent ( $done ['handle'] );//
                $error = curl_error ( $done ['handle'] );//
                
                $this->set_error ( $error );
                
                //調用回調函數,若是存在的話
                $callback = $this->callback;
                if (is_callable ( $callback )) {
                    $key = ( string ) $done ['handle'];
                    $request = $this->requests [$this->request_map [$key]];
                    unset ( $this->request_map [$key] );
                    call_user_func ( $callback, $output, $info, $error, $request );
                }
                curl_close ( $done ['handle'] );
                //從列隊中移除已經完成的request
                curl_multi_remove_handle ( $master, $done ['handle'] );
            }
            //等待全部cURL批處理中的活動鏈接
            if ($active)
                curl_multi_select ( $master, $this->timeout );
        } while ( $active );
        //完成關閉
        curl_multi_close ( $master );
        return true;
    }
    /**
     * 獲取沒得請求對象的cURL配置
     * @access private
     * @param object $request
     * @return array
     */
    private function get_options($request) {
        $options = $this->__get ( 'options' );
        if (ini_get ( 'safe_mode' ) == 'Off' || ! ini_get ( 'safe_mode' )) {
            $options [CURLOPT_FOLLOWLOCATION] = 1;
            $options [CURLOPT_MAXREDIRS] = 5;
        }
        $headers = $this->__get ( 'headers' );
        
        if ($request->options) {
            $options = $request->options + $options;
        }
        
        $options [CURLOPT_URL] = $request->url;
        
        if ($request->post_data && strtolower($request->method) == 'post' ) {
            $options [CURLOPT_POST] = 1;
            $options [CURLOPT_POSTFIELDS] = $request->post_data;
        }
        if ($headers) {
            $options [CURLOPT_HEADER] = 0;
            $options [CURLOPT_HTTPHEADER] = $headers;
        }
        
        return $options;
    }
    /**
     * 設置錯誤信息
     * @access public
     * @param string $msg
     */
    public function set_error($msg) {
        if (! empty ( $msg ))
            $this->errors [] = $msg;
    }
    /**
     * 獲取錯誤信息
     * @access public
     * @param string $open
     * @param string $close
     * @return string
     */
    public function display_errors($open = '<p>', $close = '</p>') {
        $str = '';
        foreach ( $this->errors as $val ) {
            $str .= $open . $val . $close;
        }
        return $str;
    }
    /**
     * @access public
     * @param string $name
     * @param string $value
     * @return boolean
     */
    public function __set($name, $value) {
        if ($name == 'options' || $name == 'headers') {
            $this->{$name} = $value + $this->{$name};
        } else {
            $this->{$name} = $value;
        }
        return TRUE;
    }
    /**
     * 
     * @param string $name
     * @return mixed
     * @access public
     */
    public function __get($name) {
        return (isset ( $this->{$name} )) ? $this->{$name} : null;
    }
    /**
     * @return void
     * @access public
     */
    public function __destruct() {
        unset ( $this->size, $this->timeout, $this->callback, $this->options, $this->headers, $this->requests, $this->request_map, $this->errors );
    }
}
// END Curl Class
/* End of file curl.class.php */

 

http://hudeyong926.iteye.com/blog/1635386併發

相關文章
相關標籤/搜索