爬蟲性能相關 |
在編寫爬蟲時,性能的消耗主要在IO請求中,當單進程單線程模式下請求URL時必然會引發等待,從而使得請求總體變慢。html
1 import requests 2 3 def fetch_async(url): 4 response = requests.get(url) 5 return response 6 7 8 url_list = ['http://www.github.com', 'http://www.bing.com'] 9 10 for url in url_list: 11 fetch_async(url) 12 13 1.同步執行
1 from concurrent.futures import ThreadPoolExecutor 2 import requests 3 4 5 def fetch_async(url): 6 response = requests.get(url) 7 return response 8 9 10 url_list = ['http://www.github.com', 'http://www.bing.com'] 11 pool = ThreadPoolExecutor(5) 12 for url in url_list: 13 pool.submit(fetch_async, url) 14 pool.shutdown(wait=True)
1 from concurrent.futures import ThreadPoolExecutor 2 import requests 3 4 def fetch_async(url): 5 response = requests.get(url) 6 return response 7 8 9 def callback(future): 10 print(future.result()) 11 12 13 url_list = ['http://www.github.com', 'http://www.bing.com'] 14 pool = ThreadPoolExecutor(5) 15 for url in url_list: 16 v = pool.submit(fetch_async, url) 17 v.add_done_callback(callback) 18 pool.shutdown(wait=True)
1 from concurrent.futures import ProcessPoolExecutor 2 import requests 3 4 def fetch_async(url): 5 response = requests.get(url) 6 return response 7 8 9 url_list = ['http://www.github.com', 'http://www.bing.com'] 10 pool = ProcessPoolExecutor(5) 11 for url in url_list: 12 pool.submit(fetch_async, url) 13 pool.shutdown(wait=True)
from concurrent.futures import ProcessPoolExecutor import requests def fetch_async(url): response = requests.get(url) return response def callback(future): print(future.result()) url_list = ['http://www.github.com', 'http://www.bing.com'] pool = ProcessPoolExecutor(5) for url in url_list: v = pool.submit(fetch_async, url) v.add_done_callback(callback) pool.shutdown(wait=True)
經過上述代碼都可以完成對請求性能的提升,對於多線程和多進行的缺點是在IO阻塞時會形成了線程和進程的浪費,因此異步IO回事首選:python
import asyncio @asyncio.coroutine def func1(): print('before...func1......') yield from asyncio.sleep(5) print('end...func1......') tasks = [func1(), func1()] loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) loop.close()
import asyncio @asyncio.coroutine def fetch_async(host, url='/'): print(host, url) reader, writer = yield from asyncio.open_connection(host, 80) request_header_content = """GET %s HTTP/1.0\r\nHost: %s\r\n\r\n""" % (url, host,) request_header_content = bytes(request_header_content, encoding='utf-8') writer.write(request_header_content) yield from writer.drain() text = yield from reader.read() print(host, url, text) writer.close() tasks = [ fetch_async('www.cnblogs.com', '/wupeiqi/'), fetch_async('dig.chouti.com', '/pic/show?nid=4073644713430508&lid=10273091') ] loop = asyncio.get_event_loop() results = loop.run_until_complete(asyncio.gather(*tasks)) loop.close()
import aiohttp import asyncio @asyncio.coroutine def fetch_async(url): print(url) response = yield from aiohttp.request('GET', url) # data = yield from response.read() # print(url, data) print(url, response) response.close() tasks = [fetch_async('http://www.google.com/'), fetch_async('http://www.chouti.com/')] event_loop = asyncio.get_event_loop() results = event_loop.run_until_complete(asyncio.gather(*tasks)) event_loop.close()
import asyncio import requests @asyncio.coroutine def fetch_async(func, *args): loop = asyncio.get_event_loop() future = loop.run_in_executor(None, func, *args) response = yield from future print(response.url, response.content) tasks = [ fetch_async(requests.get, 'http://www.cnblogs.com/wupeiqi/'), fetch_async(requests.get, 'http://dig.chouti.com/pic/show?nid=4073644713430508&lid=10273091') ] loop = asyncio.get_event_loop() results = loop.run_until_complete(asyncio.gather(*tasks)) loop.close()
import gevent import requests from gevent import monkey monkey.patch_all() def fetch_async(method, url, req_kwargs): print(method, url, req_kwargs) response = requests.request(method=method, url=url, **req_kwargs) print(response.url, response.content) # ##### 發送請求 ##### gevent.joinall([ gevent.spawn(fetch_async, method='get', url='https://www.python.org/', req_kwargs={}), gevent.spawn(fetch_async, method='get', url='https://www.yahoo.com/', req_kwargs={}), gevent.spawn(fetch_async, method='get', url='https://github.com/', req_kwargs={}), ]) # ##### 發送請求(協程池控制最大協程數量) ##### # from gevent.pool import Pool # pool = Pool(None) # gevent.joinall([ # pool.spawn(fetch_async, method='get', url='https://www.python.org/', req_kwargs={}), # pool.spawn(fetch_async, method='get', url='https://www.yahoo.com/', req_kwargs={}), # pool.spawn(fetch_async, method='get', url='https://www.github.com/', req_kwargs={}), # ])
import grequests request_list = [ grequests.get('http://httpbin.org/delay/1', timeout=0.001), grequests.get('http://fakedomain/'), grequests.get('http://httpbin.org/status/500') ] # ##### 執行並獲取響應列表 ##### # response_list = grequests.map(request_list) # print(response_list) # ##### 執行並獲取響應列表(處理異常) ##### # def exception_handler(request, exception): # print(request,exception) # print("Request failed") # response_list = grequests.map(request_list, exception_handler=exception_handler) # print(response_list)
1 from twisted.web.client import getPage, defer 2 from twisted.internet import reactor 3 4 5 def all_done(arg): 6 reactor.stop() 7 8 9 def callback(contents): 10 print(contents) 11 12 13 deferred_list = [] 14 15 url_list = ['http://www.bing.com', 'http://www.baidu.com', ] 16 for url in url_list: 17 deferred = getPage(bytes(url, encoding='utf8')) 18 deferred.addCallback(callback) 19 deferred_list.append(deferred) 20 21 dlist = defer.DeferredList(deferred_list) 22 dlist.addBoth(all_done) 23 24 reactor.run()
1 from tornado.httpclient import AsyncHTTPClient 2 from tornado.httpclient import HTTPRequest 3 from tornado import ioloop 4 5 6 def handle_response(response): 7 """ 8 處理返回值內容(須要維護計數器,來中止IO循環),調用 ioloop.IOLoop.current().stop() 9 :param response: 10 :return: 11 """ 12 if response.error: 13 print("Error:", response.error) 14 else: 15 print(response.body) 16 17 18 def func(): 19 url_list = [ 20 'http://www.baidu.com', 21 'http://www.bing.com', 22 ] 23 for url in url_list: 24 print(url) 25 http_client = AsyncHTTPClient() 26 http_client.fetch(HTTPRequest(url), handle_response) 27 28 29 ioloop.IOLoop.current().add_callback(func) 30 ioloop.IOLoop.current().start()
1 from twisted.internet import reactor 2 from twisted.web.client import getPage 3 import urllib.parse 4 5 6 def one_done(arg): 7 print(arg) 8 reactor.stop() 9 10 post_data = urllib.parse.urlencode({'check_data': 'adf'}) 11 post_data = bytes(post_data, encoding='utf8') 12 headers = {b'Content-Type': b'application/x-www-form-urlencoded'} 13 response = getPage(bytes('http://dig.chouti.com/login', encoding='utf8'), 14 method=bytes('POST', encoding='utf8'), 15 postdata=post_data, 16 cookies={}, 17 headers=headers) 18 response.addBoth(one_done) 19 20 reactor.run()
以上均是Python內置以及第三方模塊提供異步IO請求模塊,使用簡便大大提升效率,而對於異步IO請求的本質則是【非阻塞Socket】+【IO多路複用】:react
1 import select 2 import socket 3 import time 4 5 6 class AsyncTimeoutException(TimeoutError): 7 """ 8 請求超時異常類 9 """ 10 11 def __init__(self, msg): 12 self.msg = msg 13 super(AsyncTimeoutException, self).__init__(msg) 14 15 16 class HttpContext(object): 17 """封裝請求和相應的基本數據""" 18 19 def __init__(self, sock, host, port, method, url, data, callback, timeout=5): 20 """ 21 sock: 請求的客戶端socket對象 22 host: 請求的主機名 23 port: 請求的端口 24 port: 請求的端口 25 method: 請求方式 26 url: 請求的URL 27 data: 請求時請求體中的數據 28 callback: 請求完成後的回調函數 29 timeout: 請求的超時時間 30 """ 31 self.sock = sock 32 self.callback = callback 33 self.host = host 34 self.port = port 35 self.method = method 36 self.url = url 37 self.data = data 38 39 self.timeout = timeout 40 41 self.__start_time = time.time() 42 self.__buffer = [] 43 44 def is_timeout(self): 45 """當前請求是否已經超時""" 46 current_time = time.time() 47 if (self.__start_time + self.timeout) < current_time: 48 return True 49 50 def fileno(self): 51 """請求sockect對象的文件描述符,用於select監聽""" 52 return self.sock.fileno() 53 54 def write(self, data): 55 """在buffer中寫入響應內容""" 56 self.__buffer.append(data) 57 58 def finish(self, exc=None): 59 """在buffer中寫入響應內容完成,執行請求的回調函數""" 60 if not exc: 61 response = b''.join(self.__buffer) 62 self.callback(self, response, exc) 63 else: 64 self.callback(self, None, exc) 65 66 def send_request_data(self): 67 content = """%s %s HTTP/1.0\r\nHost: %s\r\n\r\n%s""" % ( 68 self.method.upper(), self.url, self.host, self.data,) 69 70 return content.encode(encoding='utf8') 71 72 73 class AsyncRequest(object): 74 def __init__(self): 75 self.fds = [] 76 self.connections = [] 77 78 def add_request(self, host, port, method, url, data, callback, timeout): 79 """建立一個要請求""" 80 client = socket.socket() 81 client.setblocking(False) 82 try: 83 client.connect((host, port)) 84 except BlockingIOError as e: 85 pass 86 # print('已經向遠程發送鏈接的請求') 87 req = HttpContext(client, host, port, method, url, data, callback, timeout) 88 self.connections.append(req) 89 self.fds.append(req) 90 91 def check_conn_timeout(self): 92 """檢查全部的請求,是否有已經鏈接超時,若是有則終止""" 93 timeout_list = [] 94 for context in self.connections: 95 if context.is_timeout(): 96 timeout_list.append(context) 97 for context in timeout_list: 98 context.finish(AsyncTimeoutException('請求超時')) 99 self.fds.remove(context) 100 self.connections.remove(context) 101 102 def running(self): 103 """事件循環,用於檢測請求的socket是否已經就緒,從而執行相關操做""" 104 while True: 105 r, w, e = select.select(self.fds, self.connections, self.fds, 0.05) 106 107 if not self.fds: 108 return 109 110 for context in r: 111 sock = context.sock 112 while True: 113 try: 114 data = sock.recv(8096) 115 if not data: 116 self.fds.remove(context) 117 context.finish() 118 break 119 else: 120 context.write(data) 121 except BlockingIOError as e: 122 break 123 except TimeoutError as e: 124 self.fds.remove(context) 125 self.connections.remove(context) 126 context.finish(e) 127 break 128 129 for context in w: 130 # 已經鏈接成功遠程服務器,開始向遠程發送請求數據 131 if context in self.fds: 132 data = context.send_request_data() 133 context.sock.sendall(data) 134 self.connections.remove(context) 135 136 self.check_conn_timeout() 137 138 139 if __name__ == '__main__': 140 def callback_func(context, response, ex): 141 """ 142 :param context: HttpContext對象,內部封裝了請求相關信息 143 :param response: 請求響應內容 144 :param ex: 是否出現異常(若是有異常則值爲異常對象;不然值爲None) 145 :return: 146 """ 147 print(context, response, ex) 148 149 obj = AsyncRequest() 150 url_list = [ 151 {'host': 'www.google.com', 'port': 80, 'method': 'GET', 'url': '/', 'data': '', 'timeout': 5, 152 'callback': callback_func}, 153 {'host': 'www.baidu.com', 'port': 80, 'method': 'GET', 'url': '/', 'data': '', 'timeout': 5, 154 'callback': callback_func}, 155 {'host': 'www.bing.com', 'port': 80, 'method': 'GET', 'url': '/', 'data': '', 'timeout': 5, 156 'callback': callback_func}, 157 ] 158 for item in url_list: 159 print(item) 160 obj.add_request(**item) 161 162 obj.running()
Scrapy |
Scrapy是一個爲了爬取網站數據,提取結構性數據而編寫的應用框架。 其能夠應用在數據挖掘,信息處理或存儲歷史數據等一系列的程序中。
其最初是爲了頁面抓取 (更確切來講, 網絡抓取 )所設計的, 也能夠應用在獲取API所返回的數據(例如 Amazon Associates Web Services ) 或者通用的網絡爬蟲。Scrapy用途普遍,能夠用於數據挖掘、監測和自動化測試。git
Scrapy 使用了 Twisted異步網絡庫來處理網絡通信。總體架構大體以下github
Scrapy主要包括瞭如下組件:web
Scrapy運行流程大概以下:算法
1、scrapy的安裝json
Linux pip3 install scrapy Windows a. pip3 install wheel b. 下載twisted http://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted c. 進入下載目錄,執行 pip3 install Twisted‑17.1.0‑cp35‑cp35m‑win_amd64.whl d. pip3 install scrapy e. 下載並安裝pywin32:https://sourceforge.net/projects/pywin32/files/
2、scrapy的基本使用windows
一、基本命令緩存
1. scrapy startproject 項目名稱 - 在當前目錄中建立中建立一個項目文件(相似於Django) 2. scrapy genspider [-t template] <name> <domain> - 建立爬蟲應用 如: scrapy gensipider -t basic oldboy oldboy.com scrapy gensipider -t xmlfeed autohome autohome.com.cn PS: 查看全部命令:scrapy gensipider -l 查看模板命令:scrapy gensipider -d 模板名稱 3. scrapy list - 展現爬蟲應用列表 4. scrapy crawl 爬蟲應用名稱 - 運行單獨爬蟲應用
二、項目結構及其爬蟲應用簡介
project_name/ scrapy.cfg project_name/ __init__.py items.py pipelines.py settings.py spiders/ __init__.py 爬蟲1.py 爬蟲2.py 爬蟲3.py
文件說明:
注意:通常建立爬蟲文件時,以網站域名命名
import scrapy class XiaoHuarSpider(scrapy.spiders.Spider): name = "xiaohuar" # 爬蟲名稱 ***** allowed_domains = ["xiaohuar.com"] # 容許的域名 start_urls = [ "http://www.xiaohuar.com/hua/", # 其實URL ] def parse(self, response): # 訪問起始URL並獲取結果後的回調函數
關於windows的編碼問題 import sys,os sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')
3、開始項目
1 import scrapy 2 from scrapy.selector import HtmlXPathSelector 3 from scrapy.http.request import Request 4 5 6 class DigSpider(scrapy.Spider): 7 # 爬蟲應用的名稱,經過此名稱啓動爬蟲命令 8 name = "dig" 9 10 # 容許的域名 11 allowed_domains = ["chouti.com"] 12 13 # 起始URL 14 start_urls = [ 15 'http://dig.chouti.com/', 16 ] 17 18 has_request_set = {} 19 20 def parse(self, response): 21 print(response.url) 22 23 hxs = HtmlXPathSelector(response) 24 page_list = hxs.select('//div[@id="dig_lcpage"]//a[re:test(@href, "/all/hot/recent/\d+")]/@href').extract() 25 for page in page_list: 26 page_url = 'http://dig.chouti.com%s' % page 27 key = self.md5(page_url) 28 if key in self.has_request_set: 29 pass 30 else: 31 self.has_request_set[key] = page_url 32 obj = Request(url=page_url, method='GET', callback=self.parse) 33 yield obj 34 35 @staticmethod 36 def md5(val): 37 import hashlib 38 ha = hashlib.md5() 39 ha.update(bytes(val, encoding='utf-8')) 40 key = ha.hexdigest() 41 return key
執行此爬蟲文件,則在終端進入項目目錄執行以下命令:
scrapy crawl dig --nolog
對於上述代碼重要之處在於:
4、scrapy的選擇器
#!/usr/bin/env python # -*- coding:utf-8 -*- from scrapy.selector import Selector, HtmlXPathSelector from scrapy.http import HtmlResponse html = """<!DOCTYPE html> <html> <head lang="en"> <meta charset="UTF-8"> <title></title> </head> <body> <ul> <li class="item-"><a id='i1' href="link.html">first item</a></li> <li class="item-0"><a id='i2' href="llink.html">first item</a></li> <li class="item-1"><a href="llink2.html">second item<span>vv</span></a></li> </ul> <div><a href="llink2.html">second item</a></div> </body> </html> """ response = HtmlResponse(url='http://example.com', body=html,encoding='utf-8') # hxs = HtmlXPathSelector(response) # print(hxs) # hxs = Selector(response=response).xpath('//a') # print(hxs) # hxs = Selector(response=response).xpath('//a[2]') # print(hxs) # hxs = Selector(response=response).xpath('//a[@id]') # print(hxs) # hxs = Selector(response=response).xpath('//a[@id="i1"]') # print(hxs) # hxs = Selector(response=response).xpath('//a[@href="link.html"][@id="i1"]') # print(hxs) # hxs = Selector(response=response).xpath('//a[contains(@href, "link")]') # print(hxs) # hxs = Selector(response=response).xpath('//a[starts-with(@href, "link")]') # print(hxs) # hxs = Selector(response=response).xpath('//a[re:test(@id, "i\d+")]') # print(hxs) # hxs = Selector(response=response).xpath('//a[re:test(@id, "i\d+")]/text()').extract() # print(hxs) # hxs = Selector(response=response).xpath('//a[re:test(@id, "i\d+")]/@href').extract() # print(hxs) # hxs = Selector(response=response).xpath('/html/body/ul/li/a/@href').extract() # print(hxs) # hxs = Selector(response=response).xpath('//body/ul/li/a/@href').extract_first() # print(hxs) # ul_list = Selector(response=response).xpath('//body/ul/li') # for item in ul_list: # v = item.xpath('./a/span') # # 或 # # v = item.xpath('a/span') # # 或 # # v = item.xpath('*/a/span') # print(v)
# -*- coding: utf-8 -*- import scrapy from scrapy.selector import HtmlXPathSelector from scrapy.http.request import Request from scrapy.http.cookies import CookieJar from scrapy import FormRequest class ChouTiSpider(scrapy.Spider): # 爬蟲應用的名稱,經過此名稱啓動爬蟲命令 name = "chouti" # 容許的域名 allowed_domains = ["chouti.com"] cookie_dict = {} has_request_set = {} def start_requests(self): url = 'http://dig.chouti.com/' # return [Request(url=url, callback=self.login)] yield Request(url=url, callback=self.login) def login(self, response): cookie_jar = CookieJar() cookie_jar.extract_cookies(response, response.request) for k, v in cookie_jar._cookies.items(): for i, j in v.items(): for m, n in j.items(): self.cookie_dict[m] = n.value req = Request( url='http://dig.chouti.com/login', method='POST', headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'}, body='phone=8615131255089&password=pppppppp&oneMonth=1', cookies=self.cookie_dict, callback=self.check_login ) yield req def check_login(self, response): req = Request( url='http://dig.chouti.com/', method='GET', callback=self.show, cookies=self.cookie_dict, dont_filter=True ) yield req def show(self, response): # print(response) hxs = HtmlXPathSelector(response) news_list = hxs.select('//div[@id="content-list"]/div[@class="item"]') for new in news_list: # temp = new.xpath('div/div[@class="part2"]/@share-linkid').extract() link_id = new.xpath('*/div[@class="part2"]/@share-linkid').extract_first() yield Request( url='http://dig.chouti.com/link/vote?linksId=%s' %(link_id,), method='POST', cookies=self.cookie_dict, callback=self.do_favor ) page_list = hxs.select('//div[@id="dig_lcpage"]//a[re:test(@href, "/all/hot/recent/\d+")]/@href').extract() for page in page_list: page_url = 'http://dig.chouti.com%s' % page import hashlib hash = hashlib.md5() hash.update(bytes(page_url,encoding='utf-8')) key = hash.hexdigest() if key in self.has_request_set: pass else: self.has_request_set[key] = page_url yield Request( url=page_url, method='GET', callback=self.show ) def do_favor(self, response): print(response.text)
注意:settings.py中設置DEPTH_LIMIT = 1來指定「遞歸」的層數。
5、格式化處理
上述實例只是簡單的處理,因此在parse方法中直接處理。若是對於想要獲取更多的數據處理,則能夠利用Scrapy的items將數據格式化,而後統一交由pipelines來處理。
import scrapy from scrapy.selector import HtmlXPathSelector from scrapy.http.request import Request from scrapy.http.cookies import CookieJar from scrapy import FormRequest class XiaoHuarSpider(scrapy.Spider): # 爬蟲應用的名稱,經過此名稱啓動爬蟲命令 name = "xiaohuar" # 容許的域名 allowed_domains = ["xiaohuar.com"] start_urls = [ "http://www.xiaohuar.com/list-1-1.html", ] # custom_settings = { # 'ITEM_PIPELINES':{ # 'spider1.pipelines.JsonPipeline': 100 # } # } has_request_set = {} def parse(self, response): # 分析頁面 # 找到頁面中符合規則的內容(校花圖片),保存 # 找到全部的a標籤,再訪問其餘a標籤,一層一層的搞下去 hxs = HtmlXPathSelector(response) items = hxs.select('//div[@class="item_list infinite_scroll"]/div') for item in items: src = item.select('.//div[@class="img"]/a/img/@src').extract_first() name = item.select('.//div[@class="img"]/span/text()').extract_first() school = item.select('.//div[@class="img"]/div[@class="btns"]/a/text()').extract_first() url = "http://www.xiaohuar.com%s" % src from ..items import XiaoHuarItem obj = XiaoHuarItem(name=name, school=school, url=url) yield obj urls = hxs.select('//a[re:test(@href, "http://www.xiaohuar.com/list-1-\d+.html")]/@href') for url in urls: key = self.md5(url) if key in self.has_request_set: pass else: self.has_request_set[key] = url req = Request(url=url,method='GET',callback=self.parse) yield req @staticmethod def md5(val): import hashlib ha = hashlib.md5() ha.update(bytes(val, encoding='utf-8')) key = ha.hexdigest() return key
import scrapy class XiaoHuarItem(scrapy.Item): name = scrapy.Field() school = scrapy.Field() url = scrapy.Field()
import json import os import requests class JsonPipeline(object): def __init__(self): self.file = open('xiaohua.txt', 'w') def process_item(self, item, spider): v = json.dumps(dict(item), ensure_ascii=False) self.file.write(v) self.file.write('\n') self.file.flush() return item class FilePipeline(object): def __init__(self): if not os.path.exists('imgs'): os.makedirs('imgs') def process_item(self, item, spider): response = requests.get(item['url'], stream=True) file_name = '%s_%s.jpg' % (item['name'], item['school']) with open(os.path.join('imgs', file_name), mode='wb') as f: f.write(response.content) return item
ITEM_PIPELINES = { 'spider1.pipelines.JsonPipeline': 100, 'spider1.pipelines.FilePipeline': 300, } # 每行後面的整型值,肯定了他們運行的順序,item按數字從低到高的順序,經過pipeline,一般將這些數字定義在0-1000範圍內。
對於pipeline能夠作更多,以下:
from scrapy.exceptions import DropItem class CustomPipeline(object): def __init__(self,v): self.value = v def process_item(self, item, spider): # 操做並進行持久化 # return表示會被後續的pipeline繼續處理 return item # 表示將item丟棄,不會被後續pipeline處理 # raise DropItem() @classmethod def from_crawler(cls, crawler): """ 初始化時候,用於建立pipeline對象 :param crawler: :return: """ val = crawler.settings.getint('MMMM') return cls(val) def open_spider(self,spider): """ 爬蟲開始執行時,調用 :param spider: :return: """ print('000000') def close_spider(self,spider): """ 爬蟲關閉時,被調用 :param spider: :return: """ print('111111')
6、中間件
class SpiderMiddleware(object): def process_spider_input(self,response, spider): """ 下載完成,執行,而後交給parse處理 :param response: :param spider: :return: """ pass def process_spider_output(self,response, result, spider): """ spider處理完成,返回時調用 :param response: :param result: :param spider: :return: 必須返回包含 Request 或 Item 對象的可迭代對象(iterable) """ return result def process_spider_exception(self,response, exception, spider): """ 異常調用 :param response: :param exception: :param spider: :return: None,繼續交給後續中間件處理異常;含 Response 或 Item 的可迭代對象(iterable),交給調度器或pipeline """ return None def process_start_requests(self,start_requests, spider): """ 爬蟲啓動時調用 :param start_requests: :param spider: :return: 包含 Request 對象的可迭代對象 """ return start_requests
class DownMiddleware1(object): def process_request(self, request, spider): """ 請求須要被下載時,通過全部下載器中間件的process_request調用 :param request: :param spider: :return: None,繼續後續中間件去下載; Response對象,中止process_request的執行,開始執行process_response Request對象,中止中間件的執行,將Request從新調度器 raise IgnoreRequest異常,中止process_request的執行,開始執行process_exception """ pass def process_response(self, request, response, spider): """ spider處理完成,返回時調用 :param response: :param result: :param spider: :return: Response 對象:轉交給其餘中間件process_response Request 對象:中止中間件,request會被從新調度下載 raise IgnoreRequest 異常:調用Request.errback """ print('response1') return response def process_exception(self, request, exception, spider): """ 當下載處理器(download handler)或 process_request() (下載中間件)拋出異常 :param response: :param exception: :param spider: :return: None:繼續交給後續中間件處理異常; Response對象:中止後續process_exception方法 Request對象:中止中間件,request將會被從新調用下載 """ return None 下載器中間件
7、自定義命令
from scrapy.commands import ScrapyCommand from scrapy.utils.project import get_project_settings class Command(ScrapyCommand): requires_project = True def syntax(self): return '[options]' def short_desc(self): return 'Runs all of the spiders' def run(self, args, opts): spider_list = self.crawler_process.spiders.list() for name in spider_list: self.crawler_process.crawl(name, **opts.__dict__) self.crawler_process.start()
8、自定義擴展
自定義擴展時,利用信號在指定位置註冊制定操做
from scrapy import signals class MyExtension(object): def __init__(self, value): self.value = value @classmethod def from_crawler(cls, crawler): val = crawler.settings.getint('MMMM') ext = cls(val) crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened) crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed) return ext def spider_opened(self, spider): print('open') def spider_closed(self, spider): print('close')
9、避免重複訪問
scrapy默認使用 scrapy.dupefilter.RFPDupeFilter 進行去重,相關配置有:
DUPEFILTER_CLASS = 'scrapy.dupefilter.RFPDupeFilter' DUPEFILTER_DEBUG = False JOBDIR = "保存範文記錄的日誌路徑,如:/root/" # 最終路徑爲 /root/requests.seen
class RepeatUrl: def __init__(self): self.visited_url = set() @classmethod def from_settings(cls, settings): """ 初始化時,調用 :param settings: :return: """ return cls() def request_seen(self, request): """ 檢測當前請求是否已經被訪問過 :param request: :return: True表示已經訪問過;False表示未訪問過 """ if request.url in self.visited_url: return True self.visited_url.add(request.url) return False def open(self): """ 開始爬去請求時,調用 :return: """ print('open replication') def close(self, reason): """ 結束爬蟲爬取時,調用 :param reason: :return: """ print('close replication') def log(self, request, spider): """ 記錄日誌 :param request: :param spider: :return: """ print('repeat', request.url)
10、其餘
# -*- coding: utf-8 -*- # Scrapy settings for step8_king project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html # 1. 爬蟲名稱 BOT_NAME = 'step8_king' # 2. 爬蟲應用路徑 SPIDER_MODULES = ['step8_king.spiders'] NEWSPIDER_MODULE = 'step8_king.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent # 3. 客戶端 user-agent請求頭 # USER_AGENT = 'step8_king (+http://www.yourdomain.com)' # Obey robots.txt rules # 4. 禁止爬蟲配置 # ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) # 5. 併發請求數 # CONCURRENT_REQUESTS = 4 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs # 6. 延遲下載秒數 # DOWNLOAD_DELAY = 2 # The download delay setting will honor only one of: # 7. 單域名訪問併發數,而且延遲下次秒數也應用在每一個域名 # CONCURRENT_REQUESTS_PER_DOMAIN = 2 # 單IP訪問併發數,若是有值則忽略:CONCURRENT_REQUESTS_PER_DOMAIN,而且延遲下次秒數也應用在每一個IP # CONCURRENT_REQUESTS_PER_IP = 3 # Disable cookies (enabled by default) # 8. 是否支持cookie,cookiejar進行操做cookie # COOKIES_ENABLED = True # COOKIES_DEBUG = True # Disable Telnet Console (enabled by default) # 9. Telnet用於查看當前爬蟲的信息,操做爬蟲等... # 使用telnet ip port ,而後經過命令操做 # TELNETCONSOLE_ENABLED = True # TELNETCONSOLE_HOST = '127.0.0.1' # TELNETCONSOLE_PORT = [6023,] # 10. 默認請求頭 # Override the default request headers: # DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # 'Accept-Language': 'en', # } # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html # 11. 定義pipeline處理請求 # ITEM_PIPELINES = { # 'step8_king.pipelines.JsonPipeline': 700, # 'step8_king.pipelines.FilePipeline': 500, # } # 12. 自定義擴展,基於信號進行調用 # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html # EXTENSIONS = { # # 'step8_king.extensions.MyExtension': 500, # } # 13. 爬蟲容許的最大深度,能夠經過meta查看當前深度;0表示無深度 # DEPTH_LIMIT = 3 # 14. 爬取時,0表示深度優先Lifo(默認);1表示廣度優先FiFo # 後進先出,深度優先 # DEPTH_PRIORITY = 0 # SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleLifoDiskQueue' # SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.LifoMemoryQueue' # 先進先出,廣度優先 # DEPTH_PRIORITY = 1 # SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue' # SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue' # 15. 調度器隊列 # SCHEDULER = 'scrapy.core.scheduler.Scheduler' # from scrapy.core.scheduler import Scheduler # 16. 訪問URL去重 # DUPEFILTER_CLASS = 'step8_king.duplication.RepeatUrl' # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html """ 17. 自動限速算法 from scrapy.contrib.throttle import AutoThrottle 自動限速設置 1. 獲取最小延遲 DOWNLOAD_DELAY 2. 獲取最大延遲 AUTOTHROTTLE_MAX_DELAY 3. 設置初始下載延遲 AUTOTHROTTLE_START_DELAY 4. 當請求下載完成後,獲取其"鏈接"時間 latency,即:請求鏈接到接受到響應頭之間的時間 5. 用於計算的... AUTOTHROTTLE_TARGET_CONCURRENCY target_delay = latency / self.target_concurrency new_delay = (slot.delay + target_delay) / 2.0 # 表示上一次的延遲時間 new_delay = max(target_delay, new_delay) new_delay = min(max(self.mindelay, new_delay), self.maxdelay) slot.delay = new_delay """ # 開始自動限速 # AUTOTHROTTLE_ENABLED = True # The initial download delay # 初始下載延遲 # AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies # 最大下載延遲 # AUTOTHROTTLE_MAX_DELAY = 10 # The average number of requests Scrapy should be sending in parallel to each remote server # 平均每秒併發數 # AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: # 是否顯示 # AUTOTHROTTLE_DEBUG = True # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings """ 18. 啓用緩存 目的用於將已經發送的請求或相應緩存下來,以便之後使用 from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.extensions.httpcache import DummyPolicy from scrapy.extensions.httpcache import FilesystemCacheStorage """ # 是否啓用緩存策略 # HTTPCACHE_ENABLED = True # 緩存策略:全部請求均緩存,下次在請求直接訪問原來的緩存便可 # HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" # 緩存策略:根據Http響應頭:Cache-Control、Last-Modified 等進行緩存的策略 # HTTPCACHE_POLICY = "scrapy.extensions.httpcache.RFC2616Policy" # 緩存超時時間 # HTTPCACHE_EXPIRATION_SECS = 0 # 緩存保存路徑 # HTTPCACHE_DIR = 'httpcache' # 緩存忽略的Http狀態碼 # HTTPCACHE_IGNORE_HTTP_CODES = [] # 緩存存儲的插件 # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' """ 19. 代理,須要在環境變量中設置 from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware 方式一:使用默認 os.environ { http_proxy:http://root:woshiniba@192.168.11.11:9999/ https_proxy:http://192.168.11.11:9999/ } 方式二:使用自定義下載中間件 def to_bytes(text, encoding=None, errors='strict'): if isinstance(text, bytes): return text if not isinstance(text, six.string_types): raise TypeError('to_bytes must receive a unicode, str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) class ProxyMiddleware(object): def process_request(self, request, spider): PROXIES = [ {'ip_port': '111.11.228.75:80', 'user_pass': ''}, {'ip_port': '120.198.243.22:80', 'user_pass': ''}, {'ip_port': '111.8.60.9:8123', 'user_pass': ''}, {'ip_port': '101.71.27.120:80', 'user_pass': ''}, {'ip_port': '122.96.59.104:80', 'user_pass': ''}, {'ip_port': '122.224.249.122:8088', 'user_pass': ''}, ] proxy = random.choice(PROXIES) if proxy['user_pass'] is not None: request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port']) encoded_user_pass = base64.encodestring(to_bytes(proxy['user_pass'])) request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass) print "**************ProxyMiddleware have pass************" + proxy['ip_port'] else: print "**************ProxyMiddleware no pass************" + proxy['ip_port'] request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port']) DOWNLOADER_MIDDLEWARES = { 'step8_king.middlewares.ProxyMiddleware': 500, } """ """ 20. Https訪問 Https訪問時有兩種狀況: 1. 要爬取網站使用的可信任證書(默認支持) DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" DOWNLOADER_CLIENTCONTEXTFACTORY = "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory" 2. 要爬取網站使用的自定義證書 DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" DOWNLOADER_CLIENTCONTEXTFACTORY = "step8_king.https.MySSLFactory" # https.py from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from twisted.internet.ssl import (optionsForClientTLS, CertificateOptions, PrivateCertificate) class MySSLFactory(ScrapyClientContextFactory): def getCertificateOptions(self): from OpenSSL import crypto v1 = crypto.load_privatekey(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.key.unsecure', mode='r').read()) v2 = crypto.load_certificate(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.pem', mode='r').read()) return CertificateOptions( privateKey=v1, # pKey對象 certificate=v2, # X509對象 verify=False, method=getattr(self, 'method', getattr(self, '_ssl_method', None)) ) 其餘: 相關類 scrapy.core.downloader.handlers.http.HttpDownloadHandler scrapy.core.downloader.webclient.ScrapyHTTPClientFactory scrapy.core.downloader.contextfactory.ScrapyClientContextFactory 相關配置 DOWNLOADER_HTTPCLIENTFACTORY DOWNLOADER_CLIENTCONTEXTFACTORY """ """ 21. 爬蟲中間件 class SpiderMiddleware(object): def process_spider_input(self,response, spider): ''' 下載完成,執行,而後交給parse處理 :param response: :param spider: :return: ''' pass def process_spider_output(self,response, result, spider): ''' spider處理完成,返回時調用 :param response: :param result: :param spider: :return: 必須返回包含 Request 或 Item 對象的可迭代對象(iterable) ''' return result def process_spider_exception(self,response, exception, spider): ''' 異常調用 :param response: :param exception: :param spider: :return: None,繼續交給後續中間件處理異常;含 Response 或 Item 的可迭代對象(iterable),交給調度器或pipeline ''' return None def process_start_requests(self,start_requests, spider): ''' 爬蟲啓動時調用 :param start_requests: :param spider: :return: 包含 Request 對象的可迭代對象 ''' return start_requests 內置爬蟲中間件: 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, """ # from scrapy.contrib.spidermiddleware.referer import RefererMiddleware # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html SPIDER_MIDDLEWARES = { # 'step8_king.middlewares.SpiderMiddleware': 543, } """ 22. 下載中間件 class DownMiddleware1(object): def process_request(self, request, spider): ''' 請求須要被下載時,通過全部下載器中間件的process_request調用 :param request: :param spider: :return: None,繼續後續中間件去下載; Response對象,中止process_request的執行,開始執行process_response Request對象,中止中間件的執行,將Request從新調度器 raise IgnoreRequest異常,中止process_request的執行,開始執行process_exception ''' pass def process_response(self, request, response, spider): ''' spider處理完成,返回時調用 :param response: :param result: :param spider: :return: Response 對象:轉交給其餘中間件process_response Request 對象:中止中間件,request會被從新調度下載 raise IgnoreRequest 異常:調用Request.errback ''' print('response1') return response def process_exception(self, request, exception, spider): ''' 當下載處理器(download handler)或 process_request() (下載中間件)拋出異常 :param response: :param exception: :param spider: :return: None:繼續交給後續中間件處理異常; Response對象:中止後續process_exception方法 Request對象:中止中間件,request將會被從新調用下載 ''' return None 默認下載中間件 { 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, 'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590, 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, } """ # from scrapy.contrib.downloadermiddleware.httpauth import HttpAuthMiddleware # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # DOWNLOADER_MIDDLEWARES = { # 'step8_king.middlewares.DownMiddleware1': 100, # 'step8_king.middlewares.DownMiddleware2': 500, # }
11、TinyScrapy
#!/usr/bin/env python # -*- coding:utf-8 -*- import types from twisted.internet import defer from twisted.web.client import getPage from twisted.internet import reactor class Request(object): def __init__(self, url, callback): self.url = url self.callback = callback self.priority = 0 class HttpResponse(object): def __init__(self, content, request): self.content = content self.request = request class ChouTiSpider(object): def start_requests(self): url_list = ['http://www.cnblogs.com/', 'http://www.bing.com'] for url in url_list: yield Request(url=url, callback=self.parse) def parse(self, response): print(response.request.url) # yield Request(url="http://www.baidu.com", callback=self.parse) from queue import Queue Q = Queue() class CallLaterOnce(object): def __init__(self, func, *a, **kw): self._func = func self._a = a self._kw = kw self._call = None def schedule(self, delay=0): if self._call is None: self._call = reactor.callLater(delay, self) def cancel(self): if self._call: self._call.cancel() def __call__(self): self._call = None return self._func(*self._a, **self._kw) class Engine(object): def __init__(self): self.nextcall = None self.crawlling = [] self.max = 5 self._closewait = None def get_response(self,content, request): response = HttpResponse(content, request) gen = request.callback(response) if isinstance(gen, types.GeneratorType): for req in gen: req.priority = request.priority + 1 Q.put(req) def rm_crawlling(self,response,d): self.crawlling.remove(d) def _next_request(self,spider): if Q.qsize() == 0 and len(self.crawlling) == 0: self._closewait.callback(None) if len(self.crawlling) >= 5: return while len(self.crawlling) < 5: try: req = Q.get(block=False) except Exception as e: req = None if not req: return d = getPage(req.url.encode('utf-8')) self.crawlling.append(d) d.addCallback(self.get_response, req) d.addCallback(self.rm_crawlling,d) d.addCallback(lambda _: self.nextcall.schedule()) @defer.inlineCallbacks def crawl(self): spider = ChouTiSpider() start_requests = iter(spider.start_requests()) flag = True while flag: try: req = next(start_requests) Q.put(req) except StopIteration as e: flag = False self.nextcall = CallLaterOnce(self._next_request,spider) self.nextcall.schedule() self._closewait = defer.Deferred() yield self._closewait @defer.inlineCallbacks def pp(self): yield self.crawl() _active = set() obj = Engine() d = obj.crawl() _active.add(d) li = defer.DeferredList(_active) li.addBoth(lambda _,*a,**kw: reactor.stop()) reactor.run()
備註: 更多文檔參見:http://scrapy-chs.readthedocs.io/zh_CN/latest/index.html