Python爬蟲案例：⼿機 App 抓包爬⾍

1. items.py

class DouyuspiderItem(scrapy.Item):
      name = scrapy.Field()# 存儲照⽚的名字
      imagesUrls = scrapy.Field()# 照⽚的 url 路徑
      imagesPath = scrapy.Field()# 照⽚保存在本地的路徑

 

2. spiders/douyu.py

import scrapy
import json
from douyuSpider.items import DouyuspiderItem
class DouyuSpider(scrapy.Spider):
name = "douyu"
allowd_domains = ["http://capi.douyucdn.cn"]
offset = 0
url = "http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&
offset="
start_urls = [url + str(offset)]
def parse(self, response): # 返回從 json⾥獲取 data 段數據集合
data = json.loads(response.text)["data"] # 若是 data⾥⾯沒有值，那麼就直接退出函數
#if not data:
# return
for each in data:
item = DouyuspiderItem()
item["name"] = each["nickname"]
item["imagesUrls"] = each["vertical_src"]
yield item
self.offset += 20
yield scrapy.Request(self.url + str(self.offset), callback =
self.parse)

　　

3. 設置 setting.py

ITEM_PIPELINES = {'douyuSpider.pipelines.ImagesPipeline': 1} # Images 的存放位置，以後會在 pipelines.py⾥調⽤
IMAGES_STORE = "/Users/Power/lesson_python/douyuSpider/Images"
# user-agent
USER_AGENT = 'DYZB/2.290 (iPhone; iOS 9.3.4; Scale/2.00)'

 

4. pipelines.py

import scrapy
import os
from scrapy.pipelines.images import ImagesPipeline
from scrapy.utils.project import get_project_settings
class ImagesPipeline(ImagesPipeline):
IMAGES_STORE = get_project_settings().get("IMAGES_STORE")
def get_media_requests(self, item, info):
image_url = item["imagesUrls"]
yield scrapy.Request(image_url)
def item_completed(self, results, item, info): # 固定寫法，獲取圖⽚路徑，同時判斷這個路徑是否正確，若是正確，就 放 到 image_path⾥，ImagesPipeline 源碼剖析可⻅
image_path = [x["path"] for ok, x in results if ok]
os.rename(self.IMAGES_STORE + "/" + image_path[0], self.IMA
GES_STORE + "/" + item["name"] + ".jpg")
item["imagesPath"] = self.IMAGES_STORE + "/" + item["name"]
return item
#get_media_requests 的做⽤就是爲每⼀個圖⽚連接⽣成⼀個 Request 對象，這
個
⽅法的輸出將做爲 item_completed 的輸⼊中的 results，results 是⼀個元組，
每 個元組包括 (success, imageinfoorfailure) 。若是 success=true ，
imageinfo or_failure 是⼀個字典，包括 url/path/checksum 三個 key。

 

在項⽬根⽬錄下新建 main.py⽂件,⽤於調試

 

from scrapy import cmdline
cmdline.execute('scrapy crawl douyu'.split())

 

執⾏程序

 

py2 main.py

獲取完整Python爬蟲視頻教程請點：python網絡爬蟲課程。