Python Spider 抓取今日頭條街拍美圖

"""
抓取今日頭條街拍美圖
"""

import os
import time
import requests

from hashlib import md5


class SpiderToutiao(object):

    def __init__(self):
        # 指定下載目錄
        self.download_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "download")
        # 經過分析請求, 咱們發現了須要的地址以下, 且分頁是經過 offset + 20 控制的
        self.url = "https://www.toutiao.com/search_content/" \
                   "?offset={0}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=3&from=gallery"
        # 構造請求頭, 假裝成 Ajax 請求
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                          "Chrome/66.0.3359.139 Safari/537.36",
            "Referer": "https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D",
            "X-Requested-With": "XMLHttpRequest"
        }

    def handler(self, offset=0):

        while True:
            response = requests.get(self.url.format(offset), headers=self.headers)
            if response.status_code == 200:
                print("INFO -> Current URL: <%s>" % response.url)
                json_data = response.json().get("data")
                # 開始解析數據
                if json_data:
                    for item in json_data:
                        _title = item.get("title")
                        _imgDict = item.get("image_list")
                        # 修正一下URL, 默認給的圖片地址是小圖, 咱們要高清大圖
                        _imgList = [str("http:" + _.get("url")).replace("list", "large") for _ in _imgDict]

                        # 建立存儲目錄
                        _downloadDir = os.path.join(self.download_dir, _title)
                        if not os.path.exists(_downloadDir):
                            os.makedirs(_downloadDir)

                        # 下載並保存文件
                        for img in _imgList:
                            r = requests.get(img)
                            _file = os.path.join(_downloadDir, md5(r.content).hexdigest() + ".jpg")
                            if not os.path.exists(_file):
                                with open(_file, "wb") as f:
                                    f.write(r.content)
                            else:
                                print("INFO -> ig <%s>" % _file)
                # 說明沒有數據了, 程序退出
                else:
                    break

                # 分頁自增
                offset += 20
                # 間隔時間
                time.sleep(.9)
            else:
                print(response.reason)
                exit(999)


if __name__ == "__main__":

    spider = SpiderToutiao()
    spider.handler()