多線程高容錯爬頭條街拍美圖

• 分析頭條的ajax,經過正則表達式，python3多線程高容錯爬取頭條的街拍美圖，保存到mongodb,並下載圖片
• 頭條的內容網頁較以前已經改版，圖牀頁不只有ajax的還有html的內容網頁
• 因此使用了兩種正則，根據條件調用

#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
@author:Aiker
@file:toutiao.py
@time:下午9:35
"""
import json
import os
import re
from json import JSONDecodeError
from multiprocessing import Pool
from urllib.parse import urlencode
from hashlib import md5
import pymongo
import requests
from requests.exceptions import RequestException

MONGO_URL = 'localhost:27017'
MONGO_DB = 'toutiao'
MONGO_TABLE = 'toutiao'
GROUP_START = 1
GROUP_END = 20
KEYWORD = '街拍'
client = pymongo.MongoClient(MONGO_URL, connect=False)
db = client[MONGO_DB]

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'

}

def get_url(url):
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('請求失敗', url)
        return None

def get_page_index(offset, keyword):
    data = {
        'aid': '24',
        'app_name': 'web_search',
        'offset': offset,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': '20',
        'en_qc': '1',
        'cur_tab': '1',
        'from': 'search_tab',
        'pd': 'synthesis',
        'timestamp': '1124216535987'
    }
    url = 'https://www.toutiao.com/api/search/content/?' + urlencode(data)  # 字典對象轉化url對象
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('請求索引頁失敗')
        return None

def parse_page_index(html):
    try:
        data = json.loads(html)  # 轉化爲json對象
        if data and 'data' in data.keys():
            # print(data.keys()) #調試，輸出全部key
            for item in data.get('data'):
                if 'article_url' in item:  # 判斷是否存在，避免出現None
                    # print(item)
                    yield item.get('article_url')  # 構造生成器
    except JSONDecodeError:
        pass
    except TypeError:
        pass

def get_page_detail(url):
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('請求詳情頁出錯', url)
        return None

def parse_page_detail(html, url):
    pattern = re.compile("articleInfo:.*?title:\s'(.*?)',.*?content:\s'(.*?)'.*?groupId", re.S)
    result = re.findall(pattern, html)
    # print(tc)
    if result:
        title, content = result[0]
        pattern = re.compile("(http://.*?)"", re.S)
        images = re.findall(pattern, content)
        # print(img)
        for image in images: download_image(image, title)
        # print(item)
        return {
            'title': title,
            'url': url,
            'images': images
        }
    else:
        pattern = re.compile('BASE_DATA.galleryInfo.*?title:\s\'(.*?)\'.*?gallery: JSON.parse\("(.*)"\)', re.S)
        result = re.findall(pattern, html)
        # print(result[0])
        if result:
            title, content = result[0]
            data = json.loads(content.replace('\\', ''))
            # print(data)
            if data and 'sub_images' in data.keys():
                sub_images = data.get('sub_images')
                images = [item.get('url') for item in sub_images]
                for image in images: download_image(image,title)
                return {
                    'title': title,
                    'url': url,
                    'images': images
                }

def save_to_mongo(result):
    if db[MONGO_TABLE].insert(result):
        print('存儲到MongoDB成功', result)
        return True
    return False

def download_image(url,title):
    print('正在下載', url)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            save_image(response.content,title)
        return None
    except RequestException:
        print('請求圖片出錯', url)
        return None

def save_image(content,title):
    try:
        if title:
            title = re.sub('[:?！!：？]', '', title)  # 替換title中的特殊字符，避免創建資料夾目錄出錯
        dir = 'z:\\toutiao\\'
        if os.path.exists(dir + title):
            pass
        else:
            os.mkdir(dir + title)
        file_path = '{0}/{1}.{2}'.format( dir + title, md5(content).hexdigest(), 'jpg')
        if not os.path.exists(file_path):
            with open(file_path, 'wb') as f:
                f.write(content)
                f.close()
    except OSError:
        pass

def main(offset):
    html = get_page_index(offset, KEYWORD)
    for url in parse_page_index(html):
        print(url)
        html = get_page_detail(url)
        if html:
            result = parse_page_detail(html, url)
            if result:
                save_to_mongo(result)

    # print(html)

if __name__ == '__main__':
    # main()
    groups = [x * 20 for x in range(GROUP_START, GROUP_END + 1)]
    pool = Pool()
    pool.map(main, groups)
    pool.close()
    pool.join()

• 下載圖片，並保存到mongodb