抓取今日頭條部分信息

import requests import re from urllib.parse import urlencode from requests.exceptions import RequestException import json from bs4 import BeautifulSoup import codecs from conm import *

header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36', } #定義獲取內容的方法 def get_page_index(offset,keyword): data = { 'offset': offset, 'format': 'json', 'keyword': keyword, 'autoload': 'true', 'count': 20, 'cur_tab': 3, } #合成url url = 'http://www.toutiao.com/search_content/?' + urlencode(data) #對請求作些異常處理 try: response = requests.get(url,headers = header) if response.status_code == 200: return response.text return None except RequestException: print('請求索引頁出錯') return None

def parse_page_index(html): data = json.loads(html) if data and 'data' in data.keys(): for item in data.get('data'): yield item.get('article_url')

#獲取詳情頁面 def get_page_detail(url): res = requests.get(url,headers = header) try: if res.status_code == 200: return res.text return None except RequestException: print('請求失敗') return None #解析詳細頁面 def parse_page_detail(html,url): soup = BeautifulSoup(html, 'lxml') title = soup.select('title')[0].get_text() images_pattern = re.compile(r'gallery: JSON.parse("(.*?)"),', re.S) result = re.search(images_pattern, html) if result: # group(0)是原始字符串, group(1)是第一個括號匹配到的字符串 # groups()以元組形式返回所有分組截獲的字符串。至關於調用group(1,2,…last) # codecs: 使不具備轉義的反斜槓具備轉義功能,最難的一步驟 data_str = codecs.getdecoder('unicode_escape')(result.group(1))[0]

json_data = json.loads(data_str)
    if json_data and 'sub_images' in json_data.keys():
        sub_images = json_data.get('sub_images')
        images = [item.get('url') for item in sub_images]
        for image in images:
            download_img(image)
        return {
            'title': title,
            'images': images,
            'url': url
        }
else:
    print('沒有搜索到符合條件的gallery數據', end='\n\n')

import pymongo #定義數據庫鏈接函數 client = pymongo.MongoClient(MONGO_URL) db = client[MONGO_DB]

#存儲方法 def save_to_mongo(result): if db[MONG_TABLE].insert(result): print('存儲成功') return True return False

#定義下在圖片的方法 def download_img(url): print('正在下載',url) try: response = requests.get(url, headers=header) if response.status_code == 200: save_image(response.content) return None except RequestException: print('請求索引頁出錯') return None #定義存儲圖片的方法 import os from hashlib import md5 def save_image(content): file_path ='{0}/{1}.{2}'.format('mao',md5(content).hexdigest(),'jpg') if not os.path.exists(file_path): with open(file_path,'wb') as f: f.write(content)

#主測試函數 def main(offset): html = get_page_index(offset,'街拍') for url in parse_page_index(html): result = get_page_detail(url) ok = parse_page_detail(result,url) save_to_mongo(ok)

from multiprocessing import Pool if name == 'main': groups = [x * 20 for x in range(GROUP_START, GROUP_END)] pool = Pool() pool.map(main,groups)