第4章 scrapy爬取知名技術文章網站(2)

4-8~9 編寫spider爬取jobbole的全部文章

# -*- coding: utf-8 -*-
import re
import scrapy
import datetime
from scrapy.http import Request
from urllib import parse
'''若是是py2 那就是import urlparse'''

from g0xukr.ArticleSpider.items import JobBoleArticleItem, ArticleItemLoader
from g0xukr.ArticleSpider.utils.common import get_md5

class JobboleSpider(scrapy.Spider):
    name = "jobbole"
    allowed_domains = ["python.jobbole.com"]
    start_urls = ['http://python.jobbole.com/all-posts/']

    def parse(self, response):
        """
        1. 獲取文章列表頁中的文章url並交給scrapy下載後並進行解析
        2. 獲取下一頁的url並交給scrapy進行下載， 下載完成後交給parse
        """

        #解析列表頁中的全部文章url並交給scrapy下載後並進行解析
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url":image_url}, callback=self.parse_detail)
            '''parse.urljoin(response.url, post_url)補全域名'''

        #提取下一頁並交給scrapy進行下載
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)

    def parse_detail(self, response):
        article_item = JobBoleArticleItem()
        #經過css選擇器提取字段
        front_image_url = response.meta.get("front_image_url", "")  #文章封面圖
        title = response.css(".entry-header h1::text").extract()[0]
        create_date = response.css("p.entry-meta-hide-on-mobile::text").extract()[0].strip().replace("·","").strip()
        praise_nums = response.css(".vote-post-up h10::text").extract()[0]
        fav_nums = response.css(".bookmark-btn::text").extract()[0]
        match_re = re.match(".*?(\d+).*", fav_nums)
        if match_re:
            fav_nums = int(match_re.group(1))
        else:
            fav_nums = 0

        comment_nums = response.css("a[href='#article-comment'] span::text").extract()[0]
        match_re = re.match(".*?(\d+).*", comment_nums)
        if match_re:
            comment_nums = int(match_re.group(1))
        else:
            comment_nums = 0

        content = response.css("div.entry").extract()[0]

        tag_list = response.css("p.entry-meta-hide-on-mobile a::text").extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("評論")]
        tags = ",".join(tag_list)

        article_item["url_object_id"] = get_md5(response.url)
        article_item["title"] = title
        article_item["url"] = response.url
        try:
            create_date = datetime.datetime.strptime(create_date, "%Y/%m/%d").date()
        except Exception as e:
            create_date = datetime.datetime.now().date()
        article_item["create_date"] = create_date
        article_item["front_image_url"] = [front_image_url]
        article_item["praise_nums"] = praise_nums
        article_item["comment_nums"] = comment_nums
        article_item["fav_nums"] = fav_nums
        article_item["tags"] = tags
        article_item["content"] = content
        yield article_item

4-10~12 items設計

一些零散的知識點：

1.meta傳遞值到item.py文件中

#例如：
yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url":image_url}, callback=self.parse_detail)

2.extract_first('')使用

extract_first('') 比 extract()[0]好用，由於後者有風險，若是爲空，就會出錯。可是前者若是爲空設置爲' '，因此更好用。

3.response.meta.get()用法

response.meta.get('front_image_url','') 前一個引號是本身定義的名稱，後一個空着，這樣若是就不會拋異常

4.scrapy自動下載圖片pipelines

ITEM_PIPELINES = {
     'scrapy.pipelines.images.ImagesPipeline': 1,
}

配置：

import os
IMAGES_URLS_FIELD='front\_image\_url' #'引號中要是一個列表，是圖片地址的字段
project_dir=os.path.abspath(os.path.dirname(__file__))  #相對的路徑，在其餘電腦上也能夠
IMAGES_STORE=os.path.join(project_dir,'存儲圖片文件名稱') #放在同級settings.py目錄下
'''若是要實現本身的需求，也能夠重載相應的函數達到需求，在pipelines中創建類，繼承ImagesPipeline就能夠了'''

5.哈希表摘要算法，輸出固定長度

python3模版：

def get_md5(url):   #傳進來url
    if isinstance(url, str):  #判斷是否是str，實際上是判斷是否是Unicode，python3中默認是Unicode編碼
        url = url.encode("utf-8") #轉換成utf-8，哈希只認utf-8
    m = hashlib.md5() 
    m.update(url)
    return m.hexdigest()

python2模版：

# -*- coding:utf-8 -*-
import hashlib
def get_md5(url='123'):
    m = hashlib.md5()
    m.update(url)
    return m.hexdigest()

4-13 數據表設計和保存item到json文件

模版：

import codecs
import json
from scrapy.exporters import JsonItemExporter
class JsonWithEncodingPipeline(object):
    #自定義json文件的導出
    def __init__(self):
        ''''''
        self.file = codecs.open('article.json', 'w', encoding="utf-8")
    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item
    def spider_closed(self, spider):
        self.file.close()


class JsonExporterPipleline(object):
    #調用scrapy提供的json export導出json文件
    def __init__(self):
        self.file = open('articleexport.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding="utf-8", ensure_ascii=False)
        self.exporter.start_exporting()

    def close_spider(self, spider):
        self.exporter.finish_exporting()
        self.file.close()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
    return item

4-14~15 經過pipeline保存數據到mysql

模版：

pip install mysqlclint 是mysql的一個驅動

import pymysql
import pymysql.cursors  
class MysqlPipeline(object):
    #採用同步的機制寫入mysql
    def __init__(self):
        self.conn = pymysql.connect('192.168.0.106', 'root', 'root', 'article_spider', charset="utf8", use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
            insert into jobbole_article(title, url, create_date, fav_nums)
            VALUES (%s, %s, %s, %s)
        """
        self.cursor.execute(insert_sql, (item["title"], item["url"], item["create_date"], item["fav_nums"]))
        self.conn.commit()


from twisted.enterprise import adbapi
class MysqlTwistedPipline(object):
    '''異步插入mysql'''
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        '''傳入settings的參數'''
        dbparms = dict(
            host = settings["MYSQL_HOST"],
            db = settings["MYSQL_DBNAME"],
            user = settings["MYSQL_USER"],
            passwd = settings["MYSQL_PASSWORD"],
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor,
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)

        return cls(dbpool)

    def process_item(self, item, spider):
        #使用twisted將mysql插入變成異步執行
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error, item, spider) #處理異常

    def handle_error(self, failure, item, spider):
        # 處理異步插入的異常
        print (failure)

    def do_insert(self, cursor, item):
        #執行具體的插入
        #根據不一樣的item 構建不一樣的sql語句並插入到mysql中
        insert_sql, params = item.get_insert_sql()
        print (insert_sql, params)
        cursor.execute(insert_sql, params)

4-16~17 scrapy item loader機制

模版：

scrapy item loader機制，便於之後的維護

items.p文件中

import datetime
import re

import scrapy
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose, TakeFirst, Join

from utils.common import extract_num
from settings import SQL_DATETIME_FORMAT, SQL_DATE_FORMAT
from w3lib.html import remove_tags
def add_jobbole(value):
    return value+"-bobby"


def date_convert(value):
    try:
        create_date = datetime.datetime.strptime(value, "%Y/%m/%d").date()
    except Exception as e:
        create_date = datetime.datetime.now().date()

    return create_date


def get_nums(value):
    match_re = re.match(".*?(\d+).*", value)
    if match_re:
        nums = int(match_re.group(1))
    else:
        nums = 0

    return nums

def return_value(value):
    return value


def remove_comment_tags(value):
    #去掉tag中提取的評論
    if "評論" in value:
        return ""
    else:
        return value

class ArticleItemLoader(ItemLoader):
    #自定義itemloader
    default_output_processor = TakeFirst()


class JobBoleArticleItem(scrapy.Item):
    title = scrapy.Field()
    create_date = scrapy.Field(
        input_processor=MapCompose(date_convert),
    )
    url = scrapy.Field()
    url_object_id = scrapy.Field()
    front_image_url = scrapy.Field(
        output_processor=MapCompose(return_value)
    )
    front_image_path = scrapy.Field()
    praise_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    comment_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    fav_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    tags = scrapy.Field(
        input_processor=MapCompose(remove_comment_tags),
        output_processor=Join(",")
    )
    content = scrapy.Field()

    def get_insert_sql(self):
        insert_sql = """
            insert into jobbole_article(title, url, create_date, fav_nums, front_image_url, front_image_path,
            praise_nums, comment_nums, tags, content)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE content=VALUES(fav_nums)
        """

        fron_image_url = ""
        # content = remove_tags(self["content"])

        if self["front_image_url"]:
            fron_image_url = self["front_image_url"][0]
        params = (self["title"], self["url"], self["create_date"], self["fav_nums"],
                  fron_image_url, self["front_image_path"], self["praise_nums"], self["comment_nums"],
                  self["tags"], self["content"])
        return insert_sql, params

spider.py文件中部分代碼

def parse_detail(self, response):
    article_item = JobBoleArticleItem()
    front_image_url = response.meta.get("front_image_url", "")  # 文章封面圖
    item_loader = ArticleItemLoader(item=JobBoleArticleItem(), response=response)
    item_loader.add_css("title", ".entry-header h1::text")
    item_loader.add_value("url", response.url)
    item_loader.add_value("url_object_id", get_md5(response.url))
    item_loader.add_css("create_date", "p.entry-meta-hide-on-mobile::text")
    item_loader.add_value("front_image_url", [front_image_url])
    item_loader.add_css("praise_nums", ".vote-post-up h10::text")
    item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
    item_loader.add_css("fav_nums", ".bookmark-btn::text")
    item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a::text")
    item_loader.add_css("content", "div.entry")

    article_item = item_loader.load_item()


    yield article_item

做者：今孝

出處：http://www.cnblogs.com/jinxiao-pu/p/6721848.html

本文版權歸做者和博客園共有，歡迎轉載，但未經做者贊成必須保留此段聲明，且在文章頁面明顯位置給出原文鏈接。