【爬蟲】隨機獲取UA

使用模塊  fake-useragent

https://github.com/hellysmile/fake-useragent

 

1.安裝模塊

2.配置

# settings.py

'''下載器中間件'''
DOWNLOADER_MIDDLEWARES = {
   'Lagou.middlewares.RandomUserAgentMiddleware': 543,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}

'''UA的類型'''
RANDOM_UA_TYPE = "random"

# middlewares.py
'''模仿middlewares的UserAgentMiddleware寫的類'''
class RandomUserAgentMiddleware(object):
    """This middleware allows spiders to override the user_agent"""

    def __init__(self, crawler):
        # 實例化UserAgent()，從配置文件讀取ua的類型
        super().__init__()
        self.ua = UserAgent()
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE","random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    # def spider_opened(self, spider):
    #     self.user_agent = getattr(spider, 'user_agent', self.user_agent)

    def process_request(self, request, spider):
        def get_ua():
            # 經過反射獲取隨機UA
            random_ua =  getattr(self.ua,self.ua_type)
            return random_ua
        request.headers.setdefault("User-Agent", get_ua())