scrapy添加下載中間鍵 User-agent

設置 user -agent 和  打開（注意路徑）

DOWNLOADER_MIDDLEWARES = {
   #'news.middlewares.NewsDownloaderMiddleware': 543,
   'news.middlewares.UserAgentMiddleware': 300,
}

#隨機換user-agent
class UserAgentMiddleware(object):
    """
        給每個請求隨機切換一個User-Agent
    """
    def process_request(self, request, spider):
        user_agent = random.choice(us_list)
        request.headers['User-Agent'] = user_agent
注意這樣能夠打印

request.headers['User-Agent']