Python爬蟲（request模塊）

 

 發送簡單的請求：

　　

發送帶header的請求：

 

 發送帶參數的請求：

例子以下：

import requests
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 "
                        "(KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"}
p = {'hhs':100}
url = 'http://www.baidu.com'
response = requests.get(url,headers=headers,params=p)
print(response.status_code)
print(response.request.url)

 

實戰（貼吧爬蟲）：

import requests

class TiebaSpider:
    #構造函數，傳入想要查詢的貼吧名字和查詢前num頁，定義後面方法須要用的屬性
    def __init__(self,tieba_name,tieba_num):
        self.tieba_num = tieba_num
        self.tieba_name = tieba_name
        #下面爲url訪問的格式,kw爲貼吧名,pn爲表明頁數，貼吧可查首頁pn=0,第一頁50，第二頁100,以此類推
        self.url_temp = "https://tieba.baidu.com/f?kw=" + tieba_name + "&ie=utf-8&pn={}"
        #模仿網頁訪問貼吧
        self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 "
                        "(KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"}

    #獲取url列表
    def get_urlList(self):
        url_list = []
        for i in range(self.tieba_num):
            url_list.append(self.url_temp.format(i * 50))
        return url_list

    #訪問網頁，解析網頁,返回
    def parse_url(self,url):
        response = requests.get(url,headers=self.headers)
        return response.text #或者response.content.decode('utf-8')

    #保存網頁
    def save_html(self,html_str,page_num):
        file_path = "{}-第{}頁.html".format(self.tieba_name,page_num)
        with open(file_path,'w',encoding='utf-8') as f:
            f.write(html_str)

    def run(self):
        url_list = self.get_urlList()
        for url in url_list:
            html_str = self.parse_url(url)  #保存的內容
            page_num = url_list.index(url) + 1  #文件的頁碼
            self.save_html(html_str,page_num)

def main():
    tieba_name = input('請輸入想查詢的貼吧名：')
    tieba_num = int(input('請輸入想要查詢前幾頁：'))
    tieba = TiebaSpider(tieba_name, tieba_num)
    tieba.run()

if __name__ == '__main__':
    main()