python瀏覽器假裝

https://www.jb51.net/article/139587.htm

python爬蟲瀏覽器假裝

1.

#導入urllib.request模塊
import urllib.request
#設置請求頭
headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")
#建立一個opener
opener=urllib.request.build_opener()
#將headers添加到opener中
opener.addheaders=[headers]
#將opener安裝爲全局
urllib.request.install_opener(opener)
#用urlopen打開網頁
data=urllib.request.urlopen(url).read().decode('utf-8','ignore')

 

2.

#定義代理ip

proxy_addr="122.241.72.191:808"

#設置代理

proxy=urllib.request.ProxyHandle({'http':proxy_addr})

#建立一個opener

opener=urllib.request.build_opener(proxy,urllib.request.HTTPHandle)

#將opener安裝爲全局

urllib.request.install_opener(opener)

#用urlopen打開網頁

data=urllib.request.urlopen(url).read().decode('utf-8','ignore')

 

 

3.

#定義代理ip

proxy_addr="122.241.72.191:808"

#建立一個請求

req=urllib.request.Request(url)

#添加headers

req.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)

#設置代理

proxy=urllib.request.ProxyHandle("http":proxy_addr)

#建立一個opener

opener=urllib.request.build_opener(proxy,urllib.request.HTTPHandle)

#將opener安裝爲全局

urllib.request.install_opener(opener)

#用urlopen打開網頁

data=urllib.request.urlopen(req).read().decode('utf-8','ignore')