爬蟲（3）

時間 2019-11-29

標籤爬蟲欄目網絡爬蟲简体版

原文原文鏈接

1、模擬登陸案例（識別驗證碼）

一、打碼平臺 - 雲打碼：www.yundama.com

　　使用步驟：php

　　　　- 註冊兩個帳戶，普通用戶和開發者用戶；html

　　　　- 登陸web

　　　　　　普通用戶查看餘額；ajax

　　　　　　登陸開發者用戶；chrome

　　　　　　　　建立一個軟件：個人軟件 -> 建立軟件；json

　　　　　　　　下載示例代碼：開發者中心 -> 下載最新雲打碼DLL -> PythonHTTP示例下載api

　　　　- 下載後解壓縮，以下：瀏覽器

import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

######################################################################

# 用戶名
username    = 'username'

# 密碼
password    = 'password'                            

# 軟件ＩＤ，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
appid       = 1                                     

# 軟件密鑰，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
appkey      = '22cc5376925e9387a23cf797cb9ba745'    

# 圖片文件
filename    = 'getimage.jpg'                        

# 驗證碼類型，# 例：1004表示4位字母數字，不一樣類型收費不一樣。請準確填寫，不然影響識別率。在此查詢全部類型 http://www.yundama.com/price.html
codetype    = 1004

# 超時時間，秒
timeout     = 60                                    

# 檢查
if (username == 'username'):
    print('請設置好相關參數再測試')
else:
    # 初始化
    yundama = YDMHttp(username, password, appid, appkey)

    # 登錄雲打碼
    uid = yundama.login();
    print('uid: %s' % uid)

    # 查詢餘額
    balance = yundama.balance();
    print('balance: %s' % balance)

    # 開始識別，圖片路徑，驗證碼類型ID，超時時間（秒），識別結果
    cid, result = yundama.decode(filename, codetype, timeout);
    print('cid: %s, result: %s' % (cid, result))

######################################################################

YDMHTTPDemo3.x.py 內容

二、案例一：模擬登陸人人網，爬取我的中心頁面數據

def getCodeData(username, pwd, codePath, codeType):
    username = username     # 用戶名（雲打碼普通用戶用戶名）
    password = pwd              # 普通用戶對應的密碼
    appid = 6003          # 軟件代碼，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
    appkey = '1f4b564483ae5c907a1d34f8e2f2776c'    # 通信密鑰，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
    filename = codePath          # 識別的圖片的路徑
    codetype = codeType           # 識別的類型，在幫助文檔可查看對應驗證碼類型
    timeout = 20 
    if (username == 'username'):
        print('請設置好相關參數再測試')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)

        # 登錄雲打碼
        uid = yundama.login();
        # print('uid: %s' % uid)

        # 查詢餘額
        balance = yundama.balance();
        # print('balance: %s' % balance)

        # 開始識別，圖片路徑，驗證碼類型ID，超時時間（秒），識別結果
        cid, result = yundama.decode(filename, codetype, timeout);
        # print('cid: %s, result: %s' % (cid, result))
    
    return result

getCodeData 函數

# 人人網的模擬登錄
import requests
import urllib
from lxml import etree
# 獲取session對象
session = requests.Session() # 下載驗證碼圖片
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
url = 'http://www.renren.com'
page_text = requests.get(url=url, headers=headers).text

tree = etree.HTML(page_text)
code_img_url = tree.xpath('//*[@id="verifyPic_login"]/@src')[0]
urllib.request.urlretrieve(url=code_img_url, filename='code.jpg')

# 識別驗證碼圖片中的數據值,2004表示4位純漢字，其餘類型代碼參考雲打碼幫助文檔
code_data = getCodeData(雲打碼用戶名', '雲打碼密碼', './code.jpg', 2004)
# print(code_data)    # code_data爲識別結果

login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019141727367'
data = {
    "email":"1547360919@qq.com",
    "icode":code_data,
    "origURL":"http://www.renren.com/home",
    "domain":"renren.com",
    "key_id":"1",
    "captcha_type":"web_login",
    "password":"38ce96b6b81595f845e55c1dd4e712ad6f1efe50fe31dbd5bf517b273d7c3b6e",
    "rkey":"07a9f1810ecf9b507634a45447a628e7",
    "f":""
}

# 若是請求成功，產生的cookie會自動保存在session對象中
session.post(url=login_url, data=data, headers=headers)

url = 'http://www.renren.com/448850039/profile'
page_text = session.get(url=url, headers=headers).text

with open('renren.html', 'w', encoding='utf8') as f:
    f.write(page_text)

三、案例二：模擬登陸古詩文網

# 模擬登錄古詩文網
from lxml import etree
import requests

s = requests.Session()

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}

login_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx'
page_text = requests.get(url=login_url, headers=headers, verify=False).text          # verify=False是解決請求https協議問題，SSL錯誤

tree = etree.HTML(page_text)
img_code_url = 'https://so.gushiwen.org' + tree.xpath('//*[@id="imgCode"]/@src')[0]

# 驗證碼圖片請求也會有session產生，所以不能使用urllib的urlretrieve方法
img_data = s.get(url=img_code_url, headers=headers, verify=False).content
with open('./gs_code_img.jpg','wb') as f:
    f.write(img_data)

# 識別驗證碼
code_data = getCodeData('bobo328410948', 'bobo328410948', './gs_code_img.jpg', 1004)

relogin_url = 'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx'
# 當有些參數是動態變化的時，咱們能夠去網頁源代碼中找找
data = {
    "__VIEWSTATE":"u+DzAnizDr8zKG7K/Q/OHyl4Kae1i0R5uxnuMk+EONOCJb5GTyGoJgnx1n/wlOx4XePU+CN5dRcmV/ptirBjyW6SyzcQqdOMuyeIbuFfEWNcUm7K00I9RH7g5gA=",
    "__VIEWSTATEGENERATOR":"C93BE1AE",
    "from":"http://so.gushiwen.org/user/collect.aspx",
    "email":"1547360919@qq.com",
    "pwd":"512abc...",
    "code":code_data,
    "denglu":"登陸"
} 
page_data = s.post(url=relogin_url, data=data, headers=headers, verify=False).text
with open('./gushici.html', 'w', encoding='utf8') as f:
    f.write(page_data)

四、隨機獲取User-Agent

# 可是因其服務器不穩定，有時候獲取不到，所以咱們不多用
from fake_useragent import UserAgent
ua = UserAgent(verify_ssl=False,use_cache_server=False).random
headers = {
    'User-Agent':ua
}

# fake_useragent的安裝方式爲 pip install fake-useragent

2、selenium

　　參考博客：http://www.javashuo.com/article/p-xihsrhom-ck.html服務器

　　selenium是一個第三方庫，對外提供的接口能夠操做瀏覽器，而後讓瀏覽器完成自動化操做。可用於獲取動態加載的數據。cookie

一、環境搭建

　　- 安裝

　　　　pip install selenium

　　- 獲取某一款瀏覽器的驅動程序（以谷歌瀏覽器爲例）

　　　　谷歌瀏覽器驅動下載地址：http://chromedriver.storage.googleapis.com/index.html

　　　　注意：下載的驅動程序必須和瀏覽器版本統一，你們能夠根據 http://blog.csdn.net/huilan_same/article/details/51896672 中提供的版本映射表進行對應

二、編碼流程

　　- 導包：from selenium import webdriver

　　- 實例化某一款瀏覽器對象

　　- 自制定自動化操做代碼

　　PS：本人在導包的時候遇到了以下錯誤：

　　查了半天，嘗試了各類都無論用，原來是我開着fiddler抓包軟件了，將其關掉就能夠了，[尷尬]！

三、簡單示例代碼

from selenium import webdriver
from time import sleep

bro = webdriver.Chrome(executable_path=r'D:\@Lily\myproject\pachong\chromedriver.exe')
bro.get(url='https://www.baidu.com/')
sleep(2)

text_input = bro.find_element_by_id('kw')    # 獲取到輸入框
text_input.send_keys('人民幣')
sleep(2)

search_btn = bro.find_element_by_id('su').click()    # 獲取到搜索按鈕，並點擊
sleep(2)

# 獲取當前的頁面源碼數據
page_text = bro.page_source print(page_text)

bro.quit()  # 關閉瀏覽器

四、示例：獲取豆瓣電影更多詳情數據

from selenium import webdriver
from time import sleep

url = 'https://movie.douban.com/typerank?type_name=%E5%96%9C%E5%89%A7&type=24&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'D:\@Lily\myproject\pachong\chromedriver.exe')

bro.get(url=url)
sleep(2)

bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')       # 瀏覽器向下滾動一屏的距離
sleep(2)

bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')   # 瀏覽器向下滾動一屏的距離
sleep(2)

bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')   # 瀏覽器向下滾動一屏的距離
sleep(2)

page_text = bro.page_source
with open('./douban.html','w',encoding='utf8') as f:
    f.write(page_text)
    
bro.quit()

五、示例：模擬登錄爬取QQ空間示例（嵌套iframe框架）

#qq空間
bro = webdriver.Chrome(executable_path=r'D:\@Lily\myproject\pachong\chromedriver.exe')
url = 'https://qzone.qq.com/'
bro.get(url=url)
sleep(2)
#定位到一個具體的iframe
bro.switch_to.frame('login_frame')
bro.find_element_by_id('switcher_plogin').click()
sleep(2)

bro.find_element_by_id('u').send_keys('QQ空間用戶名')
bro.find_element_by_id('p').send_keys('QQ空間密碼')

bro.find_element_by_id('login_button').click()

sleep(5)

page_text = bro.page_source
with open('qq.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
bro.quit()

六、PhantomJs（無界面瀏覽器）

# 以獲取豆瓣電影爲例，僅僅在實例化瀏覽器對象時有區別，其餘並沒有區別
from selenium import webdriver
from time import sleep

url = 'https://movie.douban.com/typerank?type_name=%E5%96%9C%E5%89%A7&type=24&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'D:\@Lily\myproject\pachong\phantomjs.exe')

bro.get(url=url)
sleep(2)

bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')       # 瀏覽器向下滾動一屏的距離
sleep(2)

bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')   # 瀏覽器向下滾動一屏的距離
sleep(2)

bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')   # 瀏覽器向下滾動一屏的距離
sleep(2)

page_text = bro.page_source
with open('./douban.html','w',encoding='utf8') as f:
    f.write(page_text)
    
bro.quit()

七、谷歌無頭瀏覽器

　　因爲PhantomJs最近已經中止了更新和維護，因此推薦使用谷歌的無頭瀏覽器，是一款無界面的谷歌瀏覽器。

　　示例代碼以下：

#谷歌無頭瀏覽器
from selenium.webdriver.chrome.options import Options

# 建立一個參數對象，用來控制chrome以無界面方式打開
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu') #獲取豆瓣電影中更多電影詳情數據
url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'D:\@Lily\myproject\pachong\chromedriver.exe',chrome_options=chrome_options)
bro.get(url)
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(2)
page_text = bro.page_source

with open('./douban.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
print(page_text)
sleep(1)
bro.quit()

3、線程池

一、示例：爬取梨視頻熱門視頻

# 爬取梨視頻數據
import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool
import random

# 實例化一個線程池對象
pool = Pool(5)
url = 'https://www.pearvideo.com/category_1'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
page_text = requests.get(url=url,headers=headers).text
tree = etree.HTML(page_text)
li_list = tree.xpath('//div[@id="listvideoList"]/ul/li')

video_url_list = []
for li in li_list:
    detail_url = 'https://www.pearvideo.com/'+li.xpath('./div/a/@href')[0]
    detail_page = requests.get(url=detail_url,headers=headers).text
    video_url = re.findall('srcUrl="(.*?)",vdoUrl',detail_page,re.S)[0]
    video_url_list.append(video_url)
    

video_data_list = pool.map(getVideoData, video_url_list)
pool.map(saveVideo, video_data_list)


def getVideoData(url):
    return requests.get(url=url,headers=headers).content


def saveVideo(data):
    fileName = str(random.randint(0,5000))+'.mp4'
    with open(fileName,'wb') as fp:
        fp.write(data)