爬蟲第三彈 selenium

時間 2019-12-15

原文原文鏈接

reptile_thirdphp

雲打碼平臺

註冊: 普通用戶和開發者用戶
登陸:
- 登陸普通用戶(查看餘額)
- 登陸開發者用戶:
  - 建立一個軟件: 個人軟件-> 建立軟件
  - 下載示例代碼: 開發者中心->下載最新的dll->pythonHttp示例代碼下載

import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

######################################################################

# 用戶名(普通用戶)
username    = 'bobo328410948'

# 密碼
password    = 'bobo328410948'                            

# 軟件ＩＤ，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
appid       = 6003                                    

# 軟件密鑰，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
appkey      = '1f4b564483ae5c907a1d34f8e2f2776c'    

# 圖片文件
filename    = 'getimage.jpg'                        

# 驗證碼類型，# 例：1004表示4位字母數字，不一樣類型收費不一樣。請準確填寫，不然影響識別率。在此查詢全部類型 http://www.yundama.com/price.html
codetype    = 1004

# 超時時間，秒
timeout     = 10                                    

# 檢查
if (username == 'username'):
    print('請設置好相關參數再測試')
else:
    # 初始化
    yundama = YDMHttp(username, password, appid, appkey)

    # 登錄雲打碼
    uid = yundama.login();
    print('uid: %s' % uid)

    # 查詢餘額
    balance = yundama.balance();
    print('balance: %s' % balance)

    # 開始識別，圖片路徑，驗證碼類型ID，超時時間（秒），識別結果
    cid, result = yundama.decode(filename, codetype, timeout);
    print('cid: %s, result: %s' % (cid, result))

######################################################################

def getCodeDate(userName,pwd,codePath,codeType):
    # 用戶名(普通用戶)
    username    = userName

    # 密碼
    password    = pwd                            

    # 軟件ＩＤ，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
    appid       = 6003                                    

    # 軟件密鑰，開發者分紅必要參數。登陸開發者後臺【個人軟件】得到！
    appkey      = '1f4b564483ae5c907a1d34f8e2f2776c'    

    # 圖片文件
    filename    = codePath                       

    # 驗證碼類型，# 例：1004表示4位字母數字，不一樣類型收費不一樣。請準確填寫，不然影響識別率。在此查詢全部類型 http://www.yundama.com/price.html
    codetype    = codeType

    # 超時時間，秒
    timeout     = 2                                   
    result = None
    # 檢查
    if (username == 'username'):
        print('請設置好相關參數再測試')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)

        # 登錄雲打碼
        uid = yundama.login();
        #print('uid: %s' % uid)

        # 查詢餘額
        balance = yundama.balance();
        #print('balance: %s' % balance)

        # 開始識別，圖片路徑，驗證碼類型ID，超時時間（秒），識別結果
        cid, result = yundama.decode(filename, codetype, timeout);
        #print('cid: %s, result: %s' % (cid, result))
    return result

人人網的模擬登陸

import urllib
import requests
from lxml import etree

# 獲取session對象
session = requests.Session()
# 將驗證碼圖片進行下載
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
url = 'http://www.renren.com/'
page_text = requests.get(url=url,headers=headers).text

tree = etree.HTML(page_text)
code_img_url = tree.xpath('//*[@id="verifyPic_login"]/@src')[0]
urllib,request.urlretrieve(url=code_img_url,filename='code.jpg')

#識別驗證碼中的數據值
code_data = getCodeDate('bobo328410948','bobo328410948','./code.jpg',2004)

# 模擬登陸
login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201914927558'
data = {
    "email":"www.zhangbowudi@qq.com",
    "icode":code_data,
    "origURL":"http://www.renren.com/home",
    "domain":"renren.com",
    "key_id":"1",
    "captcha_type":"web_login",
    "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1",
    "rkey":"9e75e8dc3457b14c55a74627fa64fb43",
    "f":"http%3A%2F%2Fwww.renren.com%2F289676607",
}

#該次請求產生的coolie會被自動存儲到session對象中
session.post(url=login_url,data=data,headers=headers)

url = 'http://www.renren.com/289676607/profile'
page_text = session.get(url=url,headers=headers).text

with open('renren.html', 'w', encoding='utf-8') as fp:
    fp.write(page_text)

模擬登陸古詩文網

import urllib
import requests
from lxml import etree
s = requests.Session()
login_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx'
page_text = requests.get(url=login_url,headers=headers).text
tree = etree.HTML(page_text)
img_src = 'https://so.gushiwen.org' + tree.xpath('//*[@id="imgCode"]/@src')[0]
img_data = s.get(url=img_src,headers=headers).content
with open('./img.jpg','wb') as fp:
    fp.write(img_data)

img_text = getCodeDate('bobo328410948','bobo328410948','./img.jpg',1004)

# 模擬登陸
url = 'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx'
data = {
    "__VIEWSTATE":"9AsGvh3Je/0pfxId7DYRUi258ayuEG4rrQ1Z3abBgLoDSOeAUatOZOrAIxudqiOauXpR9Zq+dmKJ28+AGjXYHaCZJTTtGgrEemBWI1ed7oS7kpB7Rm/4yma/+9Q=",
    "__VIEWSTATEGENERATOR":"C93BE1AE",
    "from":"http://so.gushiwen.org/user/collect.aspx",
    "email":"www.zhangbowudi@qq.com",
    "pwd":"bobo328410948",
    "code":img_text,
    "denglu":"登陸",
}

page_text = s.post(url=url,headers=headers,data=data).text
with open ('./gushiwen.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

環境安裝: pip install selenium
編碼流程:
- 導包 : from selenium import webdriver
- 實例化某一款瀏覽器對象
- 自指定自動化操做代碼

百度人民幣

from selenium import webdriver
from time import sleep


bro = webdriver.Chrome(executable_path=r'C:\Users\Admin\Documents\Tencent Files\1973727272\FileRecv\day_03_爬蟲\chromedriver.exe')
bro.get(url='https://www.baidu.com/')
sleep(2)
text_input = bro.find_element_by_id('kw')

text_input.send_keys('人民幣')
sleep(2)
bro.find_element_by_id('su').click()

sleep(3)

#獲取當前的頁面源碼數據
print(bro.page_source)
bro.quit()

獲取豆瓣電影中更多電影詳情數據

from selenium import webdriver
from time import sleep
url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'C:\Users\Admin\Documents\Tencent Files\1973727272\FileRecv\day_03_爬蟲\chromedriver.exe')
bro.get(url)
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(2)
page_text = bro.page_source

with open('./douban.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

sleep(1)
bro.quit()

qq空間

bro = webdriver.Chrome(executable_path=r'C:\Users\Admin\Documents\Tencent Files\1973727272\FileRecv\day_03_爬蟲\chromedriver.exe')
url = 'https://qzone.qq.com/'
bro.get(url=url)
sleep(2)
# 定位到一個具體的iframe
bro.switch_to.frame('login_frame')
bro.find_element_by_id('switcher_plogin').click()
sleep(2)

bro.find_element_by_id('u').send_keys('1973727272')
bro.find_element_by_id('p').send_keys('')

bro.find_element_by_id('login_button').click()
sleep(5)

page_text = bro.page_source
with open('qq.html', 'w', encoding='utf-8') as fp:
    fp.write(page_text)
bro.quit()

相關標籤/搜索

每日一句

每一个你不满意的现在，都有一个你没有努力的曾经。