百度指數
百度指數數據是拼成圖片在客戶端顯示,要用到OCR識別,在此我了google和百度自帶的兩張OCR識別
# !/usr/bin/python3.4
# -*- coding: utf-8 -*-
# 百度指數的抓取
# 截圖教程：http://www.myexception.cn/web/2040513.html
#
# 登錄百度地址：https://passport.baidu.com/v2/?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2F
# 百度指數地址：http://index.baidu.com
import time
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from PIL import Image
import pytesseract
from aip import AipOcr
import os
import re
import threading
from queue import Queue
import json
import codecs
# 打開瀏覽器
def openbrowser():
global browser
# https://passport.baidu.com/v2/?login
url = 'http://index.baidu.com/?tpl=trend&word=%D5%BD%C0%C7'
# 打開谷歌瀏覽器
# Firefox()
# Chrome()
browser = webdriver.Chrome()
# 輸入網址
browser.get(url)
# 打開瀏覽器時間
# print("等待10秒打開瀏覽器...")
# time.sleep(10)
# 找到id="TANGRAM__PSP_3__userName"的對話框
# 清空輸入框
# TANGRAM__PSP_4__userName
# TANGRAM__PSP_4__password
browser.find_element_by_id("TANGRAM__PSP_4__userName").clear()
browser.find_element_by_id("TANGRAM__PSP_4__password").clear()
# 輸入帳號密碼
# 輸入帳號密碼
account = []
try:
fileaccount = open("../baidu/account.txt", encoding='UTF-8')
accounts = fileaccount.readlines()
for acc in accounts:
account.append(acc.strip())
fileaccount.close()
except Exception as err:
print(err)
input("請正確在account.txt裏面寫入帳號密碼")
exit()
browser.find_element_by_id("TANGRAM__PSP_4__userName").send_keys(account[0])
browser.find_element_by_id("TANGRAM__PSP_4__password").send_keys(account[1])
# 點擊登錄登錄
# id="TANGRAM__PSP_3__submit"
browser.find_element_by_id("TANGRAM__PSP_4__submit").click()
# 等待登錄10秒
# print('等待登錄10秒...')
# time.sleep(10)
print("等待網址加載完畢...")
# select = input("請觀察瀏覽器網站是否已經登錄(y/n)：")
select = 'y'
while 1:
if select == "y" or select == "Y":
print("登錄成功！")
print("準備打開新的窗口...")
# time.sleep(1)
# browser.quit()
break
elif select == "n" or select == "N":
selectno = input("帳號密碼錯誤請按0，驗證碼出現請按1...")
# 帳號密碼錯誤則從新輸入
if selectno == "0":
# 找到id="TANGRAM__PSP_3__userName"的對話框
# 清空輸入框
browser.find_element_by_id("TANGRAM__PSP_4__userName").clear()
browser.find_element_by_id("TANGRAM__PSP_4__password").clear()
# 輸入帳號密碼
account = []
try:
fileaccount = open("../baidu/account.txt", encoding='UTF-8')
accounts = fileaccount.readlines()
for acc in accounts:
account.append(acc.strip())
fileaccount.close()
except Exception as err:
print(err)
input("請正確在account.txt裏面寫入帳號密碼")
exit()
browser.find_element_by_id("TANGRAM__PSP_4__userName").send_keys(account[0])
browser.find_element_by_id("TANGRAM__PSP_4__password").send_keys(account[1])
# 點擊登錄sign in
# id="TANGRAM__PSP_3__submit"
browser.find_element_by_id("TANGRAM__PSP_4__submit").click()
elif selectno == "1":
# 驗證碼的id爲id="ap_captcha_guess"的對話框
input("請在瀏覽器中輸入驗證碼並登錄...")
select = input("請觀察瀏覽器網站是否已經登錄(y/n)：")
else:
print("請輸入「y」或者「n」！")
select = input("請觀察瀏覽器網站是否已經登錄(y/n)：")
def getindex(keyword, day):
try:
openbrowser()
time.sleep(3)
# try:
# 這裏開始進入百度指數
# 要不這裏就不要關閉了，新打開一個窗口
# http://blog.csdn.net/DongGeGe214/article/details/52169761
# 新開一個窗口，經過執行js來新開一個窗口
js = 'window.open("http://index.baidu.com");'
browser.execute_script(js)
# 新窗口句柄切換，進入百度指數
# 得到當前打開全部窗口的句柄handles
# handles爲一個數組
handles = browser.window_handles
# print(handles)
# 切換到當前最新打開的窗口
browser.switch_to_window(handles[-1])
# 在新窗口裏面輸入網址百度指數
# 清空輸入框
# time.sleep(1)
browser.find_element_by_id("search-input-word").clear()
# time.sleep(1)
# 寫入須要搜索的百度指數
browser.find_element_by_id("search-input-word").send_keys(keyword)
# 點擊搜索
# <input type="submit" value="" id="searchWords" onclick="searchDemoWords()">
browser.find_element_by_id("searchWords").click()
# time.sleep(2)
# 最大化窗口
browser.maximize_window()
time.sleep(5)
# 構造天數
sel = '//a[@rel="' + str(day) + '"]'
browser.find_element_by_xpath(sel).click()
# 太快了
time.sleep(2)
# 滑動思路：http://blog.sina.com.cn/s/blog_620987bf0102v2r8.html
# 滑動思路：http://blog.csdn.net/zhouxuan623/article/details/39338511
# 向上移動鼠標80個像素，水平方向不一樣
# ActionChains(browser).move_by_offset(0,-80).perform()
# <div id="trend" class="R_paper" style="height:480px;_background-color:#fff;"><svg height="460" version="1.1" width="954" xmlns="http://www.w3.org/2000/svg" style="overflow: hidden; position: relative; left: -0.5px;">
# <rect x="20" y="130" width="914" height="207.66666666666666" r="0" rx="0" ry="0" fill="#ff0000" stroke="none" opacity="0" style="-webkit-tap-highlight-color: rgba(0, 0, 0, 0); opacity: 0;"></rect>
# xoyelement = browser.find_element_by_xpath('//rect[@stroke="none"]')
# xoyelement = browser.find_elements_by_css_selector("#trend rect")[2]
xoyelement = browser.find_element_by_xpath('//div[@id="trend"]')
print('xoyelement:%s' % xoyelement)
num = 0
# 得到座標長寬
# x = xoyelement.location['x']
# y = xoyelement.location['y']
# width = xoyelement.size['width']
# height = xoyelement.size['height']
# print(x,y,width,height)
# 經常使用js:http://www.cnblogs.com/hjhsysu/p/5735339.html
# 搜索詞：selenium JavaScript模擬鼠標懸浮
x_0 = 30
y_0 = 248
if day == "all":
day = 1000000
# 儲存數字的數組
index = []
try:
# webdriver.ActionChains(driver).move_to_element().click().perform()
# 只有移動位置xoyelement[2]是準確的
for i in range(day):
# 座標偏移量???
ActionChains(browser).move_to_element_with_offset(xoyelement, x_0, y_0).perform()
# ActionChains(browser).move_to_element(xoyelement, x_0, y_0).perform()
time.sleep(2)
# 構造規則
if day == 7:
x_0 = x_0 + 202
elif day == 30:
# x_0 = x_0 + 41.68
x_0 = x_0 + 40
elif day == 90:
x_0 = x_0 + 13.64
elif day == 180:
x_0 = x_0 + 6.78
elif day == 1000000:
x_0 = x_0 + 3.37222222
# <div class="imgtxt" style="margin-left:-117px;"></div>
imgelement = browser.find_element_by_xpath('//div[@id="viewbox"]')
imgelement_index = browser.find_element_by_xpath('//div[@id="trend"]')
# imgelement = browser.find_element_by_xpath('//div[@id="trend"]')
print('imgelement:%s' % imgelement)
# 找到圖片座標
locations = imgelement.location
locations_index = imgelement_index.location
print('locations_index:%s' % locations_index)
print('locations:%s' % locations)
time.sleep(1)
# 跨瀏覽器兼容
scroll = browser.execute_script("return window.scrollY;")
print('scroll:%s' % scroll)
top = locations['y'] - scroll
# 找到圖片大小
sizes = imgelement.size
print('sizes:%s' % sizes)
# 構造關鍵詞長度
add_length = (len(keyword) - 2) * sizes['width'] / 15
# 構造指數的位置
rangle = (
int(locations['x'] + sizes['width'] / 4 + add_length), int(top + sizes['height'] / 2),
int(locations['x'] + sizes['width'] * 2 / 3), int(top + sizes['height']))
# 構造指數圖片位置
rangle_index = (
int(locations_index['x']), int(locations_index['y']),
int(int(locations_index['x']) + 1256), int(int(locations_index['y']) + 365))
# 截取當前瀏覽器
path = "../baidu/" + str(num)
browser.save_screenshot(str(path) + ".png")
# 打開截圖切割
img = Image.open(str(path) + ".png")
jpg = img.crop(rangle)
jpg.save(str(path) + ".jpg")
jpg_index = img.crop(rangle_index)
jpg_index.save(str(path) + str(time.strftime('%Y-%m-%d', time.localtime(time.time()))) + ".jpg")
# 將圖片放大一倍
# 原圖大小73.29
jpgzoom = Image.open(str(path) + ".jpg")
(x, y) = jpgzoom.size
x_s = x * 2
y_s = y * 3
out = jpgzoom.resize((x_s, y_s), Image.ANTIALIAS)
out.save(path + 'zoom.jpg', 'png', quality=95)
# 圖像識別
try:
# 調用goggle圖像識別
image = Image.open(str(path) + "zoom.jpg")
code = pytesseract.image_to_string(image)
# # 調用百度識別接口
# """ 你的 APPID AK SK """
# APP_ID = '10913815'
# API_KEY = 'oLZyPe7PwKzeWhHw7d81oLzy'
# SECRET_KEY = 'KK5BotVGvbsgDI8Sy29ir5gVwEdXB5oM'
#
# client = AipOcr(APP_ID, API_KEY, SECRET_KEY)
#
# """ 讀取圖片 """
#
# def get_file_content(filePath):
# with open(filePath, 'rb') as fp:
# return fp.read()
#
# # image = get_file_content('29zoom.jpg')
# print('str(path) + "zoom.jpg":%s'%(str(path) + "zoom.jpg"))
# image = get_file_content(str(path) + "zoom.jpg")
#
# """ 調用通用文字識別, 圖片參數爲本地圖片 """
# data = client.basicGeneral(image);
# code = data['words_result'][0]['words']
#
# print(data['words_result'][0]['words'])
# 建立儲存字段
data_list = {}
data_list['keyword'] = keyword
data_list['current_time'] = int(time.time()) - 86400
if code:
# index.append(code)
print('type(code):%s' % type(code))
# 去掉非字母和數字
data_initial = re.findall(r"[\w]+", code)
data = ''.join(data_initial)
print(data)
# 數據中的字母提早出來
letters = re.findall(r"[a-zA-Z]+", data)
print(letters)
print(type(letters))
num = 0
# 字母用數值替換
if letters:
for letter in letters:
num += 1
print(num)
if letter == 'T' or 't':
data = data.replace(letter, '7')
elif letter == 'B' or 'b':
data = data.replace(letter, '8')
else:
data = data.replace(letter, '這個數據匹配有問題')
data_list['index'] = data
print(data_list)
# 數據保存json格式
data_json = json.dumps(data_list, ensure_ascii=False) + ",\n"
print('*' * 100)
print('data_json:%s' % data_json)
print('*' * 100)
print(type(data_json))
with open("index.json", "ab") as fi:
fi.write(data_json.encode('utf-8'))
# data_json = json.dump(data_list, ensure_ascii=False)
# print(type(data_json))
# with open("index.json", "wb", encoding='UTF-8') as f:
# f.write(data_json.encode("utf-8") + "\n")
# # json.dump(data_list, codecs.open('data.json', 'a', 'utf-8'), ensure_ascii=False, indent=2)
print(data)
index.append(data)
else:
index.append("數據獲取沒有成功")
data_list['index'] = '數據獲取沒有成功'
print(data_list)
data_json = json.dumps(data_list, ensure_ascii=False) + ",\n"
print(type(data_json))
with open("error.json", "ab") as f:
f.write(data_json.encode('utf-8'))
except Exception as f:
index.append("數據獲取沒有成功")
data_list['index'] = '數據獲取沒有成功'
print(data_list)
data_json = json.dumps(data_list, ensure_ascii=False) + ",\n"
print(type(data_json))
with open("error.json", "ab") as f:
f.write(data_json.encode('utf-8'))
print('*' * 20)
print(f)
finally:
num = num + 1
os.remove(str(path) + "zoom.jpg")
os.remove(str(path) + ".jpg")
os.remove(str(path) + ".png")
os.remove(str(path) + "index.jpg")
except Exception as err:
print('err:%s' % err)
print('num:%s' % num)
finally:
print(index)
browser.quit()
# 日期也是能夠圖像識別下來的
# 只是要構造rangle就行，可是我就是懶
file = open("../baidu/index.txt", "w")
for item in index:
file.write(str(item) + "\n")
file.close()
time.sleep(2)
except Exception as err:
print(err)
browser.quit()
if __name__ == "__main__":
# 每一個字大約佔橫座標12.5這樣
# 按照字節可自行更改切割橫座標的大小rangle
# keyword = input("請輸入查詢關鍵字：")
# sel = int(input("查詢7天請按0，30天請按1，90天請按2，半年請按3，所有請按4："))
# day = 0
# if sel == 0:
# day = 7
# elif sel == 1:
# day = 30
# elif sel == 2:
# day = 90
# elif sel == 3:
# day = 180
# elif sel == 4:
# day = "all"
# 讀取文件的數據
file = open('TVB電視劇名.txt', 'r', encoding='UTF-8')
line = file.readlines()
# line = ['戰狼', '小雨', '大雨']
# # 取出文件並去掉換行符號
thread_list = []
for i in line:
data_initial = re.findall(r"[\u4e00-\u9fa5,0-9,a-z,A-Z]+", i)
keyword_data = ''.join(data_initial)
getindex(keyword_data, 7)
# # data_initial = re.findall(r"[\w]+", i)
# # data = ''.join(data_initial)
# thread = threading.Thread(target=getindex, args=[keyword, 30])
# # 執行線程
# thread.start()
# # 將全部線程存到列表裏
# thread_list.append(thread)
# time.sleep(10)
#
# for thread in thread_list:
# # 讓父線程等待全部的子線程結束，本身再結束
# thread.join()
#
# # print(a)
# # print(type(line))
# #
# # print(line)
#
# # file.close()
#
# # getindex('戰狼', 30)
相關文章
- 1. 百度指數爬取
- 2. python3百度指數抓取
- 3. 百度指數幫助
- 4. 百度指數感想
- 5. 關鍵詞百度指數疊加-關鍵詞百度指數疊加
- 6. 什麼是百度權重 百度指數詞
- 7. 百度指數完美採集器
- 8. 論文百度指數抓取方法
- 9. 百度指數和谷歌趨勢
- 10. 百度指數 爬蟲 更新版
- 更多相關文章...
- • PHP 獲取圖像寬度與高度 - PHP參考手冊
- • SQL 指南 - 網站建設指南
- • 算法總結-雙指針
- • Flink 數據傳輸及反壓詳解
相關標籤/搜索
每日一句
-
每一个你不满意的现在，都有一个你没有努力的曾经。
歡迎關注本站公眾號,獲取更多信息
相關文章
- 1. 百度指數爬取
- 2. python3百度指數抓取
- 3. 百度指數幫助
- 4. 百度指數感想
- 5. 關鍵詞百度指數疊加-關鍵詞百度指數疊加
- 6. 什麼是百度權重 百度指數詞
- 7. 百度指數完美採集器
- 8. 論文百度指數抓取方法
- 9. 百度指數和谷歌趨勢
- 10. 百度指數 爬蟲 更新版