Python爬蟲-selenium

時間 2021-01-24

標籤 html web mongodb 數據庫瀏覽器 echarts less dom 異步 ide 欄目 Python 简体版

原文原文鏈接

有態度地學習

對於Ajax加載的網頁已經分析了好幾次，這回來講說利用selenium自動化獲取網頁信息。html

一般對於異步加載的網頁，咱們須要查找網頁的真正請求，而且去構造請求參數，最後才能獲得真正的請求網址。而利用selenium經過模擬瀏覽器操做，則無需去考慮那麼多，作到可見便可爬。web

固然帶來便捷的同時，也有着不利，好比說時間上會有所增長，效率下降。但是對於業餘爬蟲而言，更快的爬取，並非那麼的重要。mongodb

首先在電腦的PyCharm上安裝selenium，而後下載與電腦上谷歌瀏覽器相對應版本的ChromeDriver。因爲個人Mac系統版本較新，須要先關閉Rootless內核保護機制，纔可以安裝，因此也是折騰一番後才成功安裝。數據庫

針對京東商城筆記本的網頁進行分析，這回只要在網頁源碼上分析，就能夠獲取筆記本價格、標題、評論數、商家名稱、商家性質。瀏覽器

爬取代碼以下：echarts

from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium import webdriver
from bs4 import BeautifulSoup
import pymongo
import time

# 鏈接數據庫
client = pymongo.MongoClient(host='localhost', port=27017)
db = client.JD_products
collection = db.products

# 啓動瀏覽器
browser = webdriver.Chrome()
wait = WebDriverWait(browser, 50)

def to_mongodb(data):
    # 存儲數據信息
    try:
        collection.insert(data)
        print("Insert The Data Successfully")
    except:
        print('Insert The Data Failed')

def search():
    browser.get('https://www.jd.com/')
    try:
        # 查找搜索框及搜索按鈕，輸入信息並點擊按鈕
        input = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#key")))
        submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#search > div > div.form > button")))
        input[0].send_keys('筆記本')
        submit.click()
        # 查找筆記本按鈕及銷量按鈕，依次點擊按鈕
        button_1 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#J_selector > div:nth-child(2) > div > div.sl-value > div.sl-v-list > ul > li:nth-child(1) > a")))
        button_1.click()
        button_2 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#J_filter > div.f-line.top > div.f-sort > a:nth-child(2)")))
        button_2.click()
        # 獲取總頁數
        page = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#J_bottomPage > span.p-skip > em:nth-child(1) > b')))
        return page[0].text
    except TimeoutException:
        search()

def next_page(page_number):
    try:
        # 滑動到網頁底部，加載出全部商品信息
        browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(10)
        html = browser.page_source
        parse_html(html)
        # 當網頁到達100頁時，下一頁按鈕失效，因此選擇結束程序
        while page_number == 101:
            exit()
        # 查找下一頁按鈕，並點擊按鈕
        button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_bottomPage > span.p-num > a.pn-next > em')))
        button.click()
        wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#J_goodsList > ul > li:nth-child(60)")))
        # 判斷翻頁成功
        wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, "#J_bottomPage > span.p-num > a.curr"), str(page_number)))
    except TimeoutException:
        return next_page(page_number)

def parse_html(html):
    """
    解析商品列表網頁
    """
    data = {}
    soup = BeautifulSoup(html, 'html.parser')
    goods_info = soup.select('.gl-item')
    # 查看當前頁商品數量，瞭解是否還有未加載的商品
    quantity = 'item: ' + str(len(goods_info))
    print(quantity)
    for info in goods_info:
        # 獲取商品標題信息
        title = info.select('.p-name.p-name-type-2 a em')[0].text.strip()
        title = title.replace('愛心東東', '')
        print("title: ", title)
        data['_id'] = title
        # 獲取商品價格信息
        price = info.select('.p-price i')[0].text.strip()
        price = int(float(price))
        print("price: ", price)
        data['price'] = price
        # 獲取商品的評論數量
        commit = info.select('.p-commit strong')[0].text.strip()
        commit = commit.replace('條評價', '')
        if '萬' in commit:
            commit = commit.split("萬")
            commit = int(float(commit[0]) * 10000)
        else:
            commit = int(float(commit.replace('+', '')))
        print("commit: ", commit)
        data['commit'] = commit
        # 獲取商品的商店名稱
        shop_name = info.select('.p-shop a')
        if (len(shop_name)) == 1:
            print("shop_name: ", shop_name[0].text.strip())
            data['shop_name'] = shop_name[0].text.strip()
        else:
            print("shop_name: ", '京東')
            data['shop_name'] = '京東'
        # 獲取商品的商店屬性
        shop_property = info.select('.p-icons i')
        if (len(shop_property)) >= 1:
            message = shop_property[0].text.strip()
            if message == '自營':
                print("shop_property: ", message)
                data['shop_property'] = message
            else:
                print("shop_property: ", '非自營')
                data['shop_property'] = '非自營'
        else:
            print("shop_property: ", '非自營')
            data['shop_property'] = '非自營'
        to_mongodb(data)
        print(data)
        print("\n\n")

def main():
    total = int(search())
    print(total)
    for i in range(2, total+2):
        time.sleep(20)
        print("第", i-1, "頁：")
        next_page(i)

if __name__ == "__main__":
    main()
less

雖然一開始就是以筆記本這個關鍵詞去搜索，可是這裏仍是須要再點擊一次筆記本按鈕，這是由於直接搜索筆記本會出現日常上課作筆記的那種筆記本，致使會獲取無用信息。因此利用京東自身更加詳細的歸類，獲得咱們想要的信息。dom

其中每個網頁有60條商品數據，那麼按道理應該有6000條的筆記本商品信息，可是最後卻只獲取了5992條。異步

估計兩個緣由：ide

1⃣️在MongoDB中商品的標題爲主鍵，商品標題出現重複

2⃣️網頁未能加載完全部的商品信息

最後成功獲取商品信息

讀取MongoDB中數據進行可視化分析

from pyecharts import Bar
import pandas as pd
import numpy as np
import pymongo

client = pymongo.MongoClient('localhost', 27017)
db = client.JD_products
table = db.products
df = pd.DataFrame(list(table.find()))
shop_message = df[df.shop_property == '自營'].groupby(['shop_name'])
shop_com = shop_message['shop_name'].agg(['count'])
shop_com.reset_index(inplace=True)
shop_com_last = shop_com.sort_values('count', ascending=False)[:12]
attr = np.array(shop_com_last['shop_name'])
v1 = np.array(shop_com_last['count'])
attr = ["{}".format(i.replace('京東', '').replace('旗艦店', '').replace('自營', '').replace('官方', '').replace('京東', '').replace('電腦', '').replace('產品專營店', '').replace('工做站', '').replace('筆記本', '')) for i in attr]
v1 = ["{}".format(i) for i in v1]
bar = Bar("京東自營商店筆記本種類排行", title_pos='center', title_top='18', width=800, height=400)
bar.add("商家", attr, v1, is_convert=True, xaxis_min=10, yaxis_label_textsize=12, is_yaxis_boundarygap=True, yaxis_interval=0, is_label_show=True, is_legend_show=False, label_pos='right', is_yaxis_inverse=True, is_splitline_show=False)
bar.render("京東自營商店筆記本種類排行.html")

從上面能夠看出，ThinkPad位居榜首，也與後面的詞雲圖有所呼應。商務、辦公，由於它就是一款以商務辦公爲主打品牌的筆記本。此外國內品牌聯想、華碩、宏碁、華爲也在榜上，支持國貨！！！

from pyecharts import Bar
import pandas as pd
import pymongo

client = pymongo.MongoClient('localhost', 27017)
db = client.JD_products
table = db.products
df = pd.DataFrame(list(table.find()))
price_info = df['price']
bins = [0, 2000, 2500, 3000, 3500, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 12000, 14000, 16000, 19000, 200000]
level = ['0-2000', '2000-2500', '2500-3000', '3000-3500', '3500-4000', '4000-5000', '5000-6000', '6000-7000', '7000-8000', '8000-9000', '9000-10000', '10000-12000', '12000-14000', '14000-16000', '16000-19000', '19000以上']
price_stage = pd.cut(price_info, bins=bins, labels=level).value_counts().sort_index()
attr = price_stage.index
v1 = price_stage.values
bar = Bar('筆記本價格分佈柱狀圖', title_pos='center', title_top='10', width=800, height=400)
bar.add('', attr, v1, is_stack=True, xaxis_rotate=30, yaxis_min=0, xaxis_interval=0, is_splitline_show=False, is_label_show=True)
bar.render('筆記本價格分佈柱狀圖.html')

筆記本價格區間在4000-6000有較大的集中，也必定程度反應出了如今筆記本的中間價位，記得剛上大學那會，價格在5000+的筆記本就能有着不錯的配置，LOL特效全開。

from pyecharts import Pie
import pandas as pd
import pymongo

client = pymongo.MongoClient('localhost', 27017)
db = client.JD_products
table = db.products
df = pd.DataFrame(list(table.find()))
shop_message = df.groupby(['shop_property'])
shop_com = shop_message['shop_property'].agg(['count'])
shop_com.reset_index(inplace=True)
shop_com_last = shop_com.sort_values('count', ascending=False)
attr = shop_com_last['shop_property']
v1 = shop_com_last['count']
pie = Pie('商店性質', title_pos='center', width=800, height=400)
pie.add('', attr, v1, radius=[40, 75], label_text_color=None, is_label_show=True, legend_orient='vertical', legend_pos='left')
pie.render('商店性質.html')

統計下來自營與非自營，仍是小巫見大巫。京東和淘寶最大的區別就是京東有自營產品，送貨也快。雖然說自營的也有假貨，可是仍是小几率事件。購買電子產品時，好比手機、電腦等，對於我這種小白而言，我第一選擇就是去官網或者京東自營店購買，我是絕對不會去電子城和姦商們鬥智鬥勇的，即便可能價格會低點。可是官網通常快遞比較慢，須要個3-5天，而京東可能只需1-2天，因此京東算是我購買的最優選擇。

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
import pandas as pd
import pymongo
import jieba
import re

client = pymongo.MongoClient('localhost', 27017)
db = client.JD_products
table = db.products
data = pd.DataFrame(list(table.find()))
data = data[['_id']]

text = ''
for line in data['_id']:
    r = '[a-zA-Z0-9’!"#$%&\'()*+,-./:;<=>?@，。?★、…【】《》？「」‘’！[\\]^_`{|}~]+'
    line = re.sub(r, '', line.replace('筆記本電腦', '').replace('英寸', ''))
    text += ' '.join(jieba.cut(line, cut_all=False))
backgroud_Image = plt.imread('computer.jpeg')
wc = WordCloud(
    background_color='white',
    mask=backgroud_Image,
    font_path='msyh.ttf',
    max_words=2000,
    stopwords=STOPWORDS,
    max_font_size=130,
    random_state=30
)
wc.generate_from_text(text)
img_colors = ImageColorGenerator(backgroud_Image)
wc.recolor(color_func=img_colors)
plt.imshow(wc)
plt.axis('off')
wc.to_file("computer.jpg")
print("生成詞雲成功")