利用chrome瀏覽器爬取數據

相關的庫本身下載吧，直接上代碼

 

from selenium import webdriver
from bs4 import BeautifulSoup
import time

#手動添加路徑
path = "C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe"
driver = webdriver.Chrome(executable_path=path)

url = "https://www.huomao.com/channel/lol"

# 司機開車了
driver.get(url)
time.sleep(5)
# 讓頁面移到最下面點擊加載，連續6次，司機會自動更新！！
# for i in range(6):
#     driver.find_element_by_id("獲取更多").click()
#     time.sleep(1)

# 開始解析
soup = BeautifulSoup(driver.page_source, "html.parser")


page_all = soup.find("div", attrs={"id": "channellist"})

pages = page_all.find_all("div", attrs={"class": "list-smallbox no-logo"})



for page in pages:
    aa=page.find('a')
    # print(aa)
    # print(aa.attrs['title'])
    bb=page.find('em').string.strip()
    print("主播房間:" + bb)
    cc=page.find('span',attrs={"class": "nickname"}).string.strip()
    print("主播:" + cc)

    dd = page.find_all('em')
    if len(dd)==2:
        ee = dd[1].find('span').string.strip()
        print('人氣:' + ee)
    else:
        print('人氣:主播休息了' )
    # print(len(dd))
    # for dds in dd:
    #     print(dds)