閒來無事,玩了個成語接龍,因而就想用python爬取下成語網站上的成語,直接上代碼:html
#coding=utf-8 import requests from bs4 import BeautifulSoup import sqlite3 import uuid conn = sqlite3.connect("idiombase.db3") #建立sqlite.db數據庫 print ("open database success") conn.execute("drop table IF EXISTS idiom") query = """create table IF NOT EXISTS idiom( id VARCHAR(50), word VARCHAR(50) );""" conn.execute(query) print ("Table created successfully") all_url = 'http://chengyu.t086.com/' #http請求頭 Hostreferer = { 'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Referer':'http://chengyu.t086.com/' } word=['A','B','C','D','E','F','G','H','J','K','L','M','N','O','P','Q','R','S','T','W','X','Y','Z'] for w in word: for n in range(1,100): url=all_url+'list/'+w+'_'+str(n)+'.html' start_html = requests.get(url,headers = Hostreferer) if(start_html.status_code==404): break start_html.encoding='gb2312' soup = BeautifulSoup(start_html.text,"html.parser") listw = soup.find('div',class_='listw') lista = listw.find_all('a') for p in lista: print(p.text) ids=str(uuid.uuid1()) query = "insert into idiom (id,word) values ('"+ids+"','"+p.text+"');" conn.execute(query) conn.commit()
取到數據後,保存在裏sqlite數據庫中,之後想用的時候隨便取python