Python 爬取51cto博客數據存入MySQL

實驗環境

1.安裝Python 3.7
2.安裝requests, bs4，pymysql 模塊

實驗步驟

1.安裝環境及模塊

可參考博客https://blog.51cto.com/13760351/2500048

2.編寫代碼

# 51cto 博客頁面數據插入mysql數據庫
# 導入模塊
import re
import bs4
import pymysql
import requests

# 鏈接數據庫帳號密碼
db = pymysql.connect(host='172.171.13.229',
                     user='root', passwd='abc123',
                     db='test', port=3306,
                     charset='utf8')
# 獲取遊標
cursor = db.cursor()

def open_url(url):
    # 鏈接模擬網頁訪問
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/57.0.2987.98 Safari/537.36'}
    res = requests.get(url, headers=headers)
    return res

# 爬取網頁內容
def find_text(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')

    # 博客名
    titles = []
    targets = soup.find_all("a", class_="tit")
    for each in targets:
        each = each.text.strip()
        if "置頂" in each:
            each = each.split(' ')[0]
        titles.append(each)

    # 閱讀量
    reads = []
    read1 = soup.find_all("p", class_="read fl on")
    read2 = soup.find_all("p", class_="read fl")
    for each in read1:
        reads.append(each.text)
    for each in read2:
        reads.append(each.text)

    # 評論數
    comment = []
    targets = soup.find_all("p", class_='comment fl')
    for each in targets:
        comment.append(each.text)

    # 收藏
    collects = []
    targets = soup.find_all("p", class_='collect fl')
    for each in targets:
        collects.append(each.text)

     # 發佈時間
    dates=[]
    targets = soup.find_all("a", class_='time fl')
    for each in targets:
        each = each.text.split('：')[1]
        dates.append(each)

    # 插入sql 語句
    sql = """insert into blog (blog_title,read_number,comment_number, collect, dates)
    values( '%s', '%s', '%s', '%s', '%s');"""
    # 替換頁面 \xa0
    for titles, reads, comment, collects, dates in zip(titles, reads, comment, collects, dates):
        reads = re.sub('\s', '', reads)
        comment = re.sub('\s', '', comment)
        collects = re.sub('\s', '', collects)
        cursor.execute(sql % (titles, reads, comment, collects，dates))
        db.commit()
        pass

# 統計總頁數
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.find('li', class_='next').previous_sibling.previous_sibling.text
    return int(depth)

# 主函數
def main():
    host = "https://blog.51cto.com/13760351"
    res = open_url(host)  # 打開首頁連接
    depth = find_depth(res)  # 獲取總頁數

    # 爬取其餘頁面信息
    for i in range(1, depth + 1):
        url = host + '/p' + str(i)  # 完整連接
        res = open_url(url)  # 打開其餘連接
        find_text(res)  # 爬取數據

    # 關閉遊標
    cursor.close()
    # 關閉數據庫鏈接
    db.close()

if __name__ == '__main__':
    main()

3..MySQL建立對應的表

CREATE TABLE `blog` (
  `row_id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主鍵',
  `blog_title` varchar(52) DEFAULT NULL COMMENT '博客標題',
  `read_number` varchar(26) DEFAULT NULL COMMENT '閱讀數量',
  `comment_number` varchar(16) DEFAULT NULL COMMENT '評論數量',
  `collect` varchar(16) DEFAULT NULL COMMENT '收藏數量',
  `dates` varchar(16) DEFAULT NULL COMMENT '發佈日期',
  PRIMARY KEY (`row_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

4.運行代碼，查看效果:

改進版：

改進內容：
1.數據庫裏面的某些字段只保留數字便可
2.默認爬取的內容都是字符串，存放數據庫的某些字段，最好改成整型，方便後面數據庫操做

1.代碼以下：

import re
import bs4
import pymysql
import requests

# 鏈接數據庫
db = pymysql.connect(host='172.171.13.229',
                     user='root', passwd='abc123',
                     db='test', port=3306,
                     charset='utf8')
# 獲取遊標
cursor = db.cursor()

def open_url(url):
    # 鏈接模擬網頁訪問
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/57.0.2987.98 Safari/537.36'}
    res = requests.get(url, headers=headers)
    return res

# 爬取網頁內容
def find_text(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')

    # 博客標題
    titles = []
    targets = soup.find_all("a", class_="tit")
    for each in targets:
        each = each.text.strip()
        if "置頂" in each:
            each = each.split(' ')[0]
        titles.append(each)

    # 閱讀量
    reads = []
    read1 = soup.find_all("p", class_="read fl on")
    read2 = soup.find_all("p", class_="read fl")
    for each in read1:
        reads.append(each.text)
    for each in read2:
        reads.append(each.text)

    # 評論數
    comment = []
    targets = soup.find_all("p", class_='comment fl')
    for each in targets:
        comment.append(each.text)

    # 收藏
    collects = []
    targets = soup.find_all("p", class_='collect fl')
    for each in targets:
        collects.append(each.text)

    # 發佈時間
    dates=[]
    targets = soup.find_all("a", class_='time fl')
    for each in targets:
        each = each.text.split('：')[1]
        dates.append(each)

    # 插入sql 語句
    sql = """insert into blogs (blog_title,read_number,comment_number, collect, dates)
    values( '%s', '%s', '%s', '%s', '%s');"""
    # 替換頁面 \xa0
    for titles, reads, comment, collects, dates in zip(titles, reads, comment, collects, dates):
        reads = re.sub('\s', '', reads)
        reads=int(re.sub('\D', "", reads))  #匹配數字，轉換爲整型
        comment = re.sub('\s', '', comment)
        comment = int(re.sub('\D', "", comment))  #匹配數字，轉換爲整型
        collects = re.sub('\s', '', collects)
        collects = int(re.sub('\D', "", collects))  #匹配數字，轉換爲整型
        dates = re.sub('\s', '', dates)
        cursor.execute(sql % (titles, reads, comment, collects,dates))
        db.commit()
        pass

# 統計總頁數
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.find('li', class_='next').previous_sibling.previous_sibling.text
    return int(depth)

# 主函數
def main():
    host = "https://blog.51cto.com/13760351"
    res = open_url(host)  # 打開首頁連接
    depth = find_depth(res)  # 獲取總頁數

    # 爬取其餘頁面信息
    for i in range(1, depth + 1):
        url = host + '/p' + str(i)  # 完整連接
        res = open_url(url)  # 打開其餘連接
        find_text(res)  # 爬取數據

    # 關閉遊標
    cursor.close()
    # 關閉數據庫鏈接
    db.close()

#主程序入口
if __name__ == '__main__':
    main()

2.建立對應表

CREATE TABLE `blogs` (
  `row_id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主鍵',
  `blog_title` varchar(52) DEFAULT NULL COMMENT '博客標題',
  `read_number` int(26) DEFAULT NULL COMMENT '閱讀數量',
  `comment_number` int(16) DEFAULT NULL COMMENT '評論數量',
  `collect` int(16) DEFAULT NULL COMMENT '收藏數量',
  `dates` varchar(16) DEFAULT NULL COMMENT '發佈日期',
  PRIMARY KEY (`row_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

3.運行代碼，驗證

升級版

爲了能讓小白就能夠使用這個程序，能夠把這個項目打包成exe格式的文件，讓其餘人，使用電腦就能夠運行代碼，這樣很是方便！

1.改進代碼：

#末尾修改成：
if __name__ == '__main__':
    main()
    print("\n\t\t全部數據已成功存放數據庫！!! \n")
    time.sleep(5)

2.安裝打包模塊pyinstaller(cmd安裝）

pip install pyinstaller -i https://pypi.tuna.tsinghua.edu.cn/simple/

3.Python代碼打包

1.切換到須要打包代碼的路徑下面
2.在cmd窗口運行 pyinstaller -F test03.py （test03爲項目名稱）

4.查看exe包

在打包後會出現dist目錄，打好包就在這個目錄裏面

5.運行exe包，查看效果

檢查數據庫

總結：

1.這一篇博客，是在上一篇的基礎上改進的，步驟是先爬取首頁的信息，再爬取其餘頁面信息，最後在改進細節，打包exe文件
2.咱們爬取網頁數據大多數仍是存放到數據庫的，因此這種方法很實用。
3.其實在此博客的基礎上仍是能夠改進的，重要的是掌握方法便可。