爬取所有的校園新聞

做業要求來源：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE2/homework/2941

要求：

一、重新聞url獲取新聞詳情

二、從列表頁的url獲取新聞url

三、生成所頁列表頁的url並獲取所有新聞

四、設置合理的爬取間隔

五、用pandas作簡單的數據處理並保存成csv和sql文件

源代碼：

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
import pandas as pd
import time
import random
import sqlite3

newsUrl = 'http://news.gzcc.cn/html/2005/xiaoyuanxinwen_0710/4.html'
listUrl = 'http://news.gzcc.cn/html/xiaoyuanxinwen/'


def click(url):
    id = re.findall('(\d{1,5})', url)[-1]
    clickUrl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(id)
    resClick = requests.get(clickUrl)
    newsClick = int(resClick.text.split('.html')[-1].lstrip("('").rstrip("');"))
    return newsClick


def newsdt(showinfo):
    newsDate = showinfo.split()[0].split(':')[1]
    newsTime = showinfo.split()[1]
    newsDT = newsDate + ' ' + newsTime
    dt = datetime.strptime(newsDT, '%Y-%m-%d %H:%M:%S')
    return dt


def anews(url):#重新聞url獲取新聞詳情： 字典,anews
    newsDetail = {}
    res = requests.get(url)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    newsDetail['newsTitle'] = soup.select('.show-title')[0].text
    showinfo = soup.select('.show-info')[0].text
    newsDetail['newsDT'] = newsdt(showinfo)
    newsDetail['newsClick'] = click(newsUrl)
    return newsDetail


def alist(url):#從列表頁的url獲取新聞url：列表append(字典) alist
    res = requests.get(listUrl)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    newsList = []
    for news in soup.select('li'):
        if len(news.select('.news-list-title')) > 0:
            newsUrl = news.select('a')[0]['href']
            newsDesc = news.select('.news-list-description')[0].text
            newsDict = anews(newsUrl)
            newsDict['description'] = newsDesc
            newsList.append(newsDict)
    return newsList


alist(listUrl)

alist(newsUrl)
res = requests.get('http://news.gzcc.cn/html/xiaoyuanxinwen/')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')

for news in soup.select('li'):
    if len(news.select('.news-list-title')) > 0:
        newsUrl = news.select('a')[0]['href']
        print(anews(newsUrl))

allnews = []
for i in range(97, 107):#爬取學號尾數開始的10個列表頁
    listUrl = 'http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html'.format(i)
    allnews.extend(alist(listUrl))

print("allnewsLength={}".format(len(allnews)))
print(allnews)

res = requests.get('http://news.gzcc.cn/html/xiaoyuanxinwen/')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
for news in soup.select('li'):
    if len(news.select('.news-list-title')) > 0:
        newsUrl = news.select('a')[0]['href']
        print(anews(newsUrl))

s1 = pd.Series([100, 23, 'bugingcode'])
print(s1)
pd.Series(anews)
newsdf = pd.DataFrame(allnews)
for i in range(5):
    print(i)
    time.sleep(random.random() * 3)#設置爬取的時間間隔
    print(newsdf)

newsdf.to_csv(r'D:\py_file\gzcc.csv',encoding='utf_8_sig')#保存成csv格式，爲避免亂碼，設置編碼格式爲utf_8_sig

with sqlite3.connect(r'D:\py_file\gzccnewsdb.sqlite') as db:#保存文件爲sql
    newsdf.to_sql('gzccnewsdb',db)

結果：

一、新聞詳情：

二、新聞列表：

三、保存成csv文件：

四、保存成爲sql文件