python爬取course課程的信息

目錄

• 1.大模塊頁面
• 2.每一個大模塊中小模塊的簡單信息
• 3.每一個小課程的詳細信息
• 4.爬取全部評論

@
  這幾天爬取了course動態網頁的課程信息，有關數據分析，機器學習,還有機率論和數理統計課程，這裏數據過多，只對數據分析的模塊進行了詳細爬取。爲以後的數據分析以及機器學習情感處理進行數據獲取，下面直接上代碼和截圖：

1.大模塊頁面

頁面以下：

爬取代碼：

# encoding: utf-8
from selenium import webdriver
import csv

class GetModel():
    '''
    類功能介紹：獲取每頁每一個大模塊的類型，標題，url
    類成員變量：browser對象，每頁的url
    類成員函數：初始化函數__init__，獲取每一個大模塊信息的函數getEveryModel，寫入csv的函數saveToCsv
    '''
    def __init__(self,url):
        self.__browser = webdriver.Chrome()
        self.__page_url = url

    def saveToCsv(self,this_list):
        with open("model.csv", "a", newline='', encoding='utf-8') as f:
            writer = csv.writer(f, delimiter=',')
            writer.writerow(this_list)

    def getEveryModel(self):
        # 隱式等待
        self.__browser.implicitly_wait(30)
        self.__browser.get(self.__page_url)
        model_type = self.__browser.find_elements_by_css_selector(".product-type-row.horizontal-box")
        model_title = self.__browser.find_elements_by_css_selector(".color-primary-text.card-title.headline-1-text")
        model_url = self.__browser.find_elements_by_css_selector(".rc-DesktopSearchCard.anchor-wrapper.browse-result-card")

        for i in range(len(model_type)):
            every_model = []
            every_model.append(model_type[i].text)
            every_model.append(model_title[i].text)
            every_model.append(model_url[i].get_attribute('href'))
            self.saveToCsv(every_model)

    def closeUrl(self):
        self.__browser.close()

結果以下：

2.每一個大模塊中小模塊的簡單信息

頁面以下：

查看所有的信息要先模擬點擊查看全部頁面：

代碼以下：

# encoding: utf-8
from selenium import webdriver
import csv
import time

class GetCourse():
    '''
    類功能介紹：獲取每一個大模塊的小課程的信息
    類成員變量：browser對象，每一個大模塊的url，大模塊類別，大模塊標題
    類成員函數：初始化函數__init__，獲取每一個小課程信息的函數getEveryCourse，寫入csv的函數saveToCsv
    '''
    def __init__(self,url,model_type,model_title):
        self.__browser = webdriver.Chrome()
        self.__page_url = url
        self.__model_type = model_type
        self.__model_title = model_title

    def saveToCsv(self,this_list):
        with open("course2.csv", "a", newline='', encoding='utf-8') as f:
            writer = csv.writer(f, delimiter=',')
            writer.writerow(this_list)

    def getEveryCourse(self):
        self.__browser.get(self.__page_url)
        # 隱式等待
        self.__browser.implicitly_wait(15)
        try:
            button = self.__browser.find_element_by_css_selector(".Button_1w8tm98-o_O-default_9vdknu-o_O-md_1jvotax.m-t-1.d-block.m-x-auto")
            button.click()
        except:
            print("沒有多餘項")
        finally:
            self.__browser.implicitly_wait(20)
            course_title = self.__browser.find_elements_by_css_selector(".H2_1pmnvep-o_O-weightBold_uvlhiv-o_O-bold_1byw3y2.m-b-2")
            # course_url   = self.__browser.find_elements_by_css_selector(".Col_i9j08c-o_O-xsCol12_1m1ceo5-o_O-mdCol10_1eb21lj-o_O-lgCol10_ra5osh.p-b-3.border-bottom a")
            course_score = self.__browser.find_elements_by_css_selector(".H4_1k76nzj-o_O-weightBold_uvlhiv-o_O-bold_1byw3y2.m-l-1s")
            commet_num   = self.__browser.find_elements_by_css_selector(".P_gjs17i-o_O-weightNormal_s9jwp5-o_O-fontBody_56f0wi.m-r-1s")
            comment_number = self.__browser.find_elements_by_css_selector(".reviewsCount")

            for i in range(len(course_title)):
                every_course = []
                every_course.append(course_title[i].text)
                every_course.append(course_score[i].text[0:4])
                every_course.append(commet_num[i].text)
                every_course.append(comment_number[i].text)
                every_course.append(self.__model_type)
                every_course.append(self.__model_title)
                self.saveToCsv(every_course)

    def closeUrl(self):
        self.__browser.close()

結果以下：

3.每一個小課程的詳細信息

頁面以下：

代碼以下：

# encoding: utf-8
from selenium import webdriver
import csv

class GetDetail():
    '''
    類功能介紹：獲取每一個課程的詳細信息
    類成員變量：browser對象，每一個課程的以前已爬取的信息
    類成員函數：初始化函數__init__，獲取每一個課程詳細信息的函數getEveryDetail，寫入csv的函數saveToCsv
    '''
    def __init__(self,title,url,score,mark_people_num,comment_people_num,model_type,model_title):
        self.__browser = webdriver.Chrome()
        self.__page_title = title
        self.__page_url = url
        self.__score    = score
        self.__mark_people_num = mark_people_num
        self.__comment_people_num = comment_people_num
        self.__model_type = model_type
        self.__model_title = model_title

    def saveToCsv(self,this_list):
        with open("course_detail.csv", "a", newline='', encoding='utf-8') as f:
            writer = csv.writer(f, delimiter=',')
            writer.writerow(this_list)

    def getEveryDetail(self):
        self.__browser.get(self.__page_url)
        # 隱式等待
        self.__browser.implicitly_wait(20)
        course_registered_num =  self.__browser.find_element_by_css_selector(".enrolledLargeFont_16g5ucx span strong span").text
        course_provide = self.__browser.find_element_by_css_selector(".partnerBanner_np2ice-o_O-Box_120drhm-o_O-displayflex_poyjc div img").get_attribute("title")
        course_information_view_num = self.__browser.find_element_by_css_selector(".viewsWithTextOnly_1fs65xr span span").text
        course_information    = self.__browser.find_element_by_css_selector(".content-inner").text

        what_we_learned       = self.__browser.find_elements_by_css_selector(".Col_i9j08c-o_O-xsCol12_1m1ceo5-o_O-mdCol6_1rbv01c.m-b-1")
        what_we_learned_result = ''
        for i in range(len(what_we_learned)):
            what_we_learned_result += what_we_learned[i].text
        acquired_skills = self.__browser.find_elements_by_css_selector(".Pill_56iw91.m-r-1s.m-b-1s")
        acquired_skills_result = ''
        for i in range(len(acquired_skills)):
            acquired_skills_result += acquired_skills[i].text
            acquired_skills_result += '  '

        course_evaluation = self.__browser.find_elements_by_css_selector(".quote.caption-text")
        course_evaluation_result = ''
        for i in range(len(course_evaluation)):
            course_evaluation_result += course_evaluation[i].text
            course_evaluation_result += '  '

        all_one = self.__browser.find_elements_by_css_selector(".H4_1k76nzj-o_O-weightBold_uvlhiv-o_O-bold_1byw3y2.m-b-0")
        all_two = self.__browser.find_elements_by_css_selector(".font-sm.text-secondary")
        finish_time = all_one[3].text
        advice_time = all_two[2].text
        course_language = all_one[4].text
        subtitle_language = all_two[3].text

        course_lecturer = self.__browser.find_elements_by_css_selector(".link-no-style")
        course_lecturer_result = ''
        for i in range(len(course_lecturer)):
            course_evaluation += course_lecturer[i].text
            course_lecturer_result += '  '

        every_course_detail = []
        every_course_detail.append(self.__page_title)
        every_course_detail.append(self.__page_url)
        every_course_detail.append(self.__score)
        every_course_detail.append(self.__mark_people_num)
        every_course_detail.append(self.__comment_people_num)
        every_course_detail.append(self.__model_type)
        every_course_detail.append(self.__model_title)
        every_course_detail.append(course_registered_num)
        every_course_detail.append(course_provide)
        every_course_detail.append(course_information_view_num)
        every_course_detail.append(course_information)
        every_course_detail.append(what_we_learned_result)
        every_course_detail.append(acquired_skills_result)
        every_course_detail.append(course_evaluation_result)
        every_course_detail.append(finish_time)
        every_course_detail.append(advice_time)
        every_course_detail.append(course_language)
        every_course_detail.append(subtitle_language)
        self.saveToCsv(every_course_detail)

    def closeUrl(self):
        self.__browser.close()

結果以下：

4.爬取全部評論

  我把全部的小課程爬取下來，並進行了排名，取前六個以下：

  而後對這六個課程進行了全部評論的爬取。代碼以下：

# encoding: utf-8
from selenium import webdriver
import csv
import pandas as pd
import re
import os.path
def saveToCsv(url_title,this_list):
    with open("{0}_review.csv".format(url_title), "a", newline='', encoding='utf-8') as f:
        writer = csv.writer(f, delimiter=',')
        writer.writerow(this_list)

def get_review(url_title,i):
    '''
    :param url_title : 課程標題
    '''
    try:
        url = 'https://www.coursera.org/learn/' + url_title + '/reviews'+ '?page=' + str(i)
        browser = webdriver.Chrome()
        browser.get(url)
        # 隱式等待
        browser.implicitly_wait(20)
        every_view = browser.find_elements_by_css_selector(".Row_nvwp6p.review.review-page-review.m-b-2")
        n = 1
        for i in every_view:
            s = i.find_elements_by_css_selector(".StarRating_1qk9an0-o_O-nonEditable_1ko0lno label")
            # 評星
            evaluation_star = 0
            for j in s:
                if j.text == 'Filled Star':
                    evaluation_star = evaluation_star + 1
                elif j.text == 'Half Faded Star':
                    evaluation_star = evaluation_star + 0.5
            # 做者和時間
            author = i.find_element_by_css_selector(".reviewerName.p-x-1s.m-b-0.text-secondary.font-xs").text[4:]
            time   = i.find_element_by_css_selector(".dateOfReview.p-x-1s.m-b-0.text-secondary.font-xs").text
            # 評論
            reviewTextAll = i.find_elements_by_css_selector(".reviewText div div p")
            ss = []
            for j in reviewTextAll:
                ss.append(j.text)
            reviewText = '@@'.join(ss)
            # 點贊數
            zan = i.find_element_by_css_selector(".Col_i9j08c-o_O-mdCol4_cuxg7k.e2e-helpful-button-col").text
            if zan == '這些頗有用：':
                like_num = 0
            else:
                number = re.findall(r"\d+\.?\d*",zan)
                like_num = int(number[0])
            list_data = []
            list_data.append(evaluation_star);list_data.append(author);list_data.append(time);list_data.append(reviewText);list_data.append(like_num)
            saveToCsv(url_title, list_data)
            n = n + 1
        browser.close()
    except:
        print("出錯，繼續獲取")
        get_review(url_title, i)

def get_result(url_title,page_num):
    # 控制頁數
    for i in range(1,page_num+1):
        try:
            get_review(url_title,i)    # 爬取第i頁的內容
            print("*********課程{course}的第{num}頁爬取完畢".format(course=url_title, num=i))
        except Exception as ex:
            print("*********課程{course}的第{num}頁爬取出錯-{ex}".format(course = url_title,num = i,ex=ex))

def chucuo(k,v):
    file_name = "{0}_review.csv".format(k)
    # 文件標題
    csv_head = ['evaluation_star', 'author', 'time', 'reviewText', 'like_num']
    # 寫入文件標題
    with open(file_name, "a", newline='', encoding='utf-8') as f:
        file_empty = os.stat(file_name).st_size == 0
        writer = csv.writer(f, delimiter=',')
        if file_empty:  # 防止因程序重複運行而寫入重複的標題
            writer.writerow(csv_head)
    get_result(k, v)


def main():
    url_title = {'data-scientists-tools':165,'what-is-datascience':136,'r-programming':124,'python-data-analysis':119,'open-source-tools-for-data-science':60,'python-for-applied-data-science-ai':54}
    for k,v in url_title.items():
        chucuo(k, v)

if __name__=='__main__':
    main()

結果以下：

對比原網頁以下：


注：多段評論用@@隔開，以下：