爬蟲基礎練習一 爬取豆瓣電影TOP250

這是一個很好的新手練習項目，爬取豆瓣top250的電影，並分別記錄排名、片名、導演、主演、評論等信息，保存在一個txt文檔裏。
對新手來講，難點部分在於如何找到併成功跳轉到下一頁，而且在最後一頁的時候識別出來並中止爬蟲。

一個很基礎的爬蟲。如下是代碼部分。

import requests
from bs4 import BeautifulSoup
import time
import re

lurl = 'https://movie.douban.com/top250'

movie = []

def getlist(listurl):
    time.sleep(2)
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'}
    res = requests.get(listurl)
    soup = BeautifulSoup(res.text, 'html.parser')
    movielist = soup.select('.grid_view li')
    for m in movielist:
        rank = m.select('em')[0].text
        title = m.select('.title')[0].text
        direct = m.select('.info .bd p')[0].text.strip()
        actor = '\n主演:'.join(direct.split('   主演:'))
        director = '年代:'.join(actor.split('                           '))
        if m.select('.inq'):
            comments = m.select('.inq')[0].text.strip()
        else:
            comments = 'None'
        movie.append('排名: '+ rank+ '\n' +'片名: '+ title + '\n'+ director + '\n' + '評論: '+ comments +'\n' + '\n')
    if soup.select('.next a'):
        asoup = soup.select('.next a')[0]['href']
        Next_page = lurl + asoup
        getlist(Next_page)
    else:
        print('結束')
    return movie



movies = getlist(lurl)

with open('movie.txt', 'w') as m:
    for a in movies:
        m.write(a)