python 學習之爬蟲練習

經過學習python，寫兩個簡單的爬蟲，沒用線程，本地抓取速度還不錯，有些瑕疵就是抓的圖片有些顯示不出來，代碼作個筆記記錄下：

# -*- coding:utf-8 -*-

import re
import urllib.request
import os

url = "http://www.58pic.com/yuanchuang/0/day-"

def getHtml(url):
    page = urllib.request.urlopen(url)
    html = page.read().decode('gbk')
    return html

def getImg(html,num):
    reg = r'src="(.*?)" '
    imgre = re.compile(reg)
    imglist = re.findall(imgre,html)
    x = 0
    os.mkdir(r"G:\collect/%d" % num)
    filePath = r"G:\collect/%d/" % num
    for imgurl in imglist:       
        f=open(filePath+str(x)+".jpg",'wb')  
        req=urllib.request.urlopen(imgurl)
        buf=req.read()  
        f.write(buf)
        x+=1

for i in range(1,10):
    getUrl = url+"%d.html" % i
    print(getUrl)
    html = getHtml(getUrl)
    #print(html)
    print(getImg(html,i))

最終的結果以下圖：

根據上面的初步代碼，優化後增強版的爬蟲代碼，對於連接的狀態異常的拋出異常後在繼續執行程序。代碼以下：

# -*- coding:utf-8 -*-

import re
import urllib.request
import os

url = "http://www.58pic.com/psd/"

def getHtml(url):
    page = urllib.request.urlopen(url)
    html = page.read().decode('gbk')
    return html

def getImg(html,num):
    reg = r'src="(.+?\.jpg)" class="show-area-pic" id="show-area-pic" alt="(.*?)"'
    imgre = re.compile(reg)
    imglist = re.findall(imgre,html)
    print(imglist)
    filePath = r"F:\Py/collect/%d/" % num
    isCreate = os.path.exists(filePath)
    if isCreate == False :
        os.mkdir(r"F:\Py/collect/%d" % num)   
        for img in imglist:
            title = img[1]
            f=open(filePath+title+".jpg",'wb') 
            req=urllib.request.urlopen(img[0])
            buf=req.read()  
            f.write(buf)
            

for i in range(22797263,22797666):
    getUrl = url+"%d.html" % i
    #status = urllib.request.urlopen(getUrl).code
    try:
        html = getHtml(getUrl)
        #print(html)
        getImg(html,i)
    except urllib.request.URLError as e:
        print(e.code)
        print(e.reason)