python網絡爬蟲

獲取http://www.qiushibaike.com/textnew/的全部段子，而且按照頁碼保存到本地一共35頁。
二話不說上代碼，正則表達式有待研究。
網站源碼片斷：

<a href="/users/32215536/" target="_blank" title="吃了兩碗又盛"> <h2>吃了兩碗又盛</h2> </a> <div class="articleGender manIcon">38</div> </div>    <a href="/article/119080581" target="_blank" class='contentHerf' > <div class="content">    <span>一聲長嘆！<br/>二十多年前，簡陋的農村小學的某一個班級裏，有兩個男生坐在同一個長條凳上，這是我和個人同桌。<br/>同桌是冤家，偶爾他回座位，我悄悄地用力，使同桌那一端微微翹起，而後不動聲色地向後旋轉，同桌一下坐空，摔在地上，我偷着樂！<br/>同一招不能總用，可是小時候哪懂這個。那一次，我故技重施，怎料到同桌早有防備，腿向後踢，凳子翻起，我順利地坐到地上。<br/>到如今，就怕陰天！</span>   </div> </a>      <div class="stats"> <span class="stats-vote"><i class="number">62</i> 可笑</span> <span class="stats-comments">     </span> </div> <div id="qiushi_counts_119080581" class="stats-buttons bar clearfix"> <ul class="clearfix"> <li id="vote-up-119080581" class="up"> <a href="javascript:voting(119080581,1)" class="voting" data-article="119080581" id="up-119080581" rel="nofollow"> <i></i> <span class="number hidden">68</span> </a> </li> <li id="vote-dn-119080581" class="down"> <a href="javascript:voting(119080581,-1)" class="voting" data-article="119080581" id="dn-119080581" rel="nofollow"> <i></i> <span class="number hidden">-6</span> </a> </li>  <li class="comments"> <a href="/article/119080581" id="c-119080581" class="qiushi_comments" target="_blank"> <i></i> </a> </li>  </ul> </div> <div class="single-share"> <a class="share-wechat" data-type="wechat" title="分享到微信" rel="nofollow">微信</a> <a class="share-qq" data-type="qq" title="分享到QQ" rel="nofollow">QQ</a> <a class="share-qzone" data-type="qzone" title="分享到QQ空間" rel="nofollow">QQ空間</a> <a class="share-weibo" data-type="weibo" title="分享到微博" rel="nofollow">微博</a> </div> <div class="single-clear"></div>     </div>          <div class="article block untagged mb15" id='qiushi_tag_119080574'>  <div class="author clearfix"> <a href="/users/31546279/" target="_blank" rel="nofollow"> <img src="//pic.qiushibaike.com/system/avtnew/3154/31546279/medium/20160530180038.jpg" alt="？最遠的距離"/> </a> <a href="/users/31546279/" target="_blank" title="？最遠的距離"> <h2>？最遠的距離</h2> </a> <div class="articleGender manIcon">21</div> </div> <a href="/article/119080574" target="_blank" class='contentHerf' > <div class="content"> <span>萬能的糗友啊 苦舞一輩子情字譜<br/>這句話 誰能借下一句。</span> </div> </a> 


python代碼

# coding=utf-8
import urllib2
import re
import os

class Spider(object):


 def __init__(self):
 self.url = 'http://www.qiushibaike.com/textnew/page/%s?s=4832452'
 self.user_agent= 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0) Gecko/20100101 Firefox/53.0'
 def get_page(self,page_index):
 headers= {'User-Agent':self.user_agent}
 try:
 url = self.url % str(page_index)
 request= urllib2.Request(url,headers=headers)
 response= urllib2.urlopen(request)
 content= response.read()
 return content
 #print 'content:' + content
 except urllib2.HTTPError as e:
 print e
 exit()
 except urllib2.URLError as e:
 print 'URLError'+ self.url
 exit()
#分析網頁源碼
 def analysis (self,content):
 #pattern = re.compile('<div class="content">(.*?)<!--(.*?)-->.*?</div>',re.S)
 pattern = re.compile('<div class="content">.*?</div>', re.S)

 #res_value = r'<span .*?>(.*?)</span>'


 items= re.findall(pattern,content)


 return items

 #保存抓取內容
 def save(self,items,i,path):
 if not os.path.exists(path):
 os.makedirs(path)
 file_path = path + '/'+str(i)+'.txt'
 f = open(file_path, 'w')
 for item in items:

 if __name__ == '__main__':
 item_new= item.replace('\n','').replace('<br/>','\n').replace('<div class="content">','').replace('</div>','').replace('</span>','\n').replace('<span>','\n')

 f.write(item_new)
 f.close()
 def run(self):
 print '開始抓取頁面'
 for i in range(1,35):
 content= self.get_page(i)
 items= self.analysis(content)
 self.save(items,i,'/Users/huangxuelian/Downloads/41527218/pythontest')
 print '內容抓取完了'
if __name__ =='__main__': spider = Spider() spider.run()