python 爬蟲實列

時間  2019-12-01
標籤 python 爬蟲 欄目 Python 简体版
原文   原文鏈接
In [5]: 
import urllib.request
In [6]: 
#建立一個ruquest對象
url="https://tieba.baidu.com/p/6310762577"
request=urllib.request.Request(url)
#鏈接url，返回response對象
response=urllib.request.urlopen(request)
#獲取內容數據
html=response.read()#read(方法)
#設置內容爲utf-8編碼
html=html.decode("utf-8")
html

Out[6]:javascript

'\n<!DOCTYPE html><!--STATUS OK--><html><head><meta name="keywords" content="百度貼吧,美圖騎單,
車的"/><meta name="description" content="騎單車的人..騎單車的人" /><meta charset="UTF-8"><meta 
furl="tieba.baidu.com/f?kw=%E7%BE%8E%E5%9B%BE&ie=utf-8" fname="美圖"><meta http-equiv="X-UA-
Compatible" content="IE=edge,chrome=1"><meta name="baidu-site-verification" content="jpBCrwX689" 
/><link rel="search" type="application/opensearchdescription+xml" href="/tb/cms/content-search.xml" 
title="百度貼吧" /><title>騎單車的人_美圖吧_百度貼吧</title><script type="text/javascript">void function(t,e,n,a,o,i,r)
{t.alogObjectName=o,t[o]=t[o]||function(){(t[o].q=t[o].q||[]).push(arguments)},t[o].l=t[o].l||+new Date,a="https:
"===t.location.protocol?"https://fex.bdstatic.com"+a:"http://fex.bdstatic.com"+a;var c=!0;if(t.alogObjectConfig&&t.
alogObjectConfig.sample){var s=Math.random();t.alogObjectConfig.rand=s,s>t.alogObjectConfig.sample&&(c=!1)}c&&(i=e.
createElement(n),i.async=!0,i.src=a+"?v="+~(new Date/864e5)+~(new Date/864e5),r=e.getElementsByTagName(n)[0],r.parentNode.
insertBefore(i,r))}(window,document,"script","/hunter/alog/alog.min.js","alog"),void function(){function t(){}window.PDC=
{mark:function(t,e){alog("speed.set",t,e||+new Date),alog.fire&&alog.fire("mark")},init:function(t){alog("speed.set","options",t)}

用正則表達式解析內容

In [7]: 
#導入正則表達式包
import re
In [8]: 
#建立一個正則對象
str='src="(.+?.jpg)" size'
imger=re.compile(str)
imglist=re.findall(imger,html)#找打全部圖片
imglist
Out[8]: 
['https://imgsa.baidu.com/forum/w%3D580/sign=4b966626e4c4b7453494b71efffd1e78/aeea0b55b319ebc4f68696968d26cffc1f17161a.jpg',
 'https://imgsa.baidu.com/forum/w%3D580/sign=4172a3f9a8efce1bea2bc8c29f50f3e8/30ca91ef76c6a7ef1f864829f2faaf51f2de667d.jpg',
 'https://imgsa.baidu.com/forum/w%3D580/sign=f1a090968d26cffc692abfba89004a7d/3c1fb64543a982268919cd288582b9014890ebc5.jpg']
In [9]: 
#把imglist保存到目錄下
import  os
import time
i=1
#if not os.path.exists("imagess"):
    #os.mkdir("imagrss")#系統裏面不存在」images「就建立一個
for img in imglist:
    time.sleep(1)#休息1秒抓取下一張
    urllib.request.urlretrieve(img,"C:/Users/1/Desktop/{}.jpg".format(i))
    i=i+1
print("爬蟲結束")
爬蟲結束  只上傳一張0.0
 
         
 
        
 
       
 
      
 
     
 
     
 
      

          
      
 
      
 
       
 
        
get方法抓取
 
       
 
      
 
     
 
     
 
      
 
       

         In [10]: 
       
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
import urllib.request
 
              
 
             
 
            
 
           
 
           

             In [25]: 
           
 
          
 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
#在淘寶轉化瀏覽設備爲6plus，F12在JS中找到內容
url="https://suggest.taobao.com/sug?q=python+%E7%88%AC%E8%99%AB&code=utf-8&area=c2c&nick=&sid=null&callback=jsonp157311230486278746"
#鏈接url，返回response對象
response=urllib.request.urlopen(url)
html=response.read().decode("utf8")
html
 
              
 
             
 
            
 
           
 
           

             Out[25]: 
           
 
          
 
         
 
        
 
       
 
      
 
      
 
       
 
        
 
         
 
          
'\r\njsonp157311230486278746({"result":[["python爬蟲<b>書籍<\\/b>","690.2666666666667"],["python<b>3網絡<\\/b>爬蟲<b>開發實戰<\\/b>","396.02"],["python<b>3網絡<\\/b>爬蟲<b>開發<\\/b>","429.95348837209303"],["python爬蟲<b>實戰<\\/b>","1046.909090909091"],["python爬蟲<b>課程網課<\\/b>","107.2"],["python<b>3網絡<\\/b>爬蟲","473.5"],["python爬蟲<b>源代碼<\\/b>","110.9375"],["python爬蟲<b>項目<\\/b>","1116.8333333333333"],["python<b>網絡<\\/b>爬蟲<b>實戰<\\/b>","658.6"],["python爬蟲<b>入門<\\/b>","1084.969696969697"]]})'
 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       

         In [42]: 
       
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
#把json的字符串格式轉換成python的字典類型
 
              
 
              
 
               
 
                
 
                 
 
                  
import json
dic=json.loads(a)
dic
#注json不支持單引號，包含單引號的字符串會解析失敗
 
                 
 
                 
Out[42]:html
 
                
 
               
 
              
 
             
 
            
 
           
 
          
 
         
 
        
 
       
 
      
 
      
 
       
 
        
 
         
 
          
{'result': [['python爬蟲<b>書籍</b>', '690.2666666666667'],
  ['python<b>3網絡</b>爬蟲<b>開發實戰</b>', '396.02'],
  ['python<b>3網絡</b>爬蟲<b>開發</b>', '429.95348837209303'],
  ['python爬蟲<b>實戰</b>', '1046.909090909091'],
  ['python爬蟲<b>課程網課</b>', '107.2'],
  ['python<b>3網絡</b>爬蟲', '473.5'],
  ['python爬蟲<b>源代碼</b>', '110.9375'],
  ['python爬蟲<b>項目</b>', '1116.8333333333333'],
  ['python<b>網絡</b>爬蟲<b>實戰</b>', '658.6'],
  ['python爬蟲<b>入門</b>', '1084.969696969697']]}
 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       

         In [41]: 
       
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
a=html[26:452]
 
              
 
             
 
            
 
           
 
           

             In [54]: 
           
 
          
 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
for item in dic["result"]:
    print(item[0].replace("<b>","").replace("</b>",""))#replace字符串替換函數
 
              
 
             
 
            
 
           
 
          
 
         
 
        
 
       
 
      
 
      
 
       
 
        
 
         
 
          
python爬蟲書籍
python3網絡爬蟲開發實戰
python3網絡爬蟲開發
python爬蟲實戰
python爬蟲課程網課
python3網絡爬蟲
python爬蟲源代碼
python爬蟲項目
python網絡爬蟲實戰
python爬蟲入門

 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       

         In [45]: 
       
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
#python 自帶工具能夠作編碼‘
str=urllib.request.quote("牛仔衣")
str
 
              
 
             
 
            
 
           
 
          
 
         
 
        
 
       
 
      
 
      
 
       
 
        
 
         

           Out[45]: 
         
 
         
 
          
'%E7%89%9B%E4%BB%94%E8%A1%A3'
 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       

         In [56]: 
       
 
       
 
        
 
         
import requests
import json
keyword = '連衣裙'
# 不須要進行URL的編碼
url = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(keyword)
response = requests.get(url)
html = response.text # 獲取響應體的內容
dic = json.loads(html)
for item in dic['result']:
    print(item[0])
 
        
 
        
連衣裙女秋冬
連衣裙2019新款秋
連衣裙女
連衣裙2019秋款新
連衣裙夏
連衣裙女春秋
連衣裙長款秋冬
連衣裙收腰顯瘦 氣質
連衣裙兩件套秋冬
連衣裙長裙女秋冬
 
        
 
         
#搜索連衣裙二級菜單
import requests
import json
#定義一個函數，替換返回值中的特殊格式
def replace_str(html):
    html = html.replace('<b>','')
    html = html.replace('<\/b>','')
    return html

keyword = '連衣裙'
url = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(keyword)
response = requests.get(url)
dic = json.loads(response.text)
for item in dic['result']:
    print(item[0])
    url2 = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(item[0])
    response2 = requests.get(url2)
    content = replace_str(response2.text)
    dic2 = json.loads(content)
    for item2 in dic2['result']:
        print('\t'+item2[0])
 
        
 
        
連衣裙2018款新款
	連衣裙2018款新款女
	連衣裙2018款新款女 雪紡
	連衣裙2018夏新款女 中長款 修身
	裙子女夏2018新款 中長款連衣裙
	裙子夏女2018新款 款連衣裙 氣質
	連衣裙夏女2018新款 中長款 氣質
	女裙子2018新款 中長款連衣裙
	女夏2018新款連衣裙長  款35歲
	女裝2018新款中長連衣裙 中長款
	連衣裙女夏2018新款拉鍊款
連衣裙夏季新款
	連衣裙夏季新款女 2018 氣質 a字
	連衣裙夏季新款 雪紡 2018
	連衣裙夏季新款 寬鬆 韓版 2018
	連衣裙夏季新款女  名媛
	連衣裙夏季新款 2018韓版孕婦裝
	連衣裙夏季新款2018顯瘦超仙
	連衣裙夏季新款女裝歐美中裙
	連衣裙夏季新款女歐洲站2018
	連衣裙夏季新款女韓版 棉麻
	連衣裙夏季新款 時尚氣質
 
       
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
反爬蟲機制java
 
              
 
 
              
加入user-agent，冒充瀏覽器 user-agent可在瀏覽器中F12 找到，且必須以字典格式傳入
 
              
 
               
#加入user-agent，冒充瀏覽器 user-agent可在瀏覽器中F12 找到，且必須以字典格式傳入
header={"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"}
keyword = '羽絨服'
url = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(keyword)
response = requests.get(url,headers=header)
dic = json.loads(response.text)
for item in dic['result']:
    print(item[0])
 
              
 
              
羽絨服女中長款 修身顯瘦
羽絨服女輕薄款
羽絨服中長款女
羽絨服女長款2017新款 韓版 潮
羽絨服 棗紅色
羽絨服女收腰
羽絨服男輕薄
羽絨服女長款 冬季 中長款
羽絨服套裝
羽絨服女長款收腰過膝
 
              
設置代理IP
 
              
 
               
#設置代理IP
proxies = {"HTTP":"122.114.31.177:808"} #百度搜索代理IP
header={"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"}
keyword = '西裝'
url = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(keyword)
response = requests.get(url,headers=header,proxies=proxies)
dic = json.loads(response.text)
for item in dic['result']:
    print(item[0])
 
              
 
              
西裝男套裝 青少年
西裝短褲套裝女
西裝 修身連衣裙
西裝領長袖女
西裝 短袖
西裝v領 連衣裙 收腰
西裝喇叭褲高腰
西裝褲 夏 女
西裝春秋女
西裝熱褲女
 
              
 
               
# 把數據寫到文件中
#搜索連衣裙二級菜單
import requests
import json
import pandas as pd
#定義一個函數，替換返回值中的特殊格式
def replace_str(html):
    html = html.replace('<b>','')
    html = html.replace('<\/b>','')
    return html

keyword = '連衣裙'
proxies = {"HTTP":"122.114.31.177:808"}
header={"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"}
url = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(keyword)
response = requests.get(url,headers=header,proxies=proxies)
dic = json.loads(response.text)
lst=[]
for item in dic['result']:
    lst.append([item[0],1])
    url2 = "https://suggest.taobao.com/sug?q={}&code=utf-8&area=c2c&nick=&sid=null".format(item[0])
    response2 = requests.get(url2,headers=header,proxies=proxies)
    content = replace_str(response2.text)
    dic2 = json.loads(content)
    for item2 in dic2['result']:
        lst.append([item2[0],2])
#寫到文件中
data = pd.DataFrame(lst,columns=['title','level'])
data.to_csv('./lyq.csv',index=False,header=True)
print('運行結束')
 
              
 
              
 
               
# 寫一個爬蟲—Post 方式抓取有道翻譯數據
import requests
import json

def translate(word):
    url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule"
    header={
    "Accept": "application/json, text/javascript, */*; q=0.01",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection":"keep-alive",
    "Content-Length": "254",
    "Content-Type": "application/x-www-form-urlencoded;",
    "Cookie": "OUTFOX_SEARCH_USER_ID=-2118096325@10.168.8.63; OUTFOX_SEARCH_USER_ID_NCOO=529585232.72911924; fanyi-ad-id=44547; fanyi-ad-closed=1; DICT_UGC=be3af0da19b5c5e6aa4e17bd8d90b28a|; JSESSIONID=abcm1o-ND2W6Y9HCq9vqw; _ntes_nnid=a7d939a02cd5c3865942d2a2051b410e,1529376772962; ___rl__test__cookies=1529398225385",
    "Host": "fanyi.youdao.com",
    "Origin": "http://fanyi.youdao.com",
    "Referer": "http://fanyi.youdao.com/",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36",
    "X-Requested-With":"XMLHttpRequest"
    }
    payload = {
    "i": word,
    "from": "AUTO",
    "to": "AUTO",
    "smartresult": "dict",
    "client": "fanyideskweb",
    "salt": "1529398225392",
    "sign": "bf09bc9795dfc7863516162c961fd97e",
    "doctype": "json",
    "version": "2.1",
    "keyfrom": "fanyi.web",
    "action": "FY_BY_CLICKBUTTION",
    "typoResult":"false"
    }
    response = requests.post(url,data=payload,headers=header)
    dic = json.loads(response.text)
    print(dic['translateResult'][0][0]['tgt'])
if __name__=='__main__':
    translate('中國')
 
              
 
              
China
 
              
 
 
             
 
            
 
           
 
          
 
         
 
        
 
       
 
      
 
     
 
     
 
      
 
       

           
       
 
       
 
        
 
         
 
          
 
           
 
            
 
             
 
              
 
               
 
                
 
                 
​
 
                
 
               
 
              
 
             
 
            
 
           
   
          
 
         
 
        
 
       
 
      
  
     
 
    
 
   
 
  
 
 
 



        

            

        	





    

		

            
相關文章

            

                

                

                    

                

            


            
相關標籤/搜索

            
        


    

        

            

        

        

        
        

        

        

    




	

    





    

    	

    

        每日一句
    

    
    
    	每一个你不满意的现在，都有一个你没有努力的曾经。
    







    

    


    



    

        本站公眾號
    

    

           歡迎關注本站公眾號,獲取更多信息

        
    




    

    




	


	







    

        聯繫我們
        最近搜索
        最新文章
        
        沪ICP备13005482号-10

            MyBatis教程
            SQL 教程
            MySQL教程
            Java 教程
            Thymeleaf 教程
            Hibernate教程
            Spring教程 
            Redis教程