簡單的python爬蟲 爬的烏雲漏洞類型

import urllib.request
import re

starturl="http://wooyun.org/searchbug.php?q=%E9%87%91%E8%9E%8D"

def get_html_response(url):
    html_response = urllib.request.urlopen(url).read().decode('utf-8')
    return html_response

def geturl(starturl):
    a=get_html_response(starturl)
    childurl=(re.findall(r'/bugs/wooyun-\w*-\w*\b',a))
    return childurl

def get_nextpage(starturl):
    d=get_html_response(starturl)
    nextpage=(re.findall(r'searchbug.php\?q=金融&pNO=\w',d))
    return nextpage

result=[]
result2=[]

for i in get_nextpage(starturl):
    result+=geturl('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))
    #掃描各類漏洞的url地址放入result中
result=set(result)#去除result中重複的地址
#print(len(result))
#測試結果爲87，正確

#信息存儲+文件處理
output=open("D:\\a.txt",'w+')

for i in result:
    k=get_html_response('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))#下載頁面到k，能夠把
    info=re.findall(r'<h3>\w*：\s*\w*\s*</h3>',k)#空白字符用/s,尋找全部適用於<h3>標籤的文字
    for j in info:
        j=re.sub(r'：\s',':',j)
        j=re.sub(r'<h3>','',j)
        j=re.sub(r'</h3>',',',j)
        j=j.split()
        output.writelines(j)
        output.writelines("\n")
output.close()

mentor要求的，簡略分析烏雲關於金融方面的漏洞並歸類

人生第一個能跑的有意義的程序，好開心

 

2014/8/14根據導師要求作了修改

import urllib.request
import re

starturl="http://www.wooyun.org/searchbug.php?q=6YeR6J6N"

def get_html_response(url):
    html_response = urllib.request.urlopen(url).read().decode('utf-8')
    return html_response

def geturl(starturl):
    a=get_html_response(starturl)
    childurl=(re.findall(r'/bugs/wooyun-\w*-\w*\b',a))
    return childurl

def get_nextpage(starturl):
    d=get_html_response(starturl)
    nextpage=(re.findall(r'searchbug.php\?q=6YeR6J6N&pNO=\w',d))
    return nextpage

result=[]for i in get_nextpage(starturl):
    result+=geturl('http://wooyun.org/'+re.sub('金融','6YeR6J6N',i))
    #掃描各類漏洞的url地址放入result中
result=set(result)#去除result中重複的地址
#print(len(result))
#測試結果爲87，正確

#信息存儲+文件處理
output=open("D:\\wooyun.txt",'w+')

for i in result:
    k=get_html_response('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))#下載頁面到k
    info=re.findall(r'<h3>\w*：\s*\w*\s*</h3>',k)#空白字符用/s,尋找全部適用於<h3>標籤的文字
    detail=re.findall(r'<p class="detail">.*.</p>',k)
    for j in info:
        j=re.sub(r'：\s',':',j)
        j=re.sub(r'<h3>','',j)
        j=re.sub(r'</h3>','',j)
        j=j.split()
        output.writelines(j)
        output.writelines("\n")
    for j in detail:
        j=re.sub(r'：\s',':',j)
        j=re.sub(r'<p\sclass="detail">','',j)
        j=re.sub(r'</p>','',j)
        j=re.sub(r'"\starget="_blank"><img\ssrc="/upload/.*.width="600"/></a>','',j)
        j=re.sub(r'<a href="',' http://www.wooyun.org',j)
        j=j.split()
        output.writelines(j)
        output.writelines("\n")
    output.writelines("\n")
output.close()