檢測 web項目 404 500 狀態的 頁面

用於發版前自動化測試

用法

一、使用參數 -f 指定url配置文件
二、url文件簡單配置, 每行一條URL 下面三種格式均可以，若是不聲明 GET、POST 默認爲GET請求
https://www.wmzy.com/api/rank/schList?sch_rank_type=QS2018
get https://www.wmzy.com/api/rank/schList
post https://www.wmzy.com/account/saveEduInfo

調用命令: ./testPage.py -f ./urls.txt

三、經過 list 配置帶有參數和信息頭的的URL
配置文件是嚴格的 python 字典類型數據，包含 option 和 urls 兩個 key
option:字典類型, 能夠配置 method,headers,data, 會被list 內 url的配置覆蓋
urls: list 類型, 每一個元素配置一條url，格式以下

簡單的get請求能夠是url字符串
帶有額外配置信息的URL使用字典配置
url: url地址
method:缺省認爲GET
headers：字段類型，請求頭信息
data:數據，缺省默認爲 None

調用命令: ./testPage.py -f ./urls2.txt

四、使用參數 -out 輸出測試報告文件
調用命令: ./testPage.py -f ./urls2.txt -out ./report.txt

五、經過參數 -print 100 指定:請求測試經過時打印請求返回內容的前100個字符
六、經過參數 -headers 指定:請求測試經過時打印請求返回的頭信息
調用命令: ./testPage2.py -f ./ajaxTestUrls.txt -out ./report.txt -print 200

 

腳本 testPage.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
# author ecalf

import sys
import urllib2
import urllib
from urllib2 import URLError, HTTPError 
import zlib
import codecs




def showMsg( msg='', isWarn=False):
    if isWarn:
        print '\033[1;91m '+msg+' \033[0m' 
    else:
        print '\033[1;0m '+msg+' \033[0m'    

def printReport(text='',reportFile=None):
    print text
    if reportFile and isinstance(reportFile,file):
        print >> reportFile, text

def getCmdParams(params):
    fileIn = ''
    fileOut = ''
    outPutConfig = {
        'printBodyLength':0,
        'printHeaders':False
    }
    


    if len(params)==1:
        fileIn = params[0]
        
    else:
        for i,param in enumerate(params):
            if param.lower() == '-f':
                if i+1>=len(params):
                    showMsg('請指定url配置文件',True)
                else:
                    fileIn = params[i+1]
            elif param.lower() == '-out':
                if i+1>=len(params):
                    showMsg('請指定測試報告輸出文件路勁',True)
                else:
                    fileOut = params[i+1]
            elif param.lower() == '-print':
                if i+1>=len(params):
                    showMsg('請指定打印數據的長度',True)
                else:
                    try:
                        outPutConfig['printBodyLength'] = int(params[i+1])
                    except Exception,err:
                        showMsg('指定打印請求返回的數據長度應該輸入數字',True)
            elif param.lower()=='-headers':
                outPutConfig['printHeaders'] = True

    return fileIn,fileOut,outPutConfig



def getUrlList(fileIn):
    try:
        fileObj = open(fileIn,'r')
        fileContent = fileObj.read()
    except Exception,err:
        showMsg('沒法讀取文件:'+fileIn,True)
    finally:
        if fileObj:
            fileObj.close()


    if fileContent[0:3]==codecs.BOM_UTF8:
        fileContent = fileContent[3:]
        
    dataType = 'string'
    try:
        urlsConfig = eval(fileContent)
        if isinstance(urlsConfig,dict):
            dataType = 'list'

    except Exception,err:
        #showMsg('parse fileContent err>>>'+str(err),True)
        pass


    urls = []
    option = None
    try:
        if dataType=='string':
            lines = fileContent.split('\n')
            for url in lines:
                if url.replace(' ', ''):
                    urls.append(url)

        elif dataType.lower()=='list':
            option = urlsConfig['option']
            urls = urlsConfig['urls']

    except Exception,err:
        showMsg('文件讀取錯誤，請使用UTF8編碼保存URL配置文件',True)

    return urls,option



def getCommonHeaders():
    headers = {
        'Connection':'keep-alive',  
        'Cache-Control':'max-age=0',  
        'Accept': 'text/html,application/xhtml+xml,application/xml,application/json;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',  
        'Pragma': 'no-cache',
        'Cache-Control': 'no-cache',
        'Accept-Encoding': 'gzip, deflate',  
        'Accept-Language': 'zh-CN,zh;q=0.9'  
    }

    return headers

def setHost(request,host):
    request.add_header('Host', host)

def setCookie(request,cookieStr):
    request.add_header('Cookie', cookieStr)

def initConfig(urlConfig,option):
    method = 'GET'
    headers = getCommonHeaders()
    data = {}

    #from option
    if isinstance(option,dict):
        if 'method' in option:
            method = option['method']

        if 'headers' in option and isinstance(option['headers'],dict):
            headers.update(option['headers'])

        if 'data' in option and isinstance(option['data'],dict):
            data.update(option['data'])

    #from urlConfig
    if 'method' in urlConfig:
            method = urlConfig['method']

    if 'headers' in urlConfig and isinstance(urlConfig['headers'],dict):
        headers.update(urlConfig['headers'])
    
    if 'data' in urlConfig and isinstance(urlConfig['data'],dict):
        data.update(urlConfig['data'])

    if len(data.keys())==0:
        data = None

    return  urlConfig['url'],method,headers,data


def initRequest(url,data=None,method='GET',headers=None):
    url =  urllib.quote(url.strip(),safe="$#&+;/:,=?@")
    if data:
        data = urllib.urlencode(data)

    if method.upper()=='GET':
        if data:
            searchStartIndex = url.find('?')
            if searchStartIndex>-1:
                url = url[0:searchStartIndex]+'?'+data+'&'+url[searchStartIndex+1:]
            else:
                anchorIndex = url.find('#')
                if anchorIndex>-1:
                    url = url[0:anchorIndex]+'?'+data+url[anchorIndex:]
                else:
                    url = url+'?'+data
        request = urllib2.Request(url,None,headers) 
    else:
        if not data:
            data = ''
        request = urllib2.Request(url,data,headers)

    return     request,url


def startTest(urls,option,fileOut,outPutConfig):
    if len(urls)==0:
        showMsg('url list is empty',True)
        return

    reportFile = ''
    if fileOut:
        try:
            reportFile = open(fileOut,'w')
        except Exception,err:
            showMsg('建立報告文件失敗:'+str(err),True)



    counter = {
        'count200':[],
        'count401':[],
        'count404':[],
        'count500':[],
        'count502':[],
        'countURLError':[],
        'countException':[],
    }

    
    for i,urlConfig in enumerate(urls):
        i=i+1

        try:
            if isinstance(urlConfig,basestring):
                url = urlConfig
                urlConfig = {}
            else:
                url = urlConfig['url']


            method = 'GET'
            if 'method' in urlConfig:
                method = urlConfig['method']

            if url[:4].upper()=='POST':
                method = 'POST'
                url = url[4:]
            elif url[:3].upper()=='GET':
                method = 'GET'
                url = url[3:]

            urlConfig['method'] = method
            urlConfig['url'] = url.strip()

            url,method,headers,data = initConfig(urlConfig,option)
            request,url = initRequest(url,data,method,headers)
            response = urllib2.urlopen(request)
            


            body = response.read()
            gzipped = response.headers.get('Content-Encoding')
            if gzipped:
                    body =zlib.decompress(body, 16+zlib.MAX_WBITS)
                    #print 'zlib.decompress body'

            if outPutConfig['printHeaders']==True or outPutConfig['printBodyLength']>0:
                printReport('----------------------------------------------------------------',reportFile)
                

            if     outPutConfig['printHeaders']==True:
                printReport(response.info(),reportFile)

            if outPutConfig['printBodyLength']>0:
                printReport(body[:outPutConfig['printBodyLength']],reportFile)


            statusCode = response.getcode()
            countKey = 'count'+str(statusCode)
            if countKey not in counter:
                counter[countKey] = []

            counter[countKey].append(i)
            msg = '['+str(i)+']'+'\t'+str(statusCode)+'\t\t'+method.upper()+' '+url

            if response.geturl()!=url:
                msg = msg+' redirect =>'+response.geturl()

            showMsg(msg)
            if reportFile and isinstance(reportFile,file):
                reportFile.write(msg+'\n')

        except HTTPError, err: 
            #print err,err.code,err.reason
            stateText = err.reason
            statusCode = err.code

            countKey = 'count'+str(statusCode)
            if countKey not in counter:
                counter[countKey] = []

            counter[countKey].append(i)
                
            #print err.reason
            msg = '['+str(i)+']'+'\t'+str(statusCode)+'\t\t'+method.upper()+' '+url
            showMsg(msg,True)
            if reportFile and isinstance(reportFile,file):
                reportFile.write(msg+'\n')
        except URLError, err: 
            counter['countURLError'].append(i)
            
            msg = '['+str(i)+']'+'\tURLError'+'\t'+method.upper()+' '+url
            showMsg(msg,True)
            if reportFile and isinstance(reportFile,file):
                reportFile.write(msg+'\n')

        except Exception,err:
            counter['countException'].append(i)

            msg = '['+str(i)+']'+'\tException:'+str(err)+'\t'+method.upper()+' '+url
            showMsg(msg,True)
            if reportFile and isinstance(reportFile,file):
                reportFile.write(msg+'\n')
                

        

    reportMsg = []
    reportMsg = reportMsg+['\n\n------------ test report,  Total:',str(i),' --------------','\n']
    reportMsg = reportMsg+['status   ','\t','count','\t','index','\n']

    countKeys = counter.keys()
    countKeys.sort()
    for i,key in enumerate(countKeys):
        statusColumn = key[len('count'):]+'         '
        statusColumn = statusColumn[:9]
        countColumn = str(len(counter[key]))
        if len(countColumn)<5:
            countColumn = countColumn+'     '
            countColumn = countColumn[:5]



        reportMsg = reportMsg+[statusColumn,'\t',countColumn,'\t',str(counter[key]),'\n']


    # reportMsg = reportMsg+['ok','\t',str(len(counter['countOk'])),'\t',str(counter['countOk']),'\n']
    # reportMsg = reportMsg+['40*','\t',str(len(count400)),'\t',str(count400),'\n']
    # reportMsg = reportMsg+['50*','\t',str(len(count500)),'\t',str(count500),'\n']
    # reportMsg = reportMsg+['err','\t',str(len(countErr)),'\t',str(countErr),'\n']
    # reportMsg = reportMsg+['??','\t',str(len(Exception)),'\t',str(Exception),'\n']
    reportMsg = reportMsg+['---------------------------------------------------','\n']
    reportMsg = reportMsg+['\n\n']
    reportMsg = ''.join(reportMsg)

    printReport(reportMsg,reportFile)
    if reportFile and isinstance(reportFile,file):
        reportFile.close()



params = sys.argv[1:]
fileIn,fileOut,outPutConfig = getCmdParams(params)
urls,option = getUrlList(fileIn)
startTest(urls,option,fileOut,outPutConfig)









    

 

 

URL配置文件（簡單）

post http://www.wmzy.com/account/saveEduInfo
http://www.wmzy.com/api/school/getSchList?prov_filter=44&type_filter=0&diploma_filter=0&flag_filter=0&page=2&page_len=20&_=1529798214101
http://www.wmzy.com/api/school/getSchList?prov_filter=shenzhen&type_filter=0&diploma_filter=0&flag_filter=0&page=2&page_len=20&_=1529798214101
http://www.wmzy.com/api/school/3bzwryxx.html
http://www.wmzy.com/static/outer/js/aq_auth.js
http://cloud.tenxasdasdcent.com/deasdasdasdvelopera/aasdasdsk/asd42279/answer/63957
http://node-img.b0.upaiyun.com/gaokao/tvpPlayer.html?vid=z0692bc6wkb&auto=1&title=AI時代·如何完美定製你的志願填報
absdad

 

URL配置文件（複雜）

{
    'option':{
        'headers':{
            'Cache-Control':'no-cache',
            'Cookie':'sessionid=s:7rn8WESdKKXcukeo03wQjTlw.QB7A4baInGRgBbS0BgqGf1Rz5TSwLlRc8MRyE6roeJ8;path=/'
        }
    },
    'urls':[
        'http://www.wmzy.com//index.html',
        {
            'method':'POST',
            'url':"http://www.wmzy.com/account/getEduInfoUIConfig"
        },
        {
            'method':'POST',
            'url':"http://www.wmzy.com/account/saveEduInfo",
            'data':{
                'province_id': '320000000000',
                'city_id': '321200000000',
                'region_id': '321283000000',
                'school_id': '5a9f64a8b97bd266633f28f9',
                'enroll_year': 2017,
                'enroll_type': 1
            }
            
        },

        {
            'method':'POST',
            'url':'http://www.wmzy.com/account/ajaxLogin',
            "headers":{
                'X-Requested-With':'XMLHttpRequest'
            },
            'data':{
                'account':'13712313131',
                'password':'123456',
                'forceBindeCard':'false'
            }
        },
        {
            'method':'POST',
            'url':'http://www.wmzy.com/zhiyuan/score',
            "headers":{
                'X-Requested-With':'XMLHttpRequest'
            },
            'data':{
                'prov':44,
                'realScore':500,
                'ty':'l',
                'diploma_id':7,
                'score_form':'scoreBox'
            }
        }
    ]
}