使用python的requests模塊採集請求中的數據

 1 # coding=gbk
 2 
 3 # Create your tests here.
 4 
 5 import requests, re, json, math
 6 
 7 # 設置全局session變量
 8 r_session = requests.session()
 9 
10 #登陸帳戶
11 username = 'admin'
12 
13 #登陸密碼
14 password = '111111'
15 
16 #登陸ip
17 ip = 'localhost'
18 
19 #登陸port
20 port = '8088'
21 
22 #模型總數
23 num = 1679
24 
25 #調用登陸接口，向session中插入登陸狀態
26 def login():
27     login_headers = {
28         'Accept': '*/*',
29         'Accept-Encoding': 'gzip, deflate',
30         'Accept-Language': 'zh-CN,zh;q=0.9',
31         'Connection': 'keep-alive',
32         'Content-Length': '40',
33         'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
34         'Host': '{ip}:{port}'.format(ip=ip, port=port),
35         'Origin': 'http://{ip}:{port}'.format(ip=ip, port=port),
36         'Referer': 'http://{ip}:{port}/login'.format(ip=ip, port=port),
37         'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',
38         'X-Requested-With': 'XMLHttpRequest'
39     }#設置登陸請求頭
40     login_data = {"username": username, "password": password}#設置登陸請求體
41     r = r_session.post('http://{ip}:{port}/login'.format(ip=ip, port=port), headers=login_headers, data=login_data)#發送登陸請求
42 
43 #獲取系統中的全部模型id
44 def get_model_id(num):
45     modelnum = math.ceil(num/100)
46     model_id = []#用於存儲全部的模型id
47     model_headers = {'Content-Type': 'application/json'}
48     for x in range(modelnum):
49         x *= 100
50         r = r_session.get(
51             "http://{}:{}/model/list?limit=100&offset={}&sortName=updatetime&sortOrder=desc&name=&id=&status=".format(ip, port,
52                 x), headers=model_headers).text
53         r = json.loads(r)
54         rows = r.get("rows")
55         for i in rows:
56             model_id.append(i.get("id"))
57     return model_id
58 
59 #獲取全部的錯誤信息
60 def get_error(model_id):
61     repeat_id = []#用於存儲全部的錯誤信息
62     for model_id_detail in model_id:
63         model_detail = r_session.get('http://{ip}:{port}/model/modify/'.format(ip=ip, port=port) + str(model_id_detail)).text
64         edbmodel = re.findall(r'^<input id="edbModel" type="hidden" value="(.*?)" />$', model_detail, re.M)
65         edbmodel = edbmodel[0]
66         edbmodel = edbmodel.replace('&quot;', '').replace(',,', ',').replace('\\', '')
67         edbmodel = re.findall(r'indicatorId:(?P<indicatorId>\d*?),indicatorName:.*?\[(?P<indicatorName>\d*?)\]', edbmodel, re.M)
68         for model in edbmodel:
69             if model[0] != model[1]:
70                 model_error = str(model_id_detail) #+ ':' + str(model[0]) + ':' + str(model[1])#將錯誤模型、錯誤指標id、正確指標id進行拼接
71                 repeat_id.append(model_error)
72     print(repeat_id)#打印錯誤信息
73     print(len(repeat_id))#打印錯誤信息長度
74 
75 if __name__ == '__main__':
76     login()#將登陸狀態保存在session中
77     model_id = get_model_id(num)#將全部的模型id以列表的形式保存在model_id中
78     get_error(model_id)#獲取錯誤信息