#!coding:utf-8 import time import random import pymysql from splinter import Browser bs="" data={} NO=0 def chrome_env(): global bs executable_path = {'executable_path':'./chromedriver'} bs=Browser('chrome',user_agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1)",**executable_path) bs.driver.set_page_load_timeout(10) def firefox_env(): global bs executable_path = {'executable_path':'/root/geckodriver'} bs=Browser('firefox',user_agent="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1)",**executable_path) def qqcheck(url): global bs chkbox='http://guanjia.qq.com/online_server/result.html?url=%s'%url bs.visit(chkbox) while True: if not bs.find_by_xpath("//div[@id='score_img']/span") and bs.status_code.code != 200: bs.reload() else: #time.sleep(3) s=bs.find_by_xpath("//div[@id='score_img']/span").text #####等待結果檢測.... while True: if '正在檢測' in s : time.sleep(3) s=bs.find_by_xpath("//div[@id='score_img']/span").text else: break break data_list({'qq':s,'ym':url,'NO':1}) def q360check(url): global bs textbody="" chkbox='http://webscan.360.cn/index/checkwebsite?url=%s'%url bs.visit(chkbox) while True: if not bs.find_by_xpath("//div[@class='report_list']") and bs.status_code.code != 200 : bs.reload() else: time.sleep(3) for i in bs.find_by_xpath("//div[@class='report_list']"): textbody=i.text+textbody break data_list({'qihoo':textbody,'ym':url,'NO':1}) ###整合數據統一入庫 def data_list(key,**keywords): global data,NO for k,v in key.items(): data.update({'%s'%k:'%s'%v}) NO=NO+int(data.get('NO')) if NO == 2: NO=0 datainsert(data) def checkdata(ym): sj=time.strftime("%Y-%m-%d",time.localtime()) cursor,con=connect() try: sql="select ym,sj from domain where ym=%s and sj like %s" cursor.execute(sql, (ym,"%s%%"%sj)) result = cursor.fetchone() if result: return 0 else: return 1 finally: con.close() def datainsert(data): sj=time.strftime("%Y-%m-%d %H:%M",time.localtime()) print(sj) cursor,con=connect() try: if checkdata(data.get('ym')) == 1: sql = "insert into `domainc`.`domain` ( `qq`, `qihoo`, `sj`, `ym`) values ( %s, %s, %s, %s)" cursor.execute(sql, (data.get('qq'), data.get('qihoo'),sj,data.get('ym'))) con.commit() else: print("update...") sql = "update `domainc`.`domain` set qq=%s,qihoo=%s,sj=%s where ym=%s" cursor.execute(sql, (data.get('qq'), data.get('qihoo'),sj,data.get('ym'))) con.commit() finally: con.close() def connect(): connection = pymysql.connect(host='127.0.0.1', user='root', password='1', db='domainc', charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) cursor=connection.cursor() return cursor,connection def readdomain(): cursor,con=connect() try: sql="select domain from domainlist" cursor.execute(sql) result = cursor.fetchall() for i in result: print(i.get('domain')) try: time.sleep(random.randint(1,9)) q360check(i.get('domain')) time.sleep(random.randint(1,9)) qqcheck(i.get('domain')) except: print("%s is fail"%i.get('domain')) finally: con.close() #firefox_env() chrome_env() while True: readdomain() time.sleep(14200)