爬取代理IP

如今爬蟲好難作啊，有些網站直接封IP，本人小白一個，還沒錢，只能找免費的代理IP，因而去爬了西刺免費代理，結果技術值過低，程序還沒調試好，
IP又被封了。。。
IP又被封了。。。
IP又被封了。。。

因而又開始找其餘的免費代理，找到了89免費代理，這個網站提供了API接口。在我調試過程當中發現，這個API接口好像對爬蟲沒啥限制，哈哈哈！！！

下面附上代碼：

1.此次找了好多User-Agent，以防萬一。

def __init__(self):
        self.get_ip_url = "http://www.89ip.cn/tqdl.html?api=1&num=3000"
        self.check_ip_url = "http://www.baidu.com"
        self.headers =  [
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36
            (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 
            (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60"},
            {"User-Agent":"Opera/8.0 (Windows NT 5.1; U; en)"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208
            Firefox/2.0.0 Opera 9.50"},
            {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera
            9.50"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101
            Firefox/34.0"},
            {"User-Agent":"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10)
            Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 
            (KHTML,like Gecko) Version/5.1.7 Safari/534.57.2 "},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML,
            like Gecko) Chrome/39.0.2171.71 Safari/537.36"},
            {"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML,like
            Gecko) Chrome/23.0.1271.64 Safari/537.11"},
            {"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) 
            AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML,
            like Gecko) Chrome/30.0.1599.101 Safari/537.36"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like
            Gecko"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML,
            like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML,
            like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER"},
            {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; 
            Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 
            3.0.30729;Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)"},
            {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;
            QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"},
            {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64;
            Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 
            3.0.30729;Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)"},
            {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;
            QQDownload 732; .NET4.0C; .NET4.0E) "},
            {"User-Agent":"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like 
            Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0"},
            {"User-Agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 
            Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, 
            like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36"},
            {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, 
            like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36"},
            {"User-Agent":"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; 
            en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2
            Safari/6533.18.5"},
            {"User-Agent":"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; 
            en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2
            Safari/6533.18.5"},
            {"User-Agent":"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) 
            AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 
            Safari/6533.18.5"},
            {"User-Agent":"Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us)
            AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 
            Safari/6533.18.5"},
            {"User-Agent":"Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; 
            HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) 
            Version/4.0 Mobile Safari/533.1"},
            {"User-Agent":"Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One
            Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile 
            Safari/533.1"},
            {"User-Agent":"MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn;
            MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) 
            Version/4.0 Mobile Safari/533.1"},
            {"User-Agent":"Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945;
            U; en-GB) Presto/2.8.149 Version/11.10"},
            {"User-Agent":"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39)
            AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13"},
            {"User-Agent":"Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) 
            AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile
            Safari/534.1+"},
            {"User-Agent":"Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) 
            AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6
            TouchPad/1.0"},
            {"User-Agent":"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019;
            Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko)
            BrowserNG/7.1.18124"},
            {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; 
            Trident/5.0; IEMobile/9.0; HTC; Titan)"},
            {"User-Agent":"UCWEB7.0.2.37/28/999"},
            {"User-Agent":"NOKIA5700/ UCWEB7.0.2.37/28/999"},
            {"User-Agent":"Openwave/ UCWEB7.0.2.37/28/999"},
            {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0;) 
            Opera/UCWEB7.0.2.37/28/999"}]
        self.get_ip_queue = Queue()
        self.pass_ip_list = []

2.發送請求，經過正則表達式獲取IP列表

def get_ip_list(self,url):
        header = random.choice(self.headers)
        response = requests.get(url,headers=header)
        ip_list = re.findall(r"(.*?)<br>", response.content.decode())[1:]
        for ip in ip_list:
            self.get_ip_queue.put(ip)

3.檢查代理IP是否可用

def check_ip(self):
        while not self.get_ip_queue.empty():
            ip = self.get_ip_queue.get()
            proxy = {"http": "http://" + ip,
                     "https": "https://" + ip}
            header = random.choice(self.headers)
            try:
                response = requests.get(self.check_ip_url,headers=header,proxies=proxy,timeout=3)
            except:
                print(ip,"鏈接超時")
            else:
                print(ip,response.status_code)
                if response.status_code == 200:
                    self.pass_ip_list.append(ip)
            self.get_ip_queue.task_done()

4.保存到csv文件中

def save_csv(self,pass_ip_list):
        pass_ip_df = pd.DataFrame({"ip": pass_ip_list})
        pass_ip_df.to_csv("./ip.csv", index=False)

5.主運行程序

def run(self):
        """運行程序"""
        t_start = datetime.datetime.now()
        # 1.獲取代理IP列表
        self.get_ip_list(self.get_ip_url)
        # 2.驗證代理IP有效性
        for i in range(100):
            t = threading.Thread(target=self.check_ip)
            t.setDaemon(True)
            t.start()
        # self.get_ip_queue.join()
        # 3.保存代理IP至csv文件
        print("*" * 100)
        self.save_csv(self.pass_ip_list)
        t_end = datetime.datetime.now()
        print("保存成功", t_end.strftime('%Y-%m-%d %H:%M:%S'))
        print("用時{}".format(t_end-t_start),"獲取{}個代理IP".format(len(self.pass_ip_list)))
        # self.pass_ip_list = []


if __name__ == '__main__':
    daili89 = Daili89()
    while True:
        schedule.run_pending()
        daili89.run()
        time.sleep(600)

運行結果以下圖：

---

最後，
人生苦短，仍是要積極地走下去！