py批量圖片抓取

說明：藍色=命令名稱

      淺綠=命令參數

      淺藍=選項

      紫色=目錄

      系統環境：CentOS  5.5  x86_64

      python版本：Python 2.7.3

 

#!/usr/bin/env python
#-*- coding:utf-8 -*-
#Author:left_left
import sys
import os
import urllib2
import threading
from optparse import OptionParser

mutex = threading.Lock()
logfile = open("wget_pic.log", "w")

class GetPic(threading.Thread):
    def __init__(self, dirname, uris, ip):
        threading.Thread.__init__(self)
        self.uris = uris
        self.dirname = dirname
        self.ip = ip

    def run(self):
        global logfile, mutex
        for uri in self.uris:
            if uri.startswith("http"):
                try:
                    if self.ip:
                        s_uri = uri.split("/")
                        host = s_uri[2]
                        s_uri[2] = self.ip
                        request = urllib2.Request("/".join(s_uri),
                        headers = {"Host":host})
                        response = urllib2.urlopen(request, timeout = 5)
                    else:
                        response = urllib2.urlopen(uri, timeout = 5)
                except urllib2.HTTPError, e:
                    if mutex.acquire():
                        print uri
                        logfile.write(uri)
                        mutex.release()
                except urllib2.URLError, e:
                    print e
                else:
                    pic_name = uri.split("/")[-1].strip()
                    f = open(self.dirname + "/" + pic_name, 'wb')
                    f.write(response.read())
                    f.close()

def parse():
    p = OptionParser()

    p.add_option('-p', '--ipaddr', dest = "ip")
    p.add_option('-g', '--log', dest = 'log')
    p.add_option('-d', '--to-dir', dest = 'dir')
    p.add_option('-t', '--threads', dest = 'threads', default = 5)

    return p

def main():
    global logfile
    p = parse()
    options, args = p.parse_args()

    for file in args:
        if options.dir:
            dirname = options.dir
        else:
            dirname = file.split("/")[-1].split('.')[0]

        try:
            os.mkdir(dirname)
        except OSError, e:
            #print "Mkdir Error:%s" % e
            #exit(1)
            pass
        if options.log:
            try:
                logfile = open(options.log, "w")
            except IOError, e:
                print e
                exit(1)

        thread_num = int(options.threads)
        uris = open(file).readlines()
        members = len(uris)/thread_num
        threads = []

        for i in xrange(thread_num):
            threads.append(GetPic(dirname,
            uris[i*members:(i+1)*members], options.ip))

        if len(uris)%thread_num:
            threads.append(GetPic(dirname,
            uris[thread_num*members:], options.ip))

        for t in threads:
            t.setDaemon(True)
            t.start()

        for t in threads:
            t.join()
logfile.close()

if __name__ == "__main__":
    main()

效果如圖：

 

day_130412.txt內容: