Web開發 - 網絡爬蟲

時間 2019-11-08

標籤 web 開發網絡爬蟲欄目 HTML 简体版

原文原文鏈接

網絡爬蟲

WebCollector是Java的爬蟲框架，比起直接採用HttpClient、JSoup爬取有強大的好處，框架中集成了斷點續爬、Url去重、自定義Http請求等。例如Nutch、Heritrix，底層實現都相似。html

下面是倆種爬蟲的實現：java

一、Node爬蟲

npm下載模塊node

var eventproxy = require('./lib/eventproxy');
var ep = new eventproxy();
var superagent = require('superagent');
var cheerio = require('cheerio');
var url = require('url');

var cnodeUrl = 'https://cnodejs.org/';

superagent.get(cnodeUrl).end(function(err,res){
    if(err)
        return console.error(err);
    var topicUrls = [];
    var $ = cheerio.load(res.text);
    //獲取首頁全部連接
    $('#topic_list .topic_title').each(function(idx,element){
        var $element = $(element);
        var href = url.resolve(cnodeUrl,$element.attr('href'));
        topicUrls.push(href);
    });
    console.log(topicUrls);

    // 命令 ep 重複監聽 topicUrls.length 次（在這裏也就是 40 次） `topic_html` 事件再行動
    ep.after('topic_html', topicUrls.length, function (topics) {
      // topics 是個數組，包含了 40 次 ep.emit('topic_html', pair) 中的那 40 個 pair

      // 開始行動
      topics = topics.map(function (topicPair) {
        // 接下來都是 jquery 的用法了
        var topicUrl = topicPair[0];
        var topicHtml = topicPair[1];
        var $ = cheerio.load(topicHtml);
        return ({
          title: $('.topic_full_title').text().trim(),
          href: topicUrl,
          comment1: $('.reply_content').eq(0).text().trim(),
        });
      });

      console.log('final:');
      console.log(topics);
    });

    topicUrls.forEach(function (topicUrl) {
      superagent.get(topicUrl)
        .end(function (err, res) {
          console.log('fetch ' + topicUrl + ' successful');
          ep.emit('topic_html', [topicUrl, res.text]);
        });
    });
});
//異步併發
ep.all('data1','data2',function(data1,data2){
    console.log(data1+","+data2);
});
superagent.get(cnodeUrl).end(function(err,res){
    ep.emit('data1',res.test);
});
superagent.get(cnodeUrl).end(function(err,res){
    ep.emit('data2',res.test);
});

二、WebCollector

須要下載的Jar：jquery

WebCollector，解壓後將webcollector-2.32-bin中的jar放入項目中。git
selenium（用於解析Html）。github

下面是爬取新浪微博的代碼：web

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import cn.edu.hfut.dmic.webcollector.net.HttpResponse;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 
 * 爬取微博
 * @author Alex
 *
 */
public class WeiboCrawler extends BreadthCrawler {

    private String cookie;

    public WeiboCrawler(String crawlPath, boolean autoParse) throws Exception {
        super(crawlPath, autoParse);
        cookie = WeiboCN.getSinaCookie("XXXXXXXXXXX", "XXXXXXXXXX");//帳號、密碼
    }

    @Override
    public HttpResponse getResponse(CrawlDatum crawlDatum) throws Exception {
        HttpRequest request = new HttpRequest(crawlDatum);
        request.setCookie(cookie);
        return request.getResponse();
    }

    public void visit(Page page, CrawlDatums next) {
        int pageNum = Integer.valueOf(page.getMetaData("pageNum"));
        Elements weibos = page.select("div.c");//或者Document doc = page.doc();
        for (Element weibo : weibos) {
            System.out.println("第" + pageNum + "頁\t" + weibo.text());
        }
    }

    public static void main(String[] args) throws Exception {
        WeiboCrawler crawler = new WeiboCrawler("WeiboCrawler", false);
        crawler.setThreads(3);//線程數
        for (int i = 1; i <= 5; i++) {//爬取XXX前5頁
            crawler.addSeed(new CrawlDatum("http://weibo.cn/zhouhongyi?vt=4&page=" + i).putMetaData("pageNum", i + ""));
        }
        //crawlerNews.setResumable(true);//斷點續爬
        crawler.start(1);
    }

}

import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import cn.edu.hfut.dmic.webcollector.net.HttpResponse;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.util.Set;
import javax.imageio.ImageIO;

import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;

import java.awt.BorderLayout;
import java.awt.Container;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.image.BufferedImage;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JTextField;

/**
 * 
 * @author Alex
 *
 */
public class WeiboCN {

    public static String getSinaCookie(String username, String password) throws Exception {
        HtmlUnitDriver driver = new HtmlUnitDriver();//加載Html解析驅動
        driver.setJavascriptEnabled(true);
        driver.get("http://login.weibo.cn/login/");
        
        WebElement ele = driver.findElementByCssSelector("img");//selenium選擇器
        String src = ele.getAttribute("src");
        String cookie = concatCookie(driver);
        
        HttpRequest request = new HttpRequest(src);//請求驗證碼
        request.setCookie(cookie);
        
        HttpResponse response = request.getResponse();
        ByteArrayInputStream is = new ByteArrayInputStream(response.getContent());
        BufferedImage img = ImageIO.read(is);
        is.close();
        ImageIO.write(img, "png", new File("result.png"));
        String userInput = new CaptchaFrame(img).getUserInput();
        
        //模擬表單登陸
        WebElement mobile = driver.findElementByCssSelector("input[name=mobile]");
        mobile.sendKeys(username);
        WebElement pass = driver.findElementByCssSelector("input[type=password]");
        pass.sendKeys(password);
        WebElement code = driver.findElementByCssSelector("input[name=code]");
        code.sendKeys(userInput);
        WebElement rem = driver.findElementByCssSelector("input[name=remember]");
        rem.click();
        WebElement submit = driver.findElementByCssSelector("input[name=submit]");
        submit.click();
        String result = concatCookie(driver);
        driver.close();
        if (result.contains("gsid_CTandWM")) {
            return result;
        } else {
            throw new Exception("weibo login failed");
        }
    }

    public static String concatCookie(HtmlUnitDriver driver) {
        Set<Cookie> cookieSet = driver.manage().getCookies();
        StringBuilder sb = new StringBuilder();
        for (Cookie cookie : cookieSet) {
            sb.append(cookie.getName() + "=" + cookie.getValue() + ";");
        }
        String result = sb.toString();
        return result;
    }

    //根據圖片生成窗體驗證碼
    public static class CaptchaFrame {
        JFrame frame;//窗口
        JPanel panel;//面板
        JTextField input;//輸入框
        int inputWidth = 100;
        BufferedImage img;
        String userInput = null;

        public CaptchaFrame(BufferedImage img) {
            this.img = img;
        }

        public String getUserInput() {
            frame = new JFrame("輸入驗證碼");
            final int imgWidth = img.getWidth();
            final int imgHeight = img.getHeight();
            int width = imgWidth * 2 + inputWidth * 2;
            int height = imgHeight * 2+50;
            Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
            int startx = (dim.width - width) / 2;
            int starty = (dim.height - height) / 2;
            frame.setBounds(startx, starty, width, height);
            Container container = frame.getContentPane();
            container.setLayout(new BorderLayout());
            panel = new JPanel() {
                @Override
                public void paintComponent(Graphics g) {//將圖片畫在面板上
                    super.paintComponent(g);
                    g.drawImage(img, 0, 0, imgWidth * 2, imgHeight * 2, null);
                }
            };
            panel.setLayout(null);
            container.add(panel);
            input = new JTextField(6);
            input.setBounds(imgWidth * 2, 0, inputWidth, imgHeight * 2);
            panel.add(input);
            JButton btn = new JButton("登陸");
            btn.addActionListener(new ActionListener() {//註冊監聽
                public void actionPerformed(ActionEvent e) {
                    userInput = input.getText().trim();
                    synchronized (CaptchaFrame.this) {//同步窗口釋放
                        CaptchaFrame.this.notify();
                    }
                }
            });
            btn.setBounds(imgWidth * 2 + inputWidth, 0, inputWidth, imgHeight * 2);
            panel.add(btn);
            frame.setVisible(true);
            synchronized (this) {
                try {
                    this.wait();
                } catch (InterruptedException ex) {
                    ex.printStackTrace();
                }
            }
            frame.dispose();
            return userInput;
        }
    }

}

你們注意password！這個name="password_9384"其中的數字是動態生成的，每次請求都會變，因此上面代碼中的selenium選擇器要用input[type=password]。npm