WebDriver 登錄 Jsoup抓取內容

 

1. 環境

pom:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>mybatis</groupId>
	<artifactId>test</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>jar</packaging>

	<name>test</name>
	<url>http://maven.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
	</properties>

	<!-- 添加mybatis-generator插件 -->
	<!-- ——>在Goals框中輸入：mybatis-generator:generate 運行mybatis插件 -->
	<build>
		<plugins>
			<plugin>
				<groupId>org.mybatis.generator</groupId>
				<artifactId>mybatis-generator-maven-plugin</artifactId>
				<version>1.3.2</version>
				<configuration>
					<verbose>true</verbose>
					<overwrite>true</overwrite>
				</configuration>
			</plugin>
		</plugins>
	</build>


	<dependencies>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.38</version>
		</dependency>

		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>3.8.1</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>org.mybatis</groupId>
			<artifactId>mybatis</artifactId>
			<version>3.3.1</version>
		</dependency>
		<dependency>
			<groupId>org.apache.poi</groupId>
			<artifactId>poi</artifactId>
			<version>3.12</version>
		</dependency>

		<dependency>
			<groupId>commons-logging</groupId>
			<artifactId>commons-logging</artifactId>
			<version>1.2</version>
		</dependency>
		<dependency>
			<groupId>net.sourceforge.jexcelapi</groupId>
			<artifactId>jxl</artifactId>
			<version>2.6.12</version>
		</dependency>

		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpclient</artifactId>
			<version>4.5.2</version>
		</dependency>

		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.12</version>
		</dependency>
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.8.3</version>
		</dependency>
		<dependency>
			<groupId>org.seleniumhq.selenium</groupId>
			<artifactId>selenium-server</artifactId>
			<version>2.53.0</version>
		</dependency>
		<dependency>
			<groupId>log4j</groupId>
			<artifactId>log4j</artifactId>
			<version>1.2.17</version>
		</dependency>


	</dependencies>

</project>

2. 初始化WebDriver的類 DriverFactory.java

 

 

package test;

import java.util.Arrays;

import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.DesiredCapabilities;

public class DriverFactory {

	public static ChromeDriver create() {

		// TODO Auto-generated method stub
		String chromdriver = "C:\\Users\\admin\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe";
		System.setProperty("webdriver.chrome.driver", chromdriver);
		ChromeOptions options = new ChromeOptions();

		DesiredCapabilities capabilities = DesiredCapabilities.chrome();
		capabilities.setCapability("chrome.switches", Arrays.asList("--start-maximized"));
		options.addArguments("--test-type", "--start-maximized");
		ChromeDriver driver = new ChromeDriver(options);
		return driver;
	}

}

 

 

3. 西祠衚衕的登錄抓取類

package test;

import java.io.File;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.openqa.jetty.http.SSORealm;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.Platform;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.ie.InternetExplorerDriver;
import org.openqa.selenium.remote.DesiredCapabilities;
import mx4j.log.Log;

public class XiciLogin2 {
	public Logger log = Logger.getLogger(Main.class);
	public static Set<Cookie> cookies = new HashSet<Cookie>();
	public static ChromeDriver driver = DriverFactory.create();

	/** * 抓取到每個分頁上全部詳細頁連接 * *@param url */
	public List<String> crawlSource(String url) {
		int time = 1;
		System.out.println("開始抓： " + url);
		log.info("開始抓： " + url);
		List<String> sourceUrls = new ArrayList<String>();
		String baseUrl = "http://www.xici.net";
		driver.get(url);
		Document document = Jsoup.parse(driver.getPageSource());
		WebElement webElement = driver.findElement(By.xpath(".//*[@id='board_t']/tbody/tr/td[2]/a"));
		Elements elements = document.select("table#board_t tbody tr");
		System.out.println(elements);
		if (elements != null) {
			for (Element element : elements) {
				if (element.select("td").isEmpty()) {
					continue;
				}
				String targets = element.select("td a[onclick=this.parentNode.className ='visited';]").attr("href");
				if (targets == "" || targets == null) {
					continue;
				}
				targets = baseUrl + targets; // System.out.println(targets);
				sourceUrls.add(targets);
			}
		} else

		{
			System.out.println(url + "中沒有詳細頁連接~~");
		}
		System.out.println(sourceUrls.size());
		if (sourceUrls.size() == 0 && time <= 5) {
			System.out.println("抓不到啦~ 從新抓一下");
			crawlSource(url);
			time++;
		}
		return sourceUrls;
	}

	/** * 解析詳細頁 出東西 */
	public void crawlTarget(String url) {
		driver.get(url);
		Document document = Jsoup.parse(driver.getPageSource());
		System.out.println("抓" + url + "的標題"); // 取標題 Element element =
		document.select("div#doc_tit h1").first();
		if (element != null) {
			System.out.println("標題:" + element.text());
		} else {
			System.out.println("");
		}
	}

	public static void main(String[] args) {
		PropertyConfigurator.configure("log4j.properties");
		XiciLogin2 xc = new XiciLogin2();
		String site = "http://www.xici.net/b1513005/";
		try {
			xc.xiciLogin(); //
			xc.crawlTarget("http://www.xici.net/d191739198.htm");
			xc.getMaxPageNum("http://www.xici.net/b1468535/");
			int page = 1; //
			int maxPageNum = xc.getMaxPageNum(site);
			do {
				String sourceUrl = site + page;
				System.out.println("分頁: " + sourceUrl);
				List<String> targetsList = xc.crawlSource(sourceUrl);
				if (targetsList.isEmpty()) {
					System.out.println("沒抓到詳細頁！！");
				} else {
					for (String target : targetsList) {
						try {
							xc.crawlTarget(target);
							Thread.sleep(3000);
						} catch (Exception e) {
							e.printStackTrace();
						}
					}
				}
				page++;
				Thread.sleep(3500);
			} while (page <= 15);
		} catch (

		Exception e) {
			e.printStackTrace();
		}
	}

	/** * 獲取當前入口site的最大分頁數 **/
	public int getMaxPageNum(String site) {
		Document document = null;
		int maxPageNum = 0;
		try {
			document = Jsoup.connect(site).get();
			Element element = document.select("div#page").first();
			String s = element.text();
			if (s.contains("共")) {
				s = s.split("共")[1];
				s = s.split("頁")[0];
			}
			System.out.println(s);
			maxPageNum = Integer.parseInt(s);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return maxPageNum;
	}

	public void xiciLogin() throws Exception {
		System.setProperty("webdriver.chrome.driver",
				"C:\\Users\\admin\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
		driver.get("http://account.xici.net/login");
		WebElement user = driver.findElement(By.name("username"));
		WebElement pwa = driver.findElement(By.name("password")); // 分別將用戶名和密碼文本框清空
																	// user.clear();
		pwa.clear(); // 輸入用戶名和密碼 user.sendKeys("*******");
		pwa.sendKeys("*********"); // 找到登錄按鈕點擊 //
		driver.findElement(By.name("TANGRAM__PSP_3__submit")).click();
		driver.findElement(By.xpath("html/body/div[3]/div[2]/div[2]/form/div[4]/button")).click();
		// 輸出title System.out.println(driver.getTitle()); cookies =
		driver.manage().getCookies();
		System.out.println(cookies);
		for (Cookie cookie2 : cookies) {
			driver.manage().addCookie(cookie2);
			System.out.println(cookie2);
		} // 能打開15頁說明登錄成功 //
		driver.get("http://www.xici.net/b1402132/15");
	}
}