java抓取網頁數據，登陸以後抓取數據。

最近作了一個從網絡上抓取數據的一個小程序。主要關於信貸方面，收集的一些黑名單網站，從該網站上抓取到本身系統中。

也找了一些資料，以爲沒有一個很好的，全面的例子。所以在這裏作個筆記提醒本身。

首先須要一個jsoup的jar包，我用的1.6.0。。下載地址爲：http://pan.baidu.com/s/1mgqOuHa

1，獲取網頁內容（核心代碼，技術有限沒封裝）。

2，登陸以後抓取網頁數據（如何在請求中攜帶cookie）。

3，獲取網站的ajax請求方法（返回json）。

以上這三點我就用一個類所有包含（比較糙望見諒，直接copy代碼過去，應該就能夠用）

一，這個類分別有這上面的1,2,3三中方法，直接main方法能夠進行測試

package com.minxinloan.black.web.utils;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.StringTokenizer;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class CookieUtil {

    public final static String CONTENT_TYPE = "Content-Type";

    public static void main(String[] args) {
        
        //String loginURL = "http://www.p2peye.com/member.php?mod=logging&action=login&loginsubmit=yes&loginhash=Lsc66&username=puqiuxiaomao&password=a1234567";
        String listURL = "http://www.p2peye.com/blacklist.php?p=2";
        String logURL = "http://www.p2peye.com/member.php";


        //********************************須要登陸的*************************************************
        try {
                Connection.Response  res = 
                        Jsoup.connect(logURL)
                            .data("mod","logging"
                                    ,"action","login"
                                    ,"loginsubmit","yes"
                                    ,"loginhash","Lsc66"
                                    ,"username","puqiuxiaomao"
                                    ,"password","a1234567")
                            .method(Method.POST)
                            .execute();
                
                
                //這兒的SESSIONID須要根據要登陸的目標網站設置的session Cookie名字而定
                Connection con=Jsoup.connect(listURL);
                //設置訪問形式（電腦訪問，手機訪問）：直接百度都參數設置
                con.header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
                //把登陸信息的cookies保存如map對象裏面
                Map <String,String>  map=res.cookies();
                Iterator<Entry<String,String>> it =map.entrySet().iterator();
                while(it.hasNext()){
                    Entry<String,String> en= it.next(); 
                    //把登陸的信息放入請求裏面
                    con =con.cookie(en.getKey(), en.getValue());
                    
                }
                //再次獲取Document對象。
                Document objectDoc = con.get();
                
                Elements elements = objectDoc.getAllElements();//獲取這個鏈接返回頁面的源碼內容（不是源碼跟源碼差很少）
                for (Element element : elements) {
                    //element是迭代出來的標籤：如：<div><span></span></div>
                    Elements elements2= element.getAllElements();//
                     for (Element element2 : elements2) {
                         element2.text();
                         element2.attr("href");//獲取標籤屬性。element2表明a標籤：href表明屬性
                         element2.text();//獲取標籤文本
                    }
                }
                
                //********************************不須要登陸的*************************************************
                
                String URL = "http://www.p2peye.com/blacklist.php?p=2";
                Document conTemp = Jsoup.connect(URL).get();
                Elements elementsTemps = conTemp.getAllElements();
                 for (Element elementsTemp : elementsTemps) {
                     elementsTemp.text();
                     elementsTemp.attr("href");//獲取標籤屬性。element2表明a標籤：href表明屬性
                     elementsTemp.text();//獲取標籤文本
                }
                
                
                //********************************ajax方法獲取內容。。。*************************************************。
                 HttpURLConnection connection = null;
                    BufferedReader reader = null;
                    try {
                        StringBuffer sb = new StringBuffer();
                        URL getUrl = new URL(URL);
                        connection = (HttpURLConnection)getUrl.openConnection();
                        reader = new BufferedReader(new InputStreamReader(
                                connection.getInputStream(),"utf-8"));
                        String lines;
                        while ((lines = reader.readLine()) != null) {
                            sb.append(lines);
                        };
                        List<Map<String, Object>> list = parseJSON2List(sb.toString());//json轉換成list
                    } catch (Exception e) {
                        
                    } finally{
                        if(reader!=null)
                            try {
                                reader.close();
                            } catch (IOException e) {
                            }
                        // 斷開鏈接
                        connection.disconnect();
                    }
                
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        
    }
    

    public static Map<String, Object> parseJSON2Map(String jsonStr){  
        Map<String, Object> map = new HashMap<String, Object>();  
        //最外層解析  
        JSONObject json = JSONObject.fromObject(jsonStr);  
        for(Object k : json.keySet()){  
            Object v = json.get(k);   
            //若是內層仍是數組的話，繼續解析  
            if(v instanceof JSONArray){  
                List<Map<String, Object>> list = new ArrayList<Map<String,Object>>();  
                Iterator<JSONObject> it = ((JSONArray)v).iterator();  
                while(it.hasNext()){  
                    JSONObject json2 = it.next();  
                    list.add(parseJSON2Map(json2.toString()));  
                }  
                map.put(k.toString(), list);  
            } else {  
                map.put(k.toString(), v);  
            }  
        }  
        return map;  
    }  
    
    public static List<Map<String, Object>> parseJSON2List(String jsonStr){  
        JSONArray jsonArr = JSONArray.fromObject(jsonStr);  
        List<Map<String, Object>> list = new ArrayList<Map<String,Object>>();  
        Iterator<JSONObject> it = jsonArr.iterator();  
        while(it.hasNext()){  
            JSONObject json2 = it.next();  
            list.add(parseJSON2Map(json2.toString()));  
        }  
        return list;  
    }  
    
    

}

二，這個是獲取驗證碼的類，能夠研究下。（但你要要分析出網站的驗證碼的請求地址）

package com.minxinloan.black.web.utils;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

public class Utils {//解析驗證碼的
    public static Content getRandom(String method, String sUrl,// 要解析的url
            Map<String, String> paramMap, // 存放用戶名和密碼的map
            Map<String, String> requestHeaderMap,// 存放COOKIE的map
            boolean isOnlyReturnHeader, String path) {

        Content content = null;
        HttpURLConnection httpUrlConnection = null;
        InputStream in = null;
        try {
            URL url = new URL(sUrl);
            boolean isPost = "POST".equals(method);
            if (method == null
                    || (!"GET".equalsIgnoreCase(method) && !"POST"
                            .equalsIgnoreCase(method))) {
                method = "POST";
            }
            URL resolvedURL = url;
            URLConnection urlConnection = resolvedURL.openConnection();
            httpUrlConnection = (HttpURLConnection) urlConnection;
            httpUrlConnection.setRequestMethod(method);
            httpUrlConnection.setRequestProperty("Accept-Language",
                    "zh-cn,zh;q=0.5");
            // Do not follow redirects, We will handle redirects ourself
            httpUrlConnection.setInstanceFollowRedirects(false);
            httpUrlConnection.setDoOutput(true);
            httpUrlConnection.setDoInput(true);
            httpUrlConnection.setConnectTimeout(5000);
            httpUrlConnection.setReadTimeout(5000);
            httpUrlConnection.setUseCaches(false);
            httpUrlConnection.setDefaultUseCaches(false);
            httpUrlConnection.connect();

            int responseCode = httpUrlConnection.getResponseCode();

            if (responseCode == HttpURLConnection.HTTP_OK
                    || responseCode == HttpURLConnection.HTTP_CREATED) {
                byte[] bytes = new byte[0];
                if (!isOnlyReturnHeader) {
                    DataInputStream ins = new DataInputStream(
                            httpUrlConnection.getInputStream());
                    // 驗證碼的位置
                    DataOutputStream out = new DataOutputStream(
                            new FileOutputStream(path + "/code.bmp"));
                    byte[] buffer = new byte[4096];
                    int count = 0;
                    while ((count = ins.read(buffer)) > 0) {
                        out.write(buffer, 0, count);
                    }
                    out.close();
                    ins.close();
                }
                String encoding = null;
                if (encoding == null) {
                    encoding = getEncodingFromContentType(httpUrlConnection
                            .getHeaderField(""));
                }
                content = new Content(sUrl, new String(bytes, encoding),
                        httpUrlConnection.getHeaderFields());
            }
        } catch (Exception e) {
            return null;
        } finally {
            if (httpUrlConnection != null) {
                httpUrlConnection.disconnect();
            }
        }
        return content;
    }

    public static String getEncodingFromContentType(String contentType) {
        String encoding = null;
        if (contentType == null) {
            return null;
        }
        StringTokenizer tok = new StringTokenizer(contentType, ";");
        if (tok.hasMoreTokens()) {
            tok.nextToken();
            while (tok.hasMoreTokens()) {
                String assignment = tok.nextToken().trim();
                int eqIdx = assignment.indexOf('=');
                if (eqIdx != -1) {
                    String varName = assignment.substring(0, eqIdx).trim();
                    if ("charset".equalsIgnoreCase(varName)) {
                        String varValue = assignment.substring(eqIdx + 1)
                                .trim();
                        if (varValue.startsWith("\"")
                                && varValue.endsWith("\"")) {
                            // substring works on indices
                            varValue = varValue.substring(1,
                                    varValue.length() - 1);
                        }
                        if (Charset.isSupported(varValue)) {
                            encoding = varValue;
                        }
                    }
                }
            }
        }
        if (encoding == null) {
            return "UTF-8";
        }
        return encoding;
    }

    // 這個是輸出
    public static boolean inFile(String content, String path) {
        PrintWriter out = null;
        File file = new File(path);
        try {
            if (!file.exists()) {
                file.createNewFile();
            }
            out = new PrintWriter(new FileWriter(file));

            out.write(content);
            out.flush();
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            out.close();
        }
        return false;
    }

    public static String getHtmlReadLine(String httpurl) {
        String CurrentLine = "";
        String TotalString = "";
        InputStream urlStream;
        String content = "";

        try {
            URL url = new URL(httpurl);

            HttpURLConnection connection = (HttpURLConnection) url
                    .openConnection();

            connection.connect();
            System.out.println(connection.getResponseCode());
            urlStream = connection.getInputStream();

            BufferedReader reader = new BufferedReader(

            new InputStreamReader(urlStream, "utf-8"));

            while ((CurrentLine = reader.readLine()) != null) {
                TotalString += CurrentLine + "\n";
            }

            content = TotalString;

        } catch (Exception e) {
        }

        return content;
    }
}


class Content {
    private String url;
    private String body;
    private Map<String, List<String>> m_mHeaders = new HashMap<String, List<String>>();

    public Content(String url, String body, Map<String, List<String>> headers) {
        this.url = url;
        this.body = body;
        this.m_mHeaders = headers;
    }

    public String getUrl() {
        return url;
    }

    public String getBody() {
        return body;
    }

    public Map<String, List<String>> getHeaders() {
        return m_mHeaders;
    }

}