抓取分析網頁批量下載評書(2)之分析下載地址

時間 2019-12-07

標籤抓取分析網頁批量下載評書地址欄目 HTML 简体版

原文原文鏈接

本系列目錄： 《一、搜索有聲小說》《二、分析詳細頁地址》《三、批量下載mp3》

本篇是系列的第二篇，主要介紹如何分析有聲讀物每集的連接地址，本文但願閱讀者根據文章就能本身寫出程序，因此寫的比較詳細，你們見諒，軟件的下載地址。

1、我們隨便找一本評書進入其詳細頁，咱們要得到評書的簡介和劇集列表兩部份內容，具體頁面以下:

分析一下Html代碼，簡介是包含在<div class="book02 padding5 line5">簡介內容</div>中，但須要注意，裏包含了br、script等html標籤,因此抓取內容後，須要清除一下html標籤。

另外有一點須要注意，因爲簡介中可能含有div，因此正則表達式用貪婪模式或非貪婪模式均可能出現沒法正確匹配，因此要保證惟一性，注意下面粗體的部分。

獲取簡介的正則表達式： <div class=""book02 padding5 line5"">(?<Summary>[\s\S]*?)</div>\r\n<div class=""js[\d]*"">

PS:使用IE開發人員工具（F12）時發現一個問題，它會對代碼進行整理，因此你看到內容有可能不是網頁的真實內容，以下面兩幅圖

title,href的順序不同，Chrome中顯示的是實際代碼順序。

具體的正則表達式： <div class=""b2""><a href=""(?<SubPath>[\d\w]*)/play_[\d]*_(?<Number>[\d]*).htm""(\s*title=""[\s\S]*?.mp3"")*>(?<Title>[\s\S]*?).mp3</a></div>

2、評書下載界面以下 :

3、主要的代碼

      Sound.cs 
    

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace psDownload
{
     /// <summary>
     /// 有聲讀物類
     /// </summary>
     public class Sound
    {
         /// <summary>
         /// 編號
         /// </summary>
         public int ID { set; get; }

         /// <summary>
         /// 標題
         /// </summary>
         public string Title { set; get; }

         /// <summary>
         /// 演播者
         /// </summary>
         public string Performer { set; get; }

         /// <summary>
         /// 詳細頁網址
         /// </summary>
         public string Url { set; get; }

         /// <summary>
         /// MP3的下載地址
         /// </summary>
         public string DownUrl { set; get; }

         /// <summary>
         /// 狀態  0：等待下載  1：下載成功  -1：下載失敗
         /// </summary>
         public int Status { set; get; }

         /// <summary>
         /// 錯誤詳情
         /// </summary>
         public string Error { set; get; }
    }
}

       C# Code  
    

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

//評書的名稱、詳細頁網址、所屬分類、文件輸出路徑
private string _title = "" , _url = "" , _category = "" , _outpath = "" ;

//評書集數列表，Sound是一個本身寫的類。
private List<Sound> _list = new List<Sound>();

public frmDetail( string Title, string Url, string OutPath)
{
    InitializeComponent();
     this ._title = Title;
     this ._url = Url;
     this ._outpath = OutPath;

     //獲取評書所屬的分類，後面須要用到，其實從上一個窗體傳過來也行。
    Match match = Regex.Match(Url, @ "http://www.tingchina.com/(?<Category>[\w]*)/disp_[\d]*.htm" , RegexOptions.IgnoreCase | RegexOptions.Multiline);
     if (match.Success)
    {
         this ._category = match.Groups[ "Category" ].Value;
    }
}

private void frmDetail_Load( object sender, EventArgs e)
{
     this .Text = this ._title;
     this .lbl_Title.Text = this ._title;
     this .llbl_Url.Text = this ._url;

     //發送異步請求，獲取評書的詳細信息和劇集列表
    HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create( this ._url);
    req.Timeout = 15 * 1000 ;
    req.Method = "GET" ;
    req.BeginGetResponse( new AsyncCallback(ResponseCallBack), req);
}

/// <summary>
/// http異步請求的回調函數，分析網頁，得到評書的簡介、劇集列表等信息
/// </summary>
/// <param name="result"></param>
public void ResponseCallBack(IAsyncResult result)
{
     string Html = "" ;
    HttpWebRequest req = (HttpWebRequest)result.AsyncState;
     try
    {
         using (HttpWebResponse response = (HttpWebResponse)req.EndGetResponse(result))
        {
            Stream resStream = response.GetResponseStream();
            StreamReader sr = new StreamReader(resStream, Encoding.GetEncoding( "GB2312" ));
            Html = sr.ReadToEnd();
        }
    }
     catch (Exception ex)
    {
         if (IsDisposed || ! this .IsHandleCreated) return ;
         this .Invoke( new Action(() =>
        {
            MessageBox.Show( "抓取網頁出現異常,緣由:" + ex.Message);
        }));
         return ;
    }

     //動態生成Label控件所需的字體
    Font font = new Font( "微軟雅黑" , 10F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, (( byte )( 134 )));

     //獲取評書的簡介
    Match ms_Summary = Regex.Match(Html, @ "<div class=""book02 padding5 line5"">(?<Summary>[\s\S]*?)</div>\r\n<div class=""js[\d]*"">" , RegexOptions.IgnoreCase | RegexOptions.Multiline);
     if (ms_Summary.Success)
    {
         if (IsDisposed || ! this .IsHandleCreated) return ;
         this .Invoke( new Action(() =>
        {
             this .txt_Summary.Text = ClearHtml(ms_Summary.Groups[ "Summary" ].Value);
        }));
    }

     //正則表達式分析網頁，獲取評書的劇集。
    MatchCollection ms = Regex.Matches(Html, @ "<div class=""b2""><a href=""(?<SubPath>[\d\w]*)/play_[\d]*_(?<Number>[\d]*).htm""(\s*title=""[\s\S]*?.mp3"")*>(?<Title>[\s\S]*?).mp3</a></div>" , RegexOptions.IgnoreCase | RegexOptions.Multiline);
     if (ms.Count > 0 )
    {
         //最大寬度
         int MaxWidth = 0 ;
         this ._list.Clear();
         foreach (Match m in ms)
        {
            Sound sound = new Sound();
            sound.ID = int .Parse(m.Groups[ "Number" ].Value);
            sound.Title = m.Groups[ "Title" ].Value;
            sound.Url = "http://www.tingchina.com/" + this ._category + "/" + m.Groups[ "SubPath" ].Value + "/play_" + m.Groups[ "SubPath" ].Value + "_" + m.Groups[ "Number" ].Value + ".htm" ;
             this ._list.Add(sound);

             //獲取字體的大小，找到最寬的那個條記錄
            Size size = TextRenderer.MeasureText(m.Groups[ "Title" ].Value, font);
             if (size.Width > MaxWidth)
            {
                MaxWidth = size.Width;
            }
        }

         if (IsDisposed || ! this .IsHandleCreated) return ;
         this .Invoke( new Action(() =>
        {
             //循環動態生成查詢結果.
             foreach (Sound sound in _list)
            {
                Panel pnl = new Panel();

                Label lbl = new Label();
                lbl.Name = "lbl_" + sound.ID.ToString();
                lbl.Text = sound.Title;
                lbl.Tag = sound.Url;
                lbl.Cursor = System.Windows.Forms.Cursors.Hand;
                lbl.Margin = new System.Windows.Forms.Padding( 15 , 0 , 0 , 10 );
                lbl.ForeColor = System.Drawing.Color.FromArgb((( int )((( byte )( 0 )))), (( int )((( byte )( 192 )))), (( int )((( byte )( 0 )))));
                lbl.Font = font;
                lbl.Width = MaxWidth;
                lbl.Click += new EventHandler(lbl_Click);
                pnl.Controls.Add(lbl);

                PictureBox pic = new PictureBox();
                pic.Name = "pic_" + sound.ID.ToString();
                pic.Left = lbl.Width;
                pic.Top = 3 ;
                pic.SizeMode = PictureBoxSizeMode.Zoom;
                pic.Height = 16 ;
                pic.Width = 16 ;

                pnl.Name = "pnl_" + sound.ID.ToString();
                pnl.Tag = sound;
                pnl.Controls.Add(pic);
                pnl.AutoSize = true ;

                 this .fpnl_Content.Controls.Add(pnl);
                Application.DoEvents();
            }
             this .btn_Download.Enabled = true ;
             this .btn_Download.ForeColor = Color.Black;
             this .btn_Download.Text = "批量下載" ;
             this .btn_Download.Focus();
        }));
    }
     else
    {
         if (IsDisposed || ! this .IsHandleCreated) return ;
         this .Invoke( new Action(() =>
        {
            MessageBox.Show( "分析網頁失敗，請檢查。" );
        }));
    }
}

/// <summary>
/// 清除字符串中HTML控制字符
/// </summary>
/// <param name="s">要清除的字符串</param>
/// <returns></returns>
public static string ClearHtml( string s)
{
     if (s == null ) return null ;
     return HtmlDecode(Regex.Replace(s, @ "(<[^>]+>)|[\r\n]" , "" , RegexOptions.IgnoreCase | RegexOptions.Singleline));
}

private static string HtmlDecodeMatchEvaluator(Match m)
{
     switch (m.Value)
    {
         case "<" : return "<" ;
         case ">" : return ">" ;
         case "&" : return "&" ;
         case """ : return "\"";
         case " " : return "\u0020" ;
         default : return m.Value;
    }
}

/// <summary>
/// HTML解碼
/// </summary>
/// <param name="s"></param>
/// <returns></returns>
private static string HtmlDecode( string s)
{
     return Regex.Replace(s, "(<)|(>)|(&)|(")|( )" , new MatchEvaluator(HtmlDecodeMatchEvaluator), RegexOptions.Singleline | RegexOptions.IgnoreCase);
}

未完待續...

做者：相信的勇氣

出處： http://www.newrain.cn/article/detail/15

本文爲博主原創文章，歡迎轉載分享但請註明出處及連接，不然將其追究法律責！

勤奮的男人和愛笑的女人,運氣通常都不會太差。