如何通过代码下载一个静态网站的所有网页

[wangjingjing390]

就是说要通过代码将一个网站上的所有网页保存到硬盘上，大家帮忙出出主意阿，分不够可以再加

[saucer]

see Vampire Bot:
http://www.informit.com/isapi/product_id~{4231CA91-D1E2-4498-8509-2A8E4FAA8FDE}/element_id~{DDB4BE48-B1C1-46A6-B2AF-43017CCF7DBB}/st~{EA7C8D03-4995-402D-B085-06E000F897B8}/session_id~{058F8A5B-CA2F-4005-A2EA-BA2E92FAF84C}/content/articlex.asp

[fgc5201314]

up && UP

[残崖孤鹰]

UP,UP,

[fgc5201314]

/// <summary>
/// 启动下载文件线程。
/// </summary>

private void StartDownLoadThreading()
{
if( this.DownLoadThread == null )
{
if( this.editUrlPre.Text.Length == 0 )
{
this.editUrlPre.Focus();
MessageBox.Show("请输入下载文件 URL。");
return;
}
if( this.editSavePath.Text.Length == 0 )
{
this.editSavePath.Focus();
MessageBox.Show("请输入下载文件保存路径。");
return;
}
try
{
this.DownLoadThread = new System.Threading.Thread( new System.Threading.ThreadStart( this.DownLoadThreading ) );
this.DownLoadThread.Start();
return;
}
catch
{
this.SetControlsStyle(0);
}
}

}

[fgc5201314]

/// <summary>
/// 下载文件线程。
/// </summary>
private void DownLoadThreading()
{
this.RequestStopDownLoadThreading = false;
int beginNumber = System.Convert.ToInt32(this.numericUpDown_begin.Value);
int endNumber = System.Convert.ToInt32(this.numericUpDown_end.Value);

this.SetControlsStyle(1);
this.DownLoadFileCount = 0;
this.DownLoadThreadingStartTime = System.DateTime.Now;
this.DownLoadThreadingSuspendTimeSpan = System.TimeSpan.Zero;
this.progressBar1.Value = this.progressBar1.Minimum = 0;
this.progressBar1.Maximum = endNumber - beginNumber + 1;

for( int i = beginNumber; i <= endNumber; i++ )
{
this.progressBar1.Value = this.DownLoadFileCount + 1;
string midurl = i.ToString( "D" + System.Convert.ToInt32(this.numericUpDown_long.Value) );
string Url = this.editUrlPre.Text + midurl + this.editUrlEnd.Text;
string FileName = System.IO.Path.GetFileName( Url );
string localDirectory = this.editSavePath.Text;
localDirectory = localDirectory.Replace('/', '\\');
if( localDirectory.EndsWith("\\") == false ) localDirectory += "\\";

string localFullName = localDirectory + FileName;
this.statusBar1.Text = Url;

try
{
if( System.IO.File.Exists(localFullName) )
{
this.DownLoadThreadingSuspendStartTime = System.DateTime.Now;
System.Windows.Forms.DialogResult dr = MessageBox.Show(this, "文件已经存在，是否覆盖：\r\n" + localFullName, "", System.Windows.Forms.MessageBoxButtons.YesNoCancel, System.Windows.Forms.MessageBoxIcon.Question, System.Windows.Forms.MessageBoxDefaultButton.Button2 );
System.DateTime cdt = System.DateTime.Now;
this.DownLoadThreadingSuspendTimeSpan += (cdt - this.DownLoadThreadingSuspendStartTime);
switch( dr )
{
case System.Windows.Forms.DialogResult.Yes:
break;
case System.Windows.Forms.DialogResult.No:
this.lbxMessage.Items.Add( "！跳过文件：" + localFullName );
continue;
case System.Windows.Forms.DialogResult.Cancel:
this.RequestStopDownLoadThreading = true;
goto Exit;
}
}
new System.Net.WebClient().DownloadFile( Url, localFullName );
this.DownLoadFileCount ++;
}
catch( System.Net.WebException wex )
{
this.lbxMessage.Items.Add( wex.Message.ToString() );
}
catch( System.Threading.ThreadAbortException taex )
{
this.lbxMessage.Items.Add( taex.Message.ToString() );
}
catch(System.Exception ex)
{
this.lbxMessage.Items.Add( ex.ToString() );
}

Exit:
if( this.RequestStopDownLoadThreading )//由用户停止文件下载任务。
{
this.lbxMessage.Items.Add( ">>用户停止。" );
break;
}

}

if( this.DownLoadThread != null )
{
this.DownLoadThreadingStopTime = System.DateTime.Now;

if( ! this.RequestStopDownLoadThreading )//完成文件下载任务（非用户干预）。
{
this.lbxMessage.Items.Add("-------------------------------------------------------");
this.lbxMessage.Items.Add(">>完成！");
this.lbxMessage.Items.Add("下载文件数量：" + this.DownLoadFileCount.ToString());
this.lbxMessage.Items.Add("总花费时间：" + (this.DownLoadThreadingStopTime - this.DownLoadThreadingStartTime).ToString());
this.lbxMessage.Items.Add("起始时间：" + this.DownLoadThreadingStartTime.ToString() + " 完成时间：" + this.DownLoadThreadingStopTime.ToString());
this.lbxMessage.Items.Add("等待时间：" + this.DownLoadThreadingSuspendTimeSpan.ToString());
this.lbxMessage.Items.Add("实际花费时间：" + ( ((this.DownLoadThreadingStopTime - this.DownLoadThreadingStartTime) - this.DownLoadThreadingSuspendTimeSpan).ToString() ));
this.lbxMessage.Items.Add("-------------------------------------------------------");
}

this.SetControlsStyle( 0 );//恢复控件风格。
System.Threading.Thread tmpThread = this.DownLoadThread;
this.DownLoadThread = null;
try
{
tmpThread.Abort();
}
catch
{
}
return;
}

}

[inethax]

up

[hlj321]

up

[jackyhzzjcn]

1、首先打开该站点的首页，将取得的文本全保存下来，并纪录下该页面的绝对路径，如/index.htm，然后判断页面的img,object等标签，根据这两面的src来取得相应的资源，一般为图片和FLASH；
2、判断<a href="XX">标签，取得地址后，到保存的地址列表中比较，如果该地址没有被下载过，那么开一个新的线程，去LOAD该新的页面，循环第一步；


简单的讲就是这样一个思路。

[速马]

我觉得应该这样比较合理:
1.所有的页面文件和资源文件的名字不要改
2.不同的资源都存放到各自的文件夹中,比如:
ROOT
Images
StyleSheet
Flash
...
aa.htm
bb.htm
...

这么安排目录而不是IE保存页面使用的目录结构,是因为我觉得这是保存一个站点,而不是保存一个单独的页面.如果一个站点真的就全是静态页面,我也会这么设计目录结构的.

而具体的下载保存就很简单了.主要是正则表达式的使用.和已下载的同名且同大小的页面就不下载,同名不同大小的就改名,等等...

[wangjingjing390]

我大概已经知道怎样保存一个网页了，但是怎样从网页中获得<a href="XX">标签以及怎样从这个标签获得超链接的方法我还不知道，请高手帮我写一下这段代码吧，谢谢了

[速马]

WebClient wc = new WebClient();
byte[] data = wc.DownloadData("http://211.83.110.75/");
string str = Encoding.Default.GetString(data);
Regex x = new Regex(" href=[\"|'](?<link>[^\"']+)[\"|'] ",RegexOptions.IgnoreCase|RegexOptions.Compiled);
MatchCollection mc = x.Matches(str);
foreach(Match m in mc)
{
Console.WriteLine(m.Result("${link}"));
}