全部

如何把互联网上指定网站的全部内容保存在本地?

bbzy 2008-04-14 10:19:21

如何把互联网上指定网站的全部内容保存在本地?
我要把N个网站的全部内容拉下来,然后进行搜索.比对

...全文

1222 点赞收藏 29

写回复

29 条回复

切换为时间正序

当前发帖距今超过3年，不再开放新的回复

发表回复

bbzy 2008-05-16

我是想搜索其中的违法关键字

回复赞

lymyx 2008-05-06

如何把互联网上指定网站的全部内容保存在本地?
//我感觉不大好办，程序在服务器上，就算楼主用软件下载了某些网页，但也是经过解析后的。

回复赞

myccn 2008-05-06

是啊！动态网页都是经过解释的

回复赞

mailto520 2008-05-05

FlashGet

回复赞

qseawd520 2008-05-05

对新手来说,非常不容易的!

回复赞

一把编程的菜刀 2008-05-05

。。。。采集内容可以，你要下载程序没门！

回复赞

bbzy 2008-05-05

请问,如何用asp实现网站采集(下载)某个网站?

回复赞

gingerkang 2008-04-30

wget -r

回复赞

ldlazl 2008-04-30

webzip

回复赞

bbzy 2008-04-30

help me

回复赞

snlixing 2008-04-28

网站整站下载器

回复赞

bbzy 2008-04-28

谢谢
大家有asp的吗?

回复赞

bbzy 2008-04-22

[Quote=引用 14 楼 live_7sky 的回复:]
网站采集；

如果是我，我会自己写一个：
以前把综艺大观，性福五月天，等网站内容一夜之间全部下载下来了。
＝＝＝＝＝＝例子＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝
/// <summary>
/// 保存网页
/// </summary>
/// <param name="FILE_NAME">文件的路径 </param>
/// <param name="data">数据 </param>
public void TextToFile(string…
[/Quote]

非常感谢,
在下愚昧,请问您用的是VC吗?
该如何安装运行环境呢?

回复赞

live_7sky 2008-04-22

不好意思，我用的是 .NET(C#)
大概原理是一样的饿

回复赞

live_7sky 2008-04-17

网站采集；

如果是我，我会自己写一个：
以前把综艺大观，性福五月天，等网站内容一夜之间全部下载下来了。
＝＝＝＝＝＝例子＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝＝
/// <summary>
/// 保存网页
/// </summary>
/// <param name="FILE_NAME">文件的路径</param>
/// <param name="data">数据</param>
public void TextToFile(string FILE_NAME, string data)
{
if (File.Exists(FILE_NAME))
{
return;
}
using (StreamWriter sw = File.CreateText(FILE_NAME))
{
sw.Write(data);
sw.Close();
}
}
/// <summary>
/// 下载文件
/// </summary>
/// <param name="PageUrl">网址</param>
/// <param name="filename">保存文件路径</param>
public void DownFile(string PageUrl, string filename)
{
if (!Directory.Exists(filename))
{
Directory.CreateDirectory(filename);
}
string path = PageUrl.Substring(PageUrl.LastIndexOf("/") + 1);
string dirname = filename + "\\" + path;

if (File.Exists(dirname))
{
return;
}
else
{
try
{
WebClient wc = new WebClient();
WebProxy wp = new WebProxy(al[proxy].ToString(), true);
wc.Proxy = wp;
wc.DownloadFile(PageUrl, dirname);
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
//文件未找到--跳出
//MessageBox.Show(ex.ToString());
return;
}
else
{
//换代理 IP
//MessageBox.Show(ex.ToString());
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
DownFile(PageUrl, filename);
} } }
}

public string ToServer(string PageUrl)
{
string responseFromServer = "";
try
{
while (1 == 1)
{
WebRequest request = WebRequest.Create(PageUrl);
WebProxy wp = new WebProxy(al[proxy].ToString(), true);
request.Proxy = wp;
request.Timeout = 1000 * 45;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream dataStream = response.GetResponseStream();
StreamReader reader=null;
reader = new StreamReader(dataStream, System.Text.Encoding.Default);
responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
if (responseFromServer.Contains("refresh") || responseFromServer == "")
{
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
//ToServer(PageUrl);
}
else
{
break;
}
}
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
responseFromServer = "";
}
else
{
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
ToServer(PageUrl); }
}
return responseFromServer;
}
/// <summary>
/// 保存XML 文件
/// </summary>
public void SaveXmls()
{
string pathxml = "";
foreach (Class1 c in cls)
{
Class1 s = c;
pathxml = s.address;

if (!File.Exists(pathxml))
{
XmlSerializer xs = new XmlSerializer(typeof(Class1));
Stream stream = new FileStream(pathxml, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
xs.Serialize(stream, s);
stream.Close();
}

}
}
/// <summary>
/// 移除HTMl 标记
/// </summary>
public static string Remove(string Html)
{
string regesstr = "<.*?>";
return Regex.Replace(Html, regesstr, string.Empty, RegexOptions.IgnoreCase);
}
public static string FilterScript(string content)
{
string regexstr = @"<(script)[^>]*>(\s*|.)*</\1>";
return Regex.Replace(content,regexstr,string.Empty,RegexOptions.IgnoreCase);

}
public void HtmlSource(string urlpri)
{
//要写入的文件路径
filename = "E:\\综艺大观2\\magazine.html";

if (!Directory.Exists("E:\\综艺大观2"))
{
Directory.CreateDirectory("E:\\综艺大观2");
}
if (File.Exists(filename))
{
responseFromServer=FileToText(filename); //存在
}
else
{
responseFromServer = ToServer(urlpri); //不存在
}
sum++;
if (responseFromServer != "")
{
//分析内容
TextToFile(filename,responseFromServer);
MatchCollection mc = Regex.Matches(responseFromServer, @"href=""/magazine/(.*)""><b>(.*)</b>", RegexOptions.IgnoreCase);
foreach (Match m in mc)
{
newurl = m.Groups[1].Value;
dirname = m.Groups[2].Value;

int key = ++keyi;
//sql = insertdl + key + ",'" + dirname + "')";
//SaveSqls(sql);
// SqlHelper.ExecuteNonQuery(SqlHelper.sqlstr, CommandType.StoredProcedure, "proc_dl", null);
getparmdl(key, dirname);
cururl = urlpri + newurl;
curdir = "E:\\综艺大观2\\" + dirname;
one(cururl, curdir,key);
}
SaveXmls();
// Executesql();

this.textBox1.Text = sum.ToString();
MessageBox.Show("采集成功!");
}
}
private void btnOK_Click(object sender, EventArgs e)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
HtmlSource("http://www.zydg.net/magazine/");
}

回复赞