62,072
社区成员
发帖
与我相关
我的任务
分享
public static string getWebContent(string Url, string referer, string ch)
{
string strResult = "";
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
request.Timeout = 30000;//设置连接超时时间
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; ccdotnet ; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Referer = referer;//设置标头
request.AllowAutoRedirect = true;
//不用缓存
//request.Headers.Set("Pragma", "no-cache");//不用缓存的话,对方服务器返回500错误
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream streamReceive = response.GetResponseStream();
Encoding encoding = Encoding.GetEncoding(ch);
StreamReader streamReader = new StreamReader(streamReceive, encoding);
strResult = streamReader.ReadToEnd();
}
catch (Exception ex) { HttpContext.Current.Response.Write(ex.ToString()); }
return strResult;
}
//抓取的地址http://www.qidian.com/BookReader/180029.aspx
//编码设置为gb2312,目标网址编码也是gb2312,但是返回结果是乱码,有什么解决的办法没?
/// <summary>
/// 得到页面HTML代码(PagePath地址为URL绝对地址)
/// </summary>
/// <param name="PagePath">页面地址(PagePath地址为URL绝对地址)</param>
/// <returns></returns>
public string GetUrlHtml(string PagePath)
{
string BackHtml;
Uri uri = new Uri(PagePath);
WebRequest req = WebRequest.Create(uri);
WebResponse resp = req.GetResponse();
Stream str = resp.GetResponseStream();
StreamReader sr = new StreamReader(str, System.Text.Encoding.UTF8);
return BackHtml = sr.ReadToEnd();
}