C# HtmlAgilityPack 爬这个网页取不到值不知道为什么
http://www.youdaili.net/Daili/guonei/3537.html
这个页不取不到值,其他的页试过可以,不知道为什么,大侠们帮我看看
public static List<string> GetIPs()
{
List<string> IpList = new List<string>();
string suburl = "http://www.youdaili.net/Daili/guonei/3537.html";
HtmlDocument _doc = getip(suburl);
HtmlNodeCollection hrefs = _doc.DocumentNode.SelectNodes("/html");//ul/li/a[@class='cont_font']
if (hrefs == null)
return null;
foreach (HtmlNode href in hrefs)
{
string ips = href.InnerText;//58.220.2.156:80@HTTP#【匿】江苏省扬州市 电信<br />
IpList.Add(ips);
}
return IpList;
}
private static HtmlDocument getip(string url)
{
HtmlAgilityPack.HtmlWeb hw = new HtmlAgilityPack.HtmlWeb();//用的HtmlAgilityPack
hw.AutoDetectEncoding = false;
//现在下面的代码很重要。
HttpWebRequest req;
req = WebRequest.Create(new Uri(url)) as HttpWebRequest;
req.Method = "GET";
HttpWebResponse rs = (HttpWebResponse)req.GetResponse();
System.IO.StreamReader sr = new StreamReader(rs.GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
try
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(sr.ReadToEnd());
return doc;
}
catch (Exception e)
{
Console.WriteLine(e.Message.ToString());
Console.WriteLine(e.StackTrace);
return null;
}
finally {
sr.Close();
rs.Close();
}
}