13,347
社区成员
发帖
与我相关
我的任务
分享采集反爬虫网站，采集不到内容！跪求大神指点迷津！
//以下是部分代码
List<string> Weburllist = new List<string>();
List<string> Weburllistzx = new List<string>();
StringBuilder weburlSB = new StringBuilder();
bool IsGenxin = false;
MatchCollection mcexplain = Regex.Matches(sjurlDR["LinkContent"].ToString(),
@"(?<=\$-\$).*?(?=\$_\$)", RegexOptions.Singleline);
foreach (Match m in mcexplain)
{
Weburllist.Add(m.Value);
}
System.Net.WebRequest newswebrequest = System.Net.WebRequest.Create(sjurlDR
["LinkUrl"].ToString());
Uri uri = new Uri(sjurlDR["LinkUrl"].ToString());
SetHeaderValue(newswebrequest.Headers, "Host", uri.Host);
SetHeaderValue(newswebrequest.Headers, "UserAgent", "Mozilla/5.0 (Windows NT
6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0");
SetHeaderValue(newswebrequest.Headers, "Accept", "text/html,application/xhtml
+xml,application/xml;q=0.9,*/*;q=0.8");
SetHeaderValue(newswebrequest.Headers, "Accept-Language", "zh-CN,zh;q=0.8,en-
US;q=0.5,en;q=0.3");
SetHeaderValue(newswebrequest.Headers, "Accept-Encoding", "gzip, deflate,
sdch");
SetHeaderValue(newswebrequest.Headers, "Cookie:",
"gscu_792856215=62888640q5c56420; _gscbrs_792856215=1");
SetHeaderValue(newswebrequest.Headers, "Connection", "Keep-Alive");
SetHeaderValue(newswebrequest.Headers, "Cache-Control", "max-age=0");
//newswebrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate,
sdch");
//newswebrequest.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-
CN,zh;q=0.8");
//newswebrequest.Headers.Add(HttpRequestHeader.CacheControl, "max-age=0");
//SetHeaderValue(newswebrequest.Headers, "Accept", "text/html,application/xhtml
+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
//SetHeaderValue(newswebrequest.Headers, "Connection", "Keep-Alive");
//newswebrequest.Headers.Add(HttpRequestHeader.Cookie,
"_gscu_792856215=62888640q5c56420; _gscbrs_792856215=1");
//SetHeaderValue(newswebrequest.Headers, "Host", "zjks.com");
//SetHeaderValue(newswebrequest.Headers, "UserAgent", "Mozilla/5.0 (Windows NT
6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36");
System.Net.WebResponse newswebresponse = newswebrequest.GetResponse();
System.IO.Stream newsstream = newswebresponse.GetResponseStream();
System.IO.StreamReader sr = new StreamReader(newsstream,
System.Text.Encoding.UTF8);
string ProductionContent = string.Empty;
ProductionContent = sr.ReadToEnd();
sr.Close();
Regex reg = new Regex(@"http(s)?://([\w-]+\.)+[\w-]+/?");
string wangzhanyuming = reg.Match(sjurlDR["LinkUrl"].ToString(), 0).Value;
MatchCollection mc = Regex.Matches(ProductionContent.Replace("href=\"/", "href=
\"" + wangzhanyuming).Replace("href='/", "href='" + wangzhanyuming).Replace("href=/", "href=" +
wangzhanyuming).Replace("href=\"./", "href=\"" + wangzhanyuming), @"<[aA][^>]* href=[^>]*>",
RegexOptions.Singleline);
int Index = 1;
foreach (Match m in mc)
{
MatchCollection mc1 = Regex.Matches(m.Value.Replace("\"", "'"), @"[a-zA-
z]+://[^']*", RegexOptions.Singleline);
if (mc1.Count > 0)
{
foreach (Match m1 in mc1)
{
string linkurlstr = string.Empty;
linkurlstr = m1.Value.Replace("\"", "").Replace("'", "").Replace
(">", "").Replace(";", "");
weburlSB.Append("$-$");
weburlSB.Append(linkurlstr);
weburlSB.Append("$_$");
if (!Weburllist.Contains(linkurlstr) && !Weburllistzx.Contains
(linkurlstr))
{
IsGenxin = true;
Weburllistzx.Add(linkurlstr);
linkSb.AppendFormat("{0}<br/>", linkurlstr);
}
}
}
else
{
if (m.Value.IndexOf("javascript") == -1)
{
string amstr = string.Empty;
string wangzhanxiangduilujin = string.Empty;
wangzhanxiangduilujin = sjurlDR["LinkUrl"].ToString().Substring(0,
sjurlDR["LinkUrl"].ToString().LastIndexOf("/") + 1);
amstr = m.Value.Replace("href=\"", "href=\"" +
wangzhanxiangduilujin).Replace("href='", "href='" + wangzhanxiangduilujin);
MatchCollection mc11 = Regex.Matches(amstr, @"[a-zA-z]+://[^\s]*",
RegexOptions.Singleline);
foreach (Match m1 in mc11)
{
string linkurlstr = string.Empty;
linkurlstr = m1.Value.Replace("\"", "").Replace("'",
"").Replace(">", "").Replace(";", "");
weburlSB.Append("$-$");
weburlSB.Append(linkurlstr);
weburlSB.Append("$_$");
if (!Weburllist.Contains(linkurlstr) && !Weburllistzx.Contains
(linkurlstr))
{
IsGenxin = true;
Weburllistzx.Add(linkurlstr);
linkSb.AppendFormat("{0}<br/>", linkurlstr);
}
}
}
}
Index++;
}
System.Threading.Thread.Sleep(1000);
if (IsGenxin)
{
originlinksInfo oinfo = new originlinksInfo();
oinfo = originlinksLogic.Get(int.Parse(sjurlDR["ID"].ToString()));
oinfo.LinkContentnext = oinfo.LinkContent;
oinfo.LinkContent = weburlSB.ToString();
originlinksLogic.Update(oinfo);
System.Threading.Thread.Sleep(2000);
}
//如http://www.zjks.com/，这个网站总是采集失败，在这句代码
System.Net.WebResponse newswebresponse = newswebrequest.GetResponse();//这里在采集时总是跳出
...全文
请发表友善的回复…
发表回复
xdashewan 2016-05-12
- 打赏
- 举报
500这个一般解释是服务器错误，这个就很难说，可能真是你的请求导致对方服务器出现错误，也可能是对方的反爬策略是这样设置的。首先你要确保你提交的请求的参数必须足够，比如如果某个页面必须要登录，那你就得带上登录后的cookie信息，这些都可以从抓包工具里看到。其次对HTTP协议来说，服务器可用来识别客户端的信息并不是很多，主要集中于IP和UserAgent还有你的实行频率，所以你也可以从这几个点入手逐一去试，比如你怀疑对方是否封锁你的ip，你可以用正常浏览器打开看看能不能访问到。逐一排查对方可能阻止你访问的途径和方法，最后得出对策
nightmareYan 2016-05-12
- 打赏
- 举报
异常内容发出来看看[/quote]
异常是报 500
xdashewan 2016-05-12
- 打赏
- 举报
异常内容发出来看看
nightmareYan 2016-05-12
- 打赏
- 举报
fiddler 没用过！
nightmareYan 2016-05-12
- 打赏
- 举报
这样的 做了一个异常处理 但是每次运行到System.Net.WebResponse newswebresponse = newswebrequest.GetResponse();就会跳到异常
Justin-Liu 2016-05-12
- 打赏
- 举报
跳出是什么意思，报错了吧
threenewbee 2016-05-11
- 打赏
- 举报
用fiddler调试下，看你的程序和浏览器的有什么不同。
