关于WebClient.HttpRequest,XMLHTTP,WebBrowser与IE访问的区别..

zyug 2012-03-19 04:12:31

现在有一个需求。比如搜索
准备怀孕要做哪些检查在百度中我公司网站的排名...
需求挺简单的
~~~
大家可以测试一下.使用各种方式

如WebClient,HttpRequest,XMLhttp,WebBrowser去请求百度，取到的HTML源码不尽相同！！！
比如请求
http://www.baidu.com/s?wd=准备怀孕要做哪些检查
将出现以下问题

问题1 : HTML源码不同，导致排位不同，
问题2 : 如果某一种方式请求过多，会出现验证码，但是这时候用IE去访问，是不需要验证码的。

我的分析是
百度可以区分来自浏览器访问与各种Com组件或是代码方式的请求

但是：
我使用各种HTTP访问监视工具，如
Fiddler记录下IE的请求时发送的头信息，然后用HttpRequest去一一对应·还是出现这样的问题

下面发代码！
使用HttpRequest自定义方式



        public static string DownLoadHtml(string url)

        {

            try

            {

                CookieContainer c = new CookieContainer();

                Uri u = new Uri("http://www.baidu.com");

                CookieCollection ccs = new CookieCollection();

                ccs.Add(new Cookie("BAIDUID", System.Guid.NewGuid().ToString().ToUpper().Replace("-", "") + ":FG=1"));

                c.Add(u, ccs);



                HttpWebRequest r = (HttpWebRequest)WebRequest.Create(url);

                

                //r.Headers["Cache-Control"] = "no-cache";

                //r.Headers["Pragma"] = "no-cache";

                r.UserAgent = @"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET4.0C; .NET4.0E)";

                r.Accept = @"*/*";

                r.Host = "www.baidu.com";

                r.Headers["Accept-Encoding"] = "gzip, deflate";

                r.Headers["Accept-Language"] = "zh-cn";

                r.Method = "get";

                

                //r.Referer = "http://www.baidu.com";

                r.CookieContainer = c;

                r.AllowAutoRedirect = true;

                HttpWebResponse rep = (HttpWebResponse)r.GetResponse();

                

                Stream receiveStream = rep.GetResponseStream();

      

                string data = string.Empty; 

                string sResponseHeader = rep.Headers["Content-Encoding"]; 

                if (!string.IsNullOrEmpty(sResponseHeader))

                {

                    if (sResponseHeader.ToLower().Contains("gzip"))

                    {

                        byte[] b = DecompressGzip(receiveStream);

                        data = System.Text.Encoding.GetEncoding("gb2312").GetString(b);

                    }

                    else if (sResponseHeader.ToLower().Contains("deflate"))

                    {

                        byte[] b = DecompressDeflate(receiveStream);

                        data = System.Text.Encoding.GetEncoding("gb2312").GetString(b); 

                    }

                }

                //

                RegexOptions options = RegexOptions.None | RegexOptions.Singleline;

                Regex regex = new Regex("<title>(?<title>.*?)</title>", options);

                MatchCollection matches = regex.Matches(data);

                if (matches != null)

                {

                    foreach (Match m in matches)

                    {

                        if (m.Groups["title"].Value.IndexOf("您的访问出错了") >= 0)

                        {

                            data = "err:访问出错，需填写验证码";

                            break;

                        }

                    }

                }

                return data;

            }

            catch(Exception er)

            {

                return "err:"+er.Message;

            }

        }



        private static byte[] DecompressGzip(Stream streamInput)

        {

            Stream streamOutput = new MemoryStream();

            int iOutputLength = 0;

            try

            {

                byte[] readBuffer = new byte[4096];

                /// read from input stream and write to gzip stream

                using (GZipStream streamGZip = new GZipStream(streamInput, CompressionMode.Decompress))

                {

                    int i;

                    while ((i = streamGZip.Read(readBuffer, 0, readBuffer.Length)) != 0)

                    {

                        streamOutput.Write(readBuffer, 0, i);

                        iOutputLength = iOutputLength + i;

                    }

                }

            }

            catch 

            {

                // todo: handle exception

            }

            /// read uncompressed data from output stream into a byte array

            byte[] buffer = new byte[iOutputLength];

            streamOutput.Position = 0;

            streamOutput.Read(buffer, 0, buffer.Length);

            return buffer;

        }



        private static byte[] DecompressDeflate(Stream streamInput)

        {

            Stream streamOutput = new MemoryStream();

            int iOutputLength = 0;

            try

            {

                byte[] readBuffer = new byte[4096];

                /// read from input stream and write to gzip stream

                using (DeflateStream streamGZip = new DeflateStream(streamInput, CompressionMode.Decompress))

                {

                    int i;

                    while ((i = streamGZip.Read(readBuffer, 0, readBuffer.Length)) != 0)

                    {

                        streamOutput.Write(readBuffer, 0, i);

                        iOutputLength = iOutputLength + i;

                    }

                }

            }

            catch 

            {

                // todo: handle exception

            }

            /// read uncompressed data from output stream into a byte array

            byte[] buffer = new byte[iOutputLength];

            streamOutput.Position = 0;

            streamOutput.Read(buffer, 0, buffer.Length);

            return buffer;

        }

使用XMLHTTP方式



        public static void GetHtml(WordShowModel wsm)

        {

            bool hasFailed = false;

            string text = "";

            try

            {

            

                Encoding gb2312 = Encoding.GetEncoding("gb2312");

                string m = System.Web.HttpUtility.UrlEncode(wsm.showword, gb2312);

                var url = string.Format(@"http://www.baidu.com/s?wd={0}&rsv_bp=0&rsv_spt=3&inputT={1}", m, (new Random()).Next(90000));

                wsm.errmsg = "正在搜索";

                MSXML2.ServerXMLHTTP wc = new MSXML2.ServerXMLHTTP();

                //wc.setRequestHeader("Content-Type", "text/html");

                //wc.setRequestHeader("Referer", "www.baidu.com");

                //wc.setRequestHeader("Cookie", "BAIDUID=" + System.Guid.NewGuid().ToString().ToUpper().Replace("-", "") + ":FG=1");

                wc.open("GET",url, false, null, null);

                wc.send("");

                if (wc.readyState == 4)

                {

                    ADODB.StreamClass stream = new ADODB.StreamClass();

                    stream.Type = ADODB.StreamTypeEnum.adTypeBinary;

                    stream.Mode =  ADODB.ConnectModeEnum.adModeReadWrite;

                    stream.Open();

                    stream.Write(wc.responseBody);

                    stream.Position = 0;

                    stream.Type =  ADODB.StreamTypeEnum.adTypeText;

                    stream.Charset = "gb2312";

                    text = stream.ReadText();

                    stream.Close();

                    stream = null;

                }

                else

                {

                    text = "";

                }

                wsm.HTML = text;

            }

            catch (Exception exception1)

            {

                wsm.errmsg = exception1.Message;

                hasFailed = true;

            }



      }

WebClient与WebBrowser就不发了~~~大同小异........

神呀我要疯了~~~~~~

百度你早点灭亡吧

...全文

269 16 打赏收藏转发到动态举报

写回复

用AI写文章

16 条回复

切换为时间正序

请发表友善的回复…

发表回复

anzhiqiang_touzi 2012-05-17

打赏
举报

[Quote=引用 14 楼的回复:]
百度肯定有反扒软件的

这个东西是算法决定的

得先了解怎么对付反扒软件。。。
[/Quote]

噬魂之主NO1 2012-05-17

打赏
举报

额。。。或许是因为百度为公开，也有能是各个浏览器的兼容性不同所导致的

lqkankan 2012-05-17

打赏
举报

百度肯定有反扒软件的

这个东西是算法决定的

得先了解怎么对付反扒软件。。。

marsmitchell 2012-05-17

打赏
举报

baidu很挫，登录功能我还没有实现，谁知道给发个代码啊。

zyug 2012-03-22

打赏
举报

7楼~
关于javascript，我有考虑
是先返回html字符串再去根据字符串去请求js文件，再执行js代码~并且js代码也在本地ie有缓存~~

如果说百度保持了你的连接几秒，等待你去请求js,然后验证你的机器请求还是人工请求，然后根据结果发送redirect命令,它的开销会很大~~~,根据我测试结果~应该不是不么回事~~~

是不是还有什么没公开的~~~~~~~~~~