8,834
社区成员
发帖
与我相关
我的任务
分享c#如何利用多线程获取大量的网页信息？网页的有些信息需要进过模拟点击才会出来。
以下代码是我在利用webbrowser控件获取几个航司页面信息的多线程代码，因为在获取了网页的信息之后还要进行模拟按钮点击的操作然后再分析数据保存到数据库中，（这一部分代码我就不放出来了）但是我现在只能多开webbrowser控件来获取，而且开了两三个之后，再多开，后面的一部分都不会运行了。有谁做过这种获取大量网页信息的？能否提供一下方法？
public delegate void threaddel(WebBrowser web);
/// <summary>
/// 建立委托
/// </summary>
/// <param name="state"></param>
private void TRun(object state)
{
threaddel md1 = new threaddel(NanFang);
int s = (int)state;
switch (s)
{
case 0:
//判断控件是否在本线程内
//if (!this.webBrowser4.InvokeRequired)
Thread.Sleep(2000);
Invoke(md1, webBrowser2);
break;
case 1:
Thread.Sleep(2000);
Invoke(md1, webBrowser3);
break;
case 2:
Thread.Sleep(2000);
Invoke(md1, webBrowser4);
break;
default: break;
}
bool loading = true; // 该变量表示网页是否正在加载.
public void Threadtest()
{
string Splace = cmb_FromCity.Text.ToString().Split('|')[1];
string Rpalce = cmb_ToCity.Text.ToString().Split('|')[1];
for (int i = 0; i < 4; i++)
{
string Date = dateTime_StartDay.Value.AddDays(i).ToString("yyyy-MM-dd");
loading = true;
string NF = NFurl(Splace, Rpalce, Date);
if (i == 0)
{
webBrowser2.Url = new Uri(NF);
}
if (i == 1)
{
webBrowser3.Url = new Uri(NF);
}
if (i == 2)
{
webBrowser4.Url= new Uri(NF);
}
Thread td = new Thread(new ThreadStart(() => { TRun(i); }));
//以下两种方法也可以
//Thread td = new Thread(new ParameterizedThreadStart(TRun));
//ThreadPool.QueueUserWorkItem(new WaitCallback(TRun), i);
td.Start();
}
}
...全文
请发表友善的回复…
发表回复
edlyly 2015-07-13
- 打赏
- 举报
帖子快要沉了，还有谁知道怎么做吗
newtee 2015-07-10
- 打赏
- 举报
但是返回来的我要怎么获取呢？
[/quote]了解下http 和ajax
edlyly 2015-07-10
- 打赏
- 举报
还有谁做过类似的吗？能解决吗？谢谢各位大牛了
edlyly 2015-07-10
- 打赏
- 举报
但是返回来的我要怎么获取呢？
[/quote]了解下http 和ajax[/quote]
这问题我已经就纠结好一段时间，如果了解一下就能解决就好了。。。就是研究不出怎么做
newtee 2015-07-09
- 打赏
- 举报
你要的信息都在这edlyly 2015-07-09
- 打赏
- 举报
@zhuankeshumo
你可能没理解我的意思，我给你看一下要获取的地址吧，像这个http://b2c.csair.com/B2C40/modules/bookingnew/main/flightSelectDirect.html?t=S&c1=PEK&c2=SZX&d1=2015-07-10&at=1&ct=0&it=0
每个价格下面有个按钮需要点击才能得到下面，不点击是没有数据的，HttpWebRequest是一下子返回所有的数据，如何去进行模拟点击操作获取下面的数据呢？
newtee 2015-07-09
- 打赏
- 举报
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using System.IO;
using System.Diagnostics;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
namespace App
{
class Program
{
static void Main(string[] args)
{
ServicePointManager.DefaultConnectionLimit = 512;
Stopwatch watcher = new Stopwatch();
CookieContainer cookie = new CookieContainer();
watcher.Start();
var task1 = HttpHelper.CreatePostHttpResponse(@"http://api.map.baidu.com/geocoder?address=%E5%90%B4%E4%B8%AD%E5%8C%BA&city=%E8%8B%8F%E5%B7%9E%E5%B8%82&output=json&key=28bcdd84fae25699606ffad27f8da77b",
null,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16",
Encoding.UTF8,
cookie);
var task2 = HttpHelper.CreatePostHttpResponse(@"http://api.map.baidu.com/geocoder?address=%E9%AB%98%E6%96%B0%E5%8C%BA&city=%E8%8B%8F%E5%B7%9E%E5%B8%82&output=json&key=28bcdd84fae25699606ffad27f8da77b",
null,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16",
Encoding.UTF8,
cookie);
var task3 = HttpHelper.CreatePostHttpResponse(@"http://api.map.baidu.com/geocoder?address=%E6%B2%A7%E6%B5%AA&city=%E8%8B%8F%E5%B7%9E%E5%B8%82&output=json&key=28bcdd84fae25699606ffad27f8da77b",
null,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16",
Encoding.UTF8, cookie);
var task4 = HttpHelper.CreatePostHttpResponse(@"http://api.map.baidu.com/geocoder?address=%E5%9B%AD%E5%8C%BA&city=%E8%8B%8F%E5%B7%9E%E5%B8%82&output=json&key=28bcdd84fae25699606ffad27f8da77b",
null,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16",
Encoding.UTF8, cookie);
var task5 = HttpHelper.CreatePostHttpResponse(@"http://api.map.baidu.com/geocoder?address=%E5%90%B4%E6%B1%9F%E5%8C%BA&city=%E8%8B%8F%E5%B7%9E%E5%B8%82&output=json&key=28bcdd84fae25699606ffad27f8da77b",
null,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16",
Encoding.UTF8,
cookie);
string content = "";
var lnglat1 = Task.Factory.StartNew<string>(() =>
{
using (Stream stream = task1.Result.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stream))
{
content = sr.ReadToEnd();
JObject obj = (JObject)JsonConvert.DeserializeObject(content);
var lnglat = obj["result"]["location"];
return lnglat.ToString();
}
}
});
var lnglat2 = Task.Factory.StartNew<string>(() =>
{
using (Stream stream = task2.Result.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stream))
{
content = sr.ReadToEnd();
JObject obj = (JObject)JsonConvert.DeserializeObject(content);
var lnglat = obj["result"]["location"];
return lnglat.ToString();
}
}
});
var lnglat3 = Task.Factory.StartNew<string>(() =>
{
using (Stream stream = task3.Result.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stream))
{
content = sr.ReadToEnd();
JObject obj = (JObject)JsonConvert.DeserializeObject(content);
var lnglat = obj["result"]["location"];
return lnglat.ToString();
}
}
});
var lnglat4 = Task.Factory.StartNew<string>(() =>
{
using (Stream stream = task4.Result.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stream))
{
content = sr.ReadToEnd();
JObject obj = (JObject)JsonConvert.DeserializeObject(content);
var lnglat = obj["result"]["location"];
return lnglat.ToString();
}
}
});
var lnglat5 = Task.Factory.StartNew<string>(() =>
{
using (Stream stream = task5.Result.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stream))
{
content = sr.ReadToEnd();
JObject obj = (JObject)JsonConvert.DeserializeObject(content);
var lnglat = obj["result"]["location"];
return lnglat.ToString();
}
}
});
Console.WriteLine(lnglat1.Result);
Console.WriteLine(lnglat2.Result);
Console.WriteLine(lnglat3.Result);
Console.WriteLine(lnglat4.Result);
Console.WriteLine(lnglat5.Result);
watcher.Stop();
Console.WriteLine(watcher.ElapsedMilliseconds.ToString());
Console.ReadKey();
}
}
}
newtee 2015-07-09
- 打赏
- 举报
点击就是请求服务器啊
给你个例子
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
namespace App
{
class HttpHelper
{
private static readonly string DefaultUserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16";
/// <summary>
/// 创建POST方式的HTTP请求
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="parameters">随同请求POST的参数名称及参数值字典</param>
/// <param name="userAgent">请求的客户端浏览器信息，可以为空</param>
/// <param name="requestEncoding">发送HTTP请求时所用的编码</param>
/// <param name="cookies">随同HTTP请求发送的Cookie信息，如果不需要身份验证可以为空</param>
/// <returns></returns>
public static Task<WebResponse> CreatePostHttpResponse(string url, IDictionary<string, string> parameters, string userAgent, Encoding requestEncoding, CookieContainer cookieContainer)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
if (requestEncoding == null)
{
throw new ArgumentNullException("requestEncoding");
}
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
if (!string.IsNullOrEmpty(userAgent))
{
request.UserAgent = userAgent;
}
else
{
request.UserAgent = DefaultUserAgent;
}
if (cookieContainer == null)
{
request.CookieContainer = new CookieContainer();
}
else
{
request.CookieContainer = cookieContainer;
}
//如果需要POST数据
if (!(parameters == null || parameters.Count == 0))
{
StringBuilder buffer = new StringBuilder();
int i = 0;
foreach (string key in parameters.Keys)
{
if (i > 0)
{
buffer.AppendFormat("&{0}={1}", key, parameters[key]);
}
else
{
buffer.AppendFormat("{0}={1}", key, parameters[key]);
}
i++;
}
byte[] data = requestEncoding.GetBytes(buffer.ToString());
var task = Task.Factory.FromAsync<Stream>(request.BeginGetRequestStream, request.EndGetRequestStream, request, TaskCreationOptions.None); //等待任务完成
task.Wait(); //执行完本任务后再连续执行写入留和返回response对象 '
using (Stream stream = task.Result)//如果上面没有等待任务完成那一句，在这里直接获取结果也是可以的
{
stream.Write(data, 0, data.Length);
}
}
return Task.Factory.FromAsync<WebResponse>(request.BeginGetResponse, request.EndGetResponse, request, TaskCreationOptions.None);
}
/// <summary>
/// 创建GET方式的HTTP请求
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="timeout">请求的超时时间</param>
/// <param name="userAgent">请求的客户端浏览器信息，可以为空</param>
/// <param name="cookies">随同HTTP请求发送的Cookie信息，如果不需要身份验证可以为空</param>
/// <returns></returns>
public static Task<WebResponse> CreateGetHttpResponse(string url, string userAgent, CookieContainer cookieContainer)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
HttpWebRequest request = WebRequest.Create(new Uri(url)) as HttpWebRequest;
request.Method = "GET";
request.UserAgent = DefaultUserAgent;
if (!string.IsNullOrEmpty(userAgent))
{
request.UserAgent = userAgent;
}
if (cookieContainer == null)
{
request.CookieContainer = new CookieContainer();
}
else
{
request.CookieContainer = cookieContainer;
}
return Task.Factory.FromAsync<WebResponse>(request.BeginGetResponse, request.EndGetResponse, request, TaskCreationOptions.None);
}
}
}
edlyly 2015-07-09
- 打赏
- 举报
有一部分内容需要模拟点击才能得到，这样可以用httpwebrequest吗？
edlyly 2015-07-09
- 打赏
- 举报
这个东西是在哪里配置？
在form里面配置出现这种错误。而且我不是不能多开，而且多开之后获取数据不能继续下去。edlyly 2015-07-09
- 打赏
- 举报
但是返回来的我要怎么获取呢？
edlyly 2015-07-09
- 打赏
- 举报
看到了
edlyly 2015-07-09
- 打赏
- 举报
这是用啥浏览器看的，我怎么找不到。。。
newtee 2015-07-08
- 打赏
- 举报
不能多开是因为 System.Net.ServicePointManager.DefaultConnectionLimit默认是2
把 设置为System.Net.ServicePointManager.DefaultConnectionLimit = 512; 就可以了
最好用httpwebrequest
把 设置为System.Net.ServicePointManager.DefaultConnectionLimit = 512; 就可以了
最好用httpwebrequest
edlyly 2015-07-08
- 打赏
- 举报
有大牛提示一下吗
