c# HttpWebRequest多线程获取网页数据，提示操作超时

toolc2013 2013-02-22 07:45:51

c# HttpWebRequest多线程获取网页数据，把网址放到线程池中，大概打开20多个网页后就会提示“操作超时”或者“连接已关闭”，求教如何解决？以下是部分代码：

//下面是线程池，把网址作为参数传给线程
for (int i = 0; i < adr.Count; i++)
{
OddsThreadParameters tpNum = new OddsThreadParameters { num = i ,singleItem=singleBaseData[i]};
ThreadPool.SetMaxThreads(5, 5);
ThreadPool.QueueUserWorkItem(new WaitCallback(SingleT), tpNum);
}
//下面是SingleT函数的主要内容，打开网页，就是这个地方出错
System.Net.ServicePointManager.DefaultConnectionLimit = 100;
string pagedata_detail = string.Empty;
for (int reGet = 0; reGet < 3; reGet++)
{
HttpWebRequest wrq_detail = HttpWebRequest.Create(adr[tpNum.num]) as HttpWebRequest;
wrq_detail.Method = "GET";
wrq_detail.ProtocolVersion = HttpVersion.Version10;
wrq_detail.KeepAlive = false;
wrq_detail.Timeout = 20000;
try
{
pagedata_detail = GetWebDataWithZipJuge(wrq_detail);
}
catch (System.Exception ex)
{
MessageBox.Show(ex.Message);
}
}
//GetWebDataWithZipJuge函数用来返回网页string
public string GetWebDataWithZipJuge(HttpWebRequest wr)
{
//下面方法用来判断网页源码是否压缩，以获取数据
System.Net.ServicePointManager.DefaultConnectionLimit = 100;
wr.Method = "GET";
wr.ProtocolVersion = HttpVersion.Version10;
wr.KeepAlive = false;
string data = null;
using (HttpWebResponse response = (HttpWebResponse)wr.GetResponse())
{
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.Default))
{
data = reader.ReadToEnd();
response.Close();
}
}
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.Default))
{
data = reader.ReadToEnd();
response.Close();
}
}
}
else
{
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream, Encoding.Default))
{
data = reader.ReadToEnd();
response.Close();
}
}
}
}
return data;
}

...全文