110,536
社区成员
发帖
与我相关
我的任务
分享
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(“https://honors.libraries.psu.edu/browse/author/all/”);
request.CookieContainer = new CookieContainer();
CookieContainer cookie = request.CookieContainer;//如果用不到Cookie,删去即可
//以下是发送的http头,随便加,其中referer挺重要的,有些网站会根据这个来反盗链
request.Referer = "https://honors.libraries.psu.edu/browse/author/all/";
request.Accept = "text/html, application/xhtml+xml, */*";
request.Headers["Accept-Language"] = "en-US";
request.UserAgent = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)";
request.Headers["UA-CPU"] = "AMD64";
request.Headers["Accept-Encoding"] = "gzip, deflate";
request.Headers["Cache-Control"] = "no-cache";
request.KeepAlive = true;
//上面的http头看情况而定,但是下面俩必须加
request.ContentType = "application/x-www-form-urlencoded";
request.Method = "POST";
request.Host = "honors.libraries.psu.edu";
Encoding encoding = Encoding.UTF8;//根据网站的编码自定义
string[] keys = dicParam.Keys.ToArray<string>();
string[] values = dicParam.Values.ToArray<string>();
int iCount = dicParam.Count;
string postDataStr = "";
for (int i = 0; i < iCount; i++)
{
postDataStr += keys[i] + "=" + values[i] + "&";
}
if (postDataStr.Length > 0)
{
postDataStr = postDataStr.Substring(0, postDataStr.Length - 1);
}
byte[] postData = encoding.GetBytes("csrfmiddlewaretoken=04696113ff3ee3e8220dd9044921e100&browse_start=all&browse_type=author&page=2&display=50&num_display_items=50");//postDataStr即为发送的数据,格式还是和上次说的一样
request.ContentLength = postData.Length*2;
Stream requestStream = request.GetRequestStream();
requestStream.Write(postData, 0, postData.Length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
//如果http头中接受gzip的话,这里就要判断是否为有压缩,有的话,直接解压缩即可
if (response.Headers["Content-Encoding"] != null && response.Headers["Content-Encoding"].ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
}
StreamReader streamReader = new StreamReader(responseStream, encoding);
string retString = streamReader.ReadToEnd();
streamReader.Close();
responseStream.Close();
return retString;
}
catch(Exception exp)
{
MessageBox.Show(exp.Message);
return "";
}