httpclient抓取网页碰到403怎么解决

zsjxlhzlc zsjxlhzlc 墨衍会员 · AI 创作全网分发 墨衍智能分发 通过墨衍一键同步至各平台 可同步平台 CSDN微信公众号微博知乎掘金百家号博客园抖音小红书 你的文章也可以这样分发 墨衍会员 ¥399起/年,写一次发全网 我也要 + 加入社区
package tools.crawler;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;

public class DownLoad {

public static void main(String[] args) throws IOException {
System.out.println(DownLoad.downfromweb("http://tech.sina.com.cn/mobile/n/2013-05-14/17328338983.shtml"));
}

public static String downfromweb(String url) throws IOException {

HttpClient httpclient = new HttpClient();
// httpclient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
// httpclient.getParams().setParameter("http.protocol.single-cookie-header",true);

GetMethod getMethod = new GetMethod(url);// http://itindex.net

// google
// getMethod.setRequestHeader("Host", "laohuang.iteye.com");
// getMethod.setRequestHeader("Connection", "Keep-Alive");
// getMethod.setRequestHeader("Accept", "*/*");
// getMethod.setRequestHeader("From", "goolebot@googlebot.com");
// getMethod.setRequestHeader("User-Agent",
// "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
// getMethod.setRequestHeader("Accept-Encoding", "gzip, deflate");

// baidu
getMethod.setRequestHeader("Host", "localhost");// itindex.net
getMethod.setRequestHeader("Connection", "Keep-Alive");
getMethod.setRequestHeader("Accept", "*/*");
getMethod.setRequestHeader("From", "goolebot@googlebot.com");
getMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
getMethod.setRequestHeader("Accept-Encoding", "gzip");

int statusCode = httpclient.executeMethod(getMethod);
System.out.println(statusCode);
if (statusCode == 200) return getBodyAsString(getMethod, getMethod.getResponseCharSet());
else return "";
}

private static String getBodyAsString(GetMethod getHC, String charset) throws IOException {
String acceptEncoding = "";
if (getHC.getResponseHeader("Content-Encoding") != null) acceptEncoding = getHC.getResponseHeader(
"Content-Encoding").getValue();
StringBuffer sb = new StringBuffer();

if (acceptEncoding.toLowerCase().indexOf("gzip") > -1) {
// 建立gzip解压工作流
InputStream is = getHC.getResponseBodyAsStream();
GZIPInputStream gzin = new GZIPInputStream(is);
InputStreamReader isr = new InputStreamReader(gzin, charset); // 设置读取流的编码格式,自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
gzin.close();
} else {
InputStreamReader isr = new InputStreamReader(getHC.getResponseBodyAsStream(), charset); // 设置读取流的编码格式,自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
}
getHC.releaseConnection();
return sb.toString();
}
}
...全文
评论 3
收藏 已收藏
分享
返回顶部
zsjxlhzlc
Java 51,407 成员 86,137 内容