httpclient抓取网页碰到403怎么解决

[zsjxlhzlc]

package tools.crawler;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;

public class DownLoad {

public static void main(String[] args) throws IOException {
System.out.println(DownLoad.downfromweb("http://tech.sina.com.cn/mobile/n/2013-05-14/17328338983.shtml"));
}

public static String downfromweb(String url) throws IOException {

HttpClient httpclient = new HttpClient();
// httpclient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
// httpclient.getParams().setParameter("http.protocol.single-cookie-header",true);

GetMethod getMethod = new GetMethod(url);// http://itindex.net

// google
// getMethod.setRequestHeader("Host", "laohuang.iteye.com");
// getMethod.setRequestHeader("Connection", "Keep-Alive");
// getMethod.setRequestHeader("Accept", "*/*");
// getMethod.setRequestHeader("From", "[e-mail]");
// getMethod.setRequestHeader("User-Agent",
// "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
// getMethod.setRequestHeader("Accept-Encoding", "gzip, deflate");

// baidu
getMethod.setRequestHeader("Host", "localhost");// itindex.net
getMethod.setRequestHeader("Connection", "Keep-Alive");
getMethod.setRequestHeader("Accept", "*/*");
getMethod.setRequestHeader("From", "[e-mail]");
getMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
getMethod.setRequestHeader("Accept-Encoding", "gzip");

int statusCode = httpclient.executeMethod(getMethod);
System.out.println(statusCode);
if (statusCode == 200) return getBodyAsString(getMethod, getMethod.getResponseCharSet());
else return "";
}

private static String getBodyAsString(GetMethod getHC, String charset) throws IOException {
String acceptEncoding = "";
if (getHC.getResponseHeader("Content-Encoding") != null) acceptEncoding = getHC.getResponseHeader(
"Content-Encoding").getValue();
StringBuffer sb = new StringBuffer();

if (acceptEncoding.toLowerCase().indexOf("gzip") > -1) {
// 建立gzip解压工作流
InputStream is = getHC.getResponseBodyAsStream();
GZIPInputStream gzin = new GZIPInputStream(is);
InputStreamReader isr = new InputStreamReader(gzin, charset); // 设置读取流的编码格式，自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
gzin.close();
} else {
InputStreamReader isr = new InputStreamReader(getHC.getResponseBodyAsStream(), charset); // 设置读取流的编码格式，自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
}
getHC.releaseConnection();
return sb.toString();
}
}

[小鬼头1111]

楼主解决了吗，我也遇到这个问题了，返回403

[昆卡卡]

403应该是权限不够引起的吧,看看你访问的页面是否需要什么特殊权限?

[小鬼头1111]

引用 2 楼 yyy269954107 的回复:

403应该是权限不够引起的吧,看看你访问的页面是否需要什么特殊权限?

有的时候可以进去，有的时候403