httpclient抓取网页碰到403怎么解决
package tools.crawler;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;
public class DownLoad {
public static void main(String[] args) throws IOException {
System.out.println(DownLoad.downfromweb("http://tech.sina.com.cn/mobile/n/2013-05-14/17328338983.shtml"));
}
public static String downfromweb(String url) throws IOException {
HttpClient httpclient = new HttpClient();
// httpclient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
// httpclient.getParams().setParameter("http.protocol.single-cookie-header",true);
GetMethod getMethod = new GetMethod(url);// http://itindex.net
// google
// getMethod.setRequestHeader("Host", "laohuang.iteye.com");
// getMethod.setRequestHeader("Connection", "Keep-Alive");
// getMethod.setRequestHeader("Accept", "*/*");
// getMethod.setRequestHeader("From", "goolebot@googlebot.com");
// getMethod.setRequestHeader("User-Agent",
// "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
// getMethod.setRequestHeader("Accept-Encoding", "gzip, deflate");
// baidu
getMethod.setRequestHeader("Host", "localhost");// itindex.net
getMethod.setRequestHeader("Connection", "Keep-Alive");
getMethod.setRequestHeader("Accept", "*/*");
getMethod.setRequestHeader("From", "goolebot@googlebot.com");
getMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
getMethod.setRequestHeader("Accept-Encoding", "gzip");
int statusCode = httpclient.executeMethod(getMethod);
System.out.println(statusCode);
if (statusCode == 200) return getBodyAsString(getMethod, getMethod.getResponseCharSet());
else return "";
}
private static String getBodyAsString(GetMethod getHC, String charset) throws IOException {
String acceptEncoding = "";
if (getHC.getResponseHeader("Content-Encoding") != null) acceptEncoding = getHC.getResponseHeader(
"Content-Encoding").getValue();
StringBuffer sb = new StringBuffer();
if (acceptEncoding.toLowerCase().indexOf("gzip") > -1) {
// 建立gzip解压工作流
InputStream is = getHC.getResponseBodyAsStream();
GZIPInputStream gzin = new GZIPInputStream(is);
InputStreamReader isr = new InputStreamReader(gzin, charset); // 设置读取流的编码格式,自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
gzin.close();
} else {
InputStreamReader isr = new InputStreamReader(getHC.getResponseBodyAsStream(), charset); // 设置读取流的编码格式,自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
}
getHC.releaseConnection();
return sb.toString();
}
}
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.GetMethod;
public class DownLoad {
public static void main(String[] args) throws IOException {
System.out.println(DownLoad.downfromweb("http://tech.sina.com.cn/mobile/n/2013-05-14/17328338983.shtml"));
}
public static String downfromweb(String url) throws IOException {
HttpClient httpclient = new HttpClient();
// httpclient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
// httpclient.getParams().setParameter("http.protocol.single-cookie-header",true);
GetMethod getMethod = new GetMethod(url);// http://itindex.net
// getMethod.setRequestHeader("Host", "laohuang.iteye.com");
// getMethod.setRequestHeader("Connection", "Keep-Alive");
// getMethod.setRequestHeader("Accept", "*/*");
// getMethod.setRequestHeader("From", "goolebot@googlebot.com");
// getMethod.setRequestHeader("User-Agent",
// "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
// getMethod.setRequestHeader("Accept-Encoding", "gzip, deflate");
// baidu
getMethod.setRequestHeader("Host", "localhost");// itindex.net
getMethod.setRequestHeader("Connection", "Keep-Alive");
getMethod.setRequestHeader("Accept", "*/*");
getMethod.setRequestHeader("From", "goolebot@googlebot.com");
getMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0");
getMethod.setRequestHeader("Accept-Encoding", "gzip");
int statusCode = httpclient.executeMethod(getMethod);
System.out.println(statusCode);
if (statusCode == 200) return getBodyAsString(getMethod, getMethod.getResponseCharSet());
else return "";
}
private static String getBodyAsString(GetMethod getHC, String charset) throws IOException {
String acceptEncoding = "";
if (getHC.getResponseHeader("Content-Encoding") != null) acceptEncoding = getHC.getResponseHeader(
"Content-Encoding").getValue();
StringBuffer sb = new StringBuffer();
if (acceptEncoding.toLowerCase().indexOf("gzip") > -1) {
// 建立gzip解压工作流
InputStream is = getHC.getResponseBodyAsStream();
GZIPInputStream gzin = new GZIPInputStream(is);
InputStreamReader isr = new InputStreamReader(gzin, charset); // 设置读取流的编码格式,自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
gzin.close();
} else {
InputStreamReader isr = new InputStreamReader(getHC.getResponseBodyAsStream(), charset); // 设置读取流的编码格式,自定义编码
java.io.BufferedReader br = new java.io.BufferedReader(isr);
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
}
getHC.releaseConnection();
return sb.toString();
}
}
...全文
相关推荐
Nginx反爬虫策略,防止UA抓取网站
如果请求中的User-Agent与这些字符串匹配,同样会返回403 Forbidden,阻止其访问。3. **限制请求方法**: 通常,网页抓取主要使用GET、HEAD和POST方法。
httpclient4之百度模拟登陆,回复与58同城自动登陆
本主题将深入探讨如何使用HttpClient4库进行百度和58同城的模拟登陆操作。HttpClient4是一个强大的Java库,它提供了丰富的功能来执行HTTP请求,非常适合进行网页交互。
C#爬虫蜘蛛程序(图片抓取).rar
一旦获取到图片URL,通常会使用WebClient或HttpClient类下载图片到本地。
识别头信息限制爬虫
为了保护这些数据不被非法抓取或滥用,很多网站会采取措施来限制网络爬虫的访问。
HTTP 403错误解决方案[可运行源码]
文件系统权限调整是解决Linux/Unix类服务器403问题的关键环节,必须确保Web服务进程用户(如www-data、nginx、apache)对目标目录具备可读权限,对可执行脚本具备可执行权限,对上传目录具备可写权限


