67,543
社区成员




private InputStream doGetRequest(final HttpContext context) throws Exception {
HttpResponse response = null;
HttpClient httpclient = null;
try {
httpclient = new DefaultHttpClient();//BasicHttpClient.getHttpClient();
response = httpclient.execute(httprequest, context);
int status = response.getStatusLine().getStatusCode();
if (status == HttpStatus.SC_OK) {
BasicEntry basicEntry = this.getBasicEntry(response);
return this.getContentStream(basicEntry);
} else {
throw new CrawlerException("http status error!" + status);
}
} catch (Exception e) {
throw e;
// throw new CrawlerException(e + " " + httprequest.getURI().toString());
}
finally{
httprequest.releaseConnection();
httpclient.getConnectionManager().closeExpiredConnections();
httpclient.getConnectionManager().closeIdleConnections(0, TimeUnit.SECONDS);
httpclient.getConnectionManager().shutdown();
}
}
private BasicEntry getBasicEntry(HttpResponse response) throws Exception {
InputStream in = null;
try {
ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
HttpEntity entity =response.getEntity();
if( entity != null ){
in = entity.getContent();
}
if(in.available() <= 0){
int c = -1;
while ((c = in.read()) != -1) {
byteArray.write(c);
}
}
byte[] bytes = byteArray.toByteArray();
String charset = EntityUtils.getContentCharSet(response.getEntity());
String contentType = null;
try {
contentType = response.getEntity().getContentType().getValue().toLowerCase();
} catch (Exception e) {
contentType = "text/html";
}
return new BasicEntry(bytes, contentType, charset);
} catch (Exception e) {
throw e;
}
finally{
if (in != null){
try
{
in.close();
}
catch (IOException e)
{
e.printStackTrace ();
}
}
}
}