首页 > 代码库 > httpclient 4.3.3
httpclient 4.3.3
import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.net.UnknownHostException; import java.nio.charset.Charset; import javax.net.ssl.SSLException; import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.http.protocol.HttpContext; public class Crawl { public static HttpGet httpget; public static HttpEntity entity; public static StringBuilder HtmltoString; public static MyRetryHandler RetryHandler; public static CloseableHttpResponse response; public static CloseableHttpClient httpclient; public static class MyRetryHandler implements HttpRequestRetryHandler { public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { if (executionCount >= 5) { // Do not retry if over max retry count return false; } if (exception instanceof InterruptedIOException) { // Timeout return false; } if (exception instanceof UnknownHostException) { // Unknown host return false; } if (exception instanceof ConnectTimeoutException) { // Connection refused return false; } if (exception instanceof SSLException) { // SSL handshake exception return false; } HttpClientContext clientContext = HttpClientContext.adapt(context); HttpRequest request = clientContext.getRequest(); boolean idempotent = !(request instanceof HttpEntityEnclosingRequest); if (idempotent) { // Retry if the request is considered idempotent return true; } return false; } } public static String GetString(final String URL, Charset charset) throws IOException { HtmltoString = new StringBuilder(); RetryHandler = new MyRetryHandler(); HttpHost proxy = new HttpHost("186.238.51.149", 8080); DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner( proxy); httpclient = HttpClients.custom().setRetryHandler(RetryHandler) .setRoutePlanner(routePlanner).build(); // httpclient = // HttpClients.custom().setRetryHandler(RetryHandler).build(); httpget = new HttpGet(URL); try { response = httpclient.execute(httpget); entity = response.getEntity(); if (entity != null) { InputStream instream = entity.getContent(); InputStreamReader ir = new InputStreamReader(instream, charset); BufferedReader reader = new BufferedReader(ir); char[] chars = new char[4096]; int length = 0; while (0 < (length = reader.read(chars))) { HtmltoString.append(chars, 0, length); } } } catch (ClientProtocolException e1) { e1.printStackTrace(); } catch (IOException e1) { e1.printStackTrace(); } finally { response.close(); } return HtmltoString.toString(); } }
声明:以上内容来自用户投稿及互联网公开渠道收集整理发布,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任,若内容有误或涉及侵权可进行投诉: 投诉/举报 工作人员会在5个工作日内联系你,一经查实,本站将立刻删除涉嫌侵权内容。