首页 > 代码库 > 获取URL列表,设置代理请求URL,https的加密方式处理
获取URL列表,设置代理请求URL,https的加密方式处理
做了一个测试的一个小工具,需求如下:
1、有一批URL列表,需要知道哪个URL请求响应内容中包含http:关键字的。
2、url请求包括http和https 2种协议
3、要部署在linux服务器上,且linux服务器只能通过代理来连接外网
帖一下我的核心代码吧:
package com.cn.util; import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.Proxy; import java.net.URL; import java.net.URLConnection; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.Map; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import javax.security.cert.CertificateException; import javax.security.cert.X509Certificate; import javax.xml.ws.Response; import net.sf.json.JSONObject; public class HttpGet { // SSL private static class TrustAnyTrustManager implements X509TrustManager { public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { } public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { } public void checkClientTrusted( java.security.cert.X509Certificate[] arg0, String arg1) throws java.security.cert.CertificateException { // TODO Auto-generated method stub } public void checkServerTrusted( java.security.cert.X509Certificate[] arg0, String arg1) throws java.security.cert.CertificateException { // TODO Auto-generated method stub } public java.security.cert.X509Certificate[] getAcceptedIssuers() { // TODO Auto-generated method stub return null; } } private static class TrustAnyHostnameVerifier implements HostnameVerifier { public boolean verify(String hostname, SSLSession session) { return true; } } // 得到URL public List getUrl(String file) { BufferedReader bf = null; List urlList = new ArrayList(); int linenum = 0; try { bf = new BufferedReader(new FileReader(file)); String info = ""; while ((info = bf.readLine()) != null) { linenum++; if (!"".equals(info)) { /* * if ((info.startsWith("http:") || * info.startsWith("https:"))) { * System.out.println("url===>" + info); urlList.add(info); * }else{ System.out.println("第"+linenum+"行,请求协议或有问题。。:"); } */ System.out.println("url===>" + info); urlList.add(info); } else { System.out.println("第" + linenum + "行url为空"); } } } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { try { bf.close(); } catch (IOException e) { e.printStackTrace(); } } return urlList; } // 获取请求非200状态的url public List<String> http404(List<String> urls) { List<String> result404 = new ArrayList<String>(); try { /* * System.setProperty("proxySet", "true"); * System.setProperty("http.proxyHost", "192.168.11.254"); * System.setProperty("http.proxyPort", "8080"); */ Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress( "192.168.11.254", 8080)); for (String url2 : urls) { if (url2.startsWith("http:") || (url2.startsWith("https:"))) { URL realUrl = new URL(new String(url2.getBytes("utf-8"))); HttpURLConnection connection = (HttpURLConnection) realUrl .openConnection(proxy); // 如果是https if (connection instanceof HttpsURLConnection) { SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, new TrustManager[] { new TrustAnyTrustManager() }, new java.security.SecureRandom()); ((HttpsURLConnection) connection) .setSSLSocketFactory(sc.getSocketFactory()); ((HttpsURLConnection) connection) .setHostnameVerifier(new TrustAnyHostnameVerifier()); } // 设置通用的请求属性 connection.setRequestProperty("accept", "*/*"); connection.setRequestProperty("connection", "Keep-Alive"); connection .setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)"); // 连接 connection.connect(); if (connection.getResponseCode() != 200) { result404.add(url2); } } else { result404.add(url2); } } } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } catch (KeyManagementException e) { e.printStackTrace(); } return result404; } // 请求获取响应,把包含搜索关键字的URL存入LIST中 public List<String> sendUrl(List<String> urls, String keyword) { List<String> result200 = new ArrayList<String>(); BufferedReader in = null; try { /* * System.setProperty("proxySet", "true"); * System.setProperty("http.proxyHost", "192.168.11.254"); * System.setProperty("http.proxyPort", "8080"); */ Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress( "192.168.11.254", 8080)); for (String url2 : urls) { StringBuffer response = new StringBuffer(); if (url2.startsWith("http:") || (url2.startsWith("https:"))) { URL realUrl = new URL(new String(url2.getBytes("utf-8"))); HttpURLConnection connection = (HttpURLConnection) realUrl .openConnection(proxy); // 如果是https if (connection instanceof HttpsURLConnection) { SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, new TrustManager[] { new TrustAnyTrustManager() }, new java.security.SecureRandom()); ((HttpsURLConnection) connection) .setSSLSocketFactory(sc.getSocketFactory()); ((HttpsURLConnection) connection) .setHostnameVerifier(new TrustAnyHostnameVerifier()); } // 设置通用的请求属性 connection.setRequestProperty("accept", "*/*"); connection.setRequestProperty("connection", "Keep-Alive"); connection .setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)"); // 连接 connection.connect(); if (connection.getResponseCode() == 200) { in = new BufferedReader(new InputStreamReader( connection.getInputStream())); String line = ""; while ((line = in.readLine()) != null) { response.append(line); } in.close(); } if (response.toString().contains(keyword)) { result200.add(url2); } } System.out.println(url2 + "=============>" + response.toString()); } } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } catch (KeyManagementException e) { e.printStackTrace(); } return result200; } public static void main(String[] args) { HttpGet ht = new HttpGet(); List urls = ht.getUrl("E:\\课件\\https\\1.txt"); String keyword = "http:"; List<String> urllist = ht.http404(urls); for (String string : urllist) { System.out.println("404=" + string); } } }
TrustAnyTrustManager 类中的方法,是我在网上找的,目地是解决SSL加密请求,针对https://协议的请求来处理的。具体是什么我也不懂
/* * System.setProperty("proxySet", "true"); * System.setProperty("http.proxyHost", "192.168.11.254"); * System.setProperty("http.proxyPort", "8080"); */
这种设置代理的方式,不行的
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(
"192.168.11.254", 8080));
HttpURLConnection connection = (HttpURLConnection) realUrl
.openConnection(proxy);
获取URL列表,设置代理请求URL,https的加密方式处理
声明:以上内容来自用户投稿及互联网公开渠道收集整理发布,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任,若内容有误或涉及侵权可进行投诉: 投诉/举报 工作人员会在5个工作日内联系你,一经查实,本站将立刻删除涉嫌侵权内容。