HTTP隧道（专业版）Java 接入指南| 阿布云

Connection HttpClient(4.5+) HtmlUnit Jsoup WebMagic OkHttp

              
    # -*-*-
    # 感谢骚男 『苗伦兽 (QQ: 519714803)』 提供的源代码
    # -*-*-

    import java.io.ByteArrayOutputStream;
    import java.io.InputStream;
    import java.net.Authenticator;
    import java.net.HttpURLConnection;
    import java.net.InetSocketAddress;
    import java.net.PasswordAuthentication;
    import java.net.Proxy;
    import java.net.URL;

    class ProxyAuthenticator extends Authenticator {
        private String user, password;

        public ProxyAuthenticator(String user, String password) {
            this.user     = user;
            this.password = password;
        }

        protected PasswordAuthentication getPasswordAuthentication() {
            return new PasswordAuthentication(user, password.toCharArray());
        }
    }

    /**
     * 注意：下面代码仅仅实现HTTP请求链接，每一次请求都是无状态保留的，仅仅是这次请求是更换IP的，如果下次请求的IP地址会改变
     * 如果是多线程访问的话，只要将下面的代码嵌入到你自己的业务逻辑里面，那么每次都会用新的IP进行访问，如果担心IP有重复，
     * 自己可以维护IP的使用情况，并做校验。
     *
     * JDK 8u111版本后环境下：要访问的目标页面为HTTPS协议时，需修改“jdk.http.auth.tunneling.disabledSchemes”值
     */
    public class ProxyTest {
        public static void main(String args[]) throws Exception {
            // 要访问的目标页面
            String targetUrl = "http://test.abuyun.com";

            // JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");

            //String targetUrl = "http://proxy.abuyun.com/switch-ip";
            //String targetUrl = "http://proxy.abuyun.com/current-ip";

            // 代理服务器
            String proxyServer = "http-pro.abuyun.com";
            int proxyPort      = 9010;

            // 代理隧道验证信息
            String proxyUser  = "H01234567890123P";
            String proxyPass  = "0123456789012345";

            try {
                URL url = new URL(targetUrl);

                Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));

                // 创建代理服务器地址对象
                InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
                // 创建HTTP类型代理对象
                Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);

                // 设置通过代理访问目标页面
                HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
                // 设置IP切换头
                connection.setRequestProperty("Proxy-Switch-Ip","yes");

                // 解析返回数据
                byte[] response = readStream(connection.getInputStream());

                System.out.println(new String(response));
            } catch (Exception e) {
                System.out.println(e.getLocalizedMessage());
            }
        }

        /**
         * 将输入流转换成字符串
         *
         * @param inStream
         * @return
         * @throws Exception
         */
        public static byte[] readStream(InputStream inStream) throws Exception {
            ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
            byte[] buffer = new byte[1024];
            int len = -1;

            while ((len = inStream.read(buffer)) != -1) {
                outSteam.write(buffer, 0, len);
            }
            outSteam.close();
            inStream.close();

            return outSteam.toByteArray();
        }
    }

              
    import java.io.BufferedReader;
    import java.io.InputStreamReader;
    import java.io.IOException;
    import java.net.URI;
    import java.util.Arrays;
    import java.util.ArrayList;
    import java.util.HashSet;
    import java.util.List;
    import java.util.Set;

    import org.apache.http.Header;
    import org.apache.http.HeaderElement;
    import org.apache.http.HttpHost;
    import org.apache.http.auth.AuthScope;
    import org.apache.http.auth.UsernamePasswordCredentials;
    import org.apache.http.client.AuthCache;
    import org.apache.http.client.CredentialsProvider;
    import org.apache.http.client.HttpRequestRetryHandler;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.config.AuthSchemes;
    import org.apache.http.client.entity.GzipDecompressingEntity;
    import org.apache.http.client.entity.UrlEncodedFormEntity;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.client.methods.HttpRequestBase;
    import org.apache.http.client.protocol.HttpClientContext;
    import org.apache.http.config.Registry;
    import org.apache.http.config.RegistryBuilder;
    import org.apache.http.conn.socket.ConnectionSocketFactory;
    import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
    import org.apache.http.conn.socket.PlainConnectionSocketFactory;
    import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
    import org.apache.http.impl.auth.BasicScheme;
    import org.apache.http.impl.client.BasicAuthCache;
    import org.apache.http.impl.client.BasicCredentialsProvider;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.impl.client.ProxyAuthenticationStrategy;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.message.BasicHeader;
    import org.apache.http.message.BasicNameValuePair;
    import org.apache.http.NameValuePair;
    import org.apache.http.util.EntityUtils;

    public class JavaHttpClient45Demo
    {
        // 代理服务器
        final static String proxyHost = "http-pro.abuyun.com";
        final static Integer proxyPort = 9010;

        // 代理隧道验证信息
        final static String proxyUser = "H01234567890123P";
        final static String proxyPass = "0123456789012345";

        // IP切换协议头
        final static String switchIpHeaderKey = "Proxy-Switch-Ip";
        final static String switchIpHeaderVal = "yes";

        private static PoolingHttpClientConnectionManager cm = null;
        private static HttpRequestRetryHandler httpRequestRetryHandler = null;
        private static HttpHost proxy = null;

        private static CredentialsProvider credsProvider = null;
        private static RequestConfig reqConfig = null;

        static {
            ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
            LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();

            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", plainsf)
                .register("https", sslsf)
                .build();

            cm = new PoolingHttpClientConnectionManager(registry);
            cm.setMaxTotal(20);
            cm.setDefaultMaxPerRoute(5);

            proxy = new HttpHost(proxyHost, proxyPort, "http");

            credsProvider = new BasicCredentialsProvider();
            credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPass));

            reqConfig = RequestConfig.custom()
                .setConnectionRequestTimeout(5000)
                .setConnectTimeout(5000)
                .setSocketTimeout(5000)
                .setExpectContinueEnabled(false)
                .setProxy(new HttpHost(proxyHost, proxyPort))
                .build();
        }

        public static void doRequest(HttpRequestBase httpReq) {
            CloseableHttpResponse httpResp = null;

            try {
                // JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
                System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");

                setHeaders(httpReq);

                httpReq.setConfig(reqConfig);

                CloseableHttpClient httpClient = HttpClients.custom()
                    .setConnectionManager(cm)
                    .setDefaultCredentialsProvider(credsProvider)
                    .build();

                AuthCache authCache = new BasicAuthCache();
                authCache.put(proxy, new BasicScheme());

                HttpClientContext localContext = HttpClientContext.create();
                localContext.setAuthCache(authCache);

                httpResp = httpClient.execute(httpReq, localContext);

                int statusCode = httpResp.getStatusLine().getStatusCode();

                System.out.println(statusCode);

                BufferedReader rd = new BufferedReader(new InputStreamReader(httpResp.getEntity().getContent()));

                String line = "";
                while((line = rd.readLine()) != null) {
                    System.out.println(line);
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                try {
                    if (httpResp != null) {
                        httpResp.close();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

        /**
         * 设置请求头
         *
         * @param httpReq
         */
        private static void setHeaders(HttpRequestBase httpReq) {
            httpReq.setHeader("Accept-Encoding", null);
            httpReq.setHeader(switchIpHeaderKey, switchIpHeaderVal);
        }

        public static void doPostRequest() {
            try {
                // 要访问的目标页面
                HttpPost httpPost = new HttpPost("https://test.abuyun.com");

                // 设置表单参数
                List<NameValuePair> params = new ArrayList<NameValuePair>();
                params.add(new BasicNameValuePair("method", "next"));
                params.add(new BasicNameValuePair("params", "{\"broker\":\"abuyun\":\"site\":\"https://www.abuyun.com\"}"));

                httpPost.setEntity(new UrlEncodedFormEntity(params, "utf-8"));

                doRequest(httpPost);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        public static void doGetRequest() {
            // 要访问的目标页面
            String targetUrl = "https://test.abuyun.com";
            //String targetUrl = "http://proxy.abuyun.com/switch-ip";
            //String targetUrl = "http://proxy.abuyun.com/current-ip";

            try {
                HttpGet httpGet = new HttpGet(targetUrl);

                doRequest(httpGet);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        public static void main(String[] args) {
            doGetRequest();

            //doPostRequest();
        }
    }

              
    package htmlunit;

    import org.apache.http.auth.AuthScope;
    import org.apache.http.auth.UsernamePasswordCredentials;
    import org.apache.http.client.CredentialsProvider;
    import org.apache.http.impl.client.BasicCredentialsProvider;

    import com.gargoylesoftware.htmlunit.BrowserVersion;
    import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
    import com.gargoylesoftware.htmlunit.WebClient;
    import com.gargoylesoftware.htmlunit.html.HtmlPage;

    public class HtmlunitDemo {
        // 代理服务器
        final static String proxyHost = "http-pro.abuyun.com";
        final static Integer proxyPort = 9010;

        // 代理隧道验证信息
        final static String proxyUser = "H01234567890123P";
        final static String proxyPass = "0123456789012345";

        public static void main(String[] args) {
            // 第一步：创建凭证对象 credsProvider
            CredentialsProvider credsProvider = new BasicCredentialsProvider();
            credsProvider.setCredentials(
            //只对proxyHost:proxyPort使用此凭证
            new AuthScope(proxyHost, proxyPort),
            new UsernamePasswordCredentials(proxyUser, proxyPass));

            // 第二步：创建WebClient对象，设置代理服务器及端口
            WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);

            // 第三步：设置WebClient的凭证
            webClient.setCredentialsProvider(credsProvider);

            // 设置隧道代理到此结束，后面是请求的代码了
            webClient.setAjaxController(new NicelyResynchronizingAjaxController());
            webClient.getOptions().setJavaScriptEnabled(true);
            webClient.getOptions().setThrowExceptionOnScriptError(false);
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
            webClient.getOptions().setActiveXNative(false);
            webClient.getOptions().setCssEnabled(false);

            HtmlPage page = null;

            try {
                page = webClient.getPage("http://test.abuyun.com");
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                webClient.close();
            }

            webClient.waitForBackgroundJavaScript(30000);

            // 直接将加载完成的页面转换成xml格式的字符串
            String pageXml = page.asXml();

            System.out.println(pageXml);
        }
    }

              
    # -*-*-
    # 感谢骚男 『不蛰不鸣』 提供的源代码
    # -*-*-

    import java.io.IOException;
    import java.net.Authenticator;
    import java.net.InetSocketAddress;
    import java.net.PasswordAuthentication;
    import java.net.Proxy;

    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;

    /***
     * 通过阿布云代理访问指定URL 内容
     * 此处 Jsoup Version 1.9.1
     * @author sun
     * JDK 8u111版本后环境下：要访问的目标页面为HTTPS协议时，需修改“jdk.http.auth.tunneling.disabledSchemes”值
     */
    public class ProxyDemo
    {
        // 代理隧道验证信息
        final static String ProxyUser = "H01234567890123P";
        final static String ProxyPass = "0123456789012345";

        // 代理服务器
        final static String ProxyHost = "http-pro.abuyun.com";
        final static Integer ProxyPort = 9010;
        // 设置IP切换头
        final static String ProxyHeadKey = "Proxy-Switch-Ip";
        final static String ProxyHeadVal = "yes";
        public static String getUrlProxyContent(String url)
        {
            Authenticator.setDefault(new Authenticator() {
                public PasswordAuthentication getPasswordAuthentication()
                {
                    return new PasswordAuthentication(ProxyUser, ProxyPass.toCharArray());
                }
            });

            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ProxyHost, ProxyPort));

            try
            {
                // 此处自己处理异常、其他参数等
                Document doc = Jsoup.connect(url)
                    .followRedirects(false)
                    .timeout(3000)
                    .header(ProxyHeadKey, ProxyHeadVal)
                    .proxy(proxy)
                    // 忽略证书校验
                    .validateTLSCertificates(false)
                    .get()
                ;

                if(doc != null) {
                    System.out.println(doc.body().html());
                }
            }
            catch (IOException e)
            {
                e.printStackTrace();
            }

            return null;
        }

        public static void main(String[] args) throws Exception
        {
            // 要访问的目标页面
            String targetUrl = "http://test.abuyun.com";

            // JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");

            //String targetUrl = "http://proxy.abuyun.com/switch-ip";
            //String targetUrl = "http://proxy.abuyun.com/current-ip";
            getUrlProxyContent(targetUrl);
        }
    }

              
    import us.codecraft.webmagic.Page;
    import us.codecraft.webmagic.Site;
    import us.codecraft.webmagic.Spider;
    import us.codecraft.webmagic.downloader.HttpClientDownloader;
    import us.codecraft.webmagic.processor.PageProcessor;
    import us.codecraft.webmagic.proxy.Proxy;
    import us.codecraft.webmagic.proxy.SimpleProxyProvider;

    /**
     * 基于WebMagic 0.7.3版本，关于WebMagic代理详细设置请看WebMagic官方文档
     **/
    public class WebMagicProxyDemo implements PageProcessor
    {
        private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

        @Override
        public void process(Page page) {
            page.putField("html", page.getHtml());
        }

        @Override
        public Site getSite() {
            return site;
        }

        public static void main(String[] args) {
            // 代理服务器
            final String    proxyHost = "http-pro.abuyun.com";
            final Integer   proxyPort = 9010;

            // 代理隧道验证信息
            final String    proxyUser = "H01234567890123P";
            final String    proxyPass = "0123456789012345";

            final String 	targetUrl ="http://test.abuyun.com/";

            HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
            httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy(proxyHost, proxyPort, proxyUser, proxyPass)));

            Spider.create(new WebMagicProxyDemo())
                  .setDownloader(httpClientDownloader)
                  .addUrl(targetUrl)
                  .thread(5)
                  .run();
        }
    }

              
    # -*-*-
    # 感谢骚男 『garen』 提供的源代码
    # -*-*-

    import okhttp3.*;

    import java.io.IOException;
    import java.net.InetSocketAddress;
    import java.net.Proxy;
    import java.util.concurrent.TimeUnit;

    public class OkHttp {

        // 代理服务器
        final static String proxyHost = "http-pro.abuyun.com";
        final static Integer proxyPort = 9010;

        // 代理隧道验证信息
        final static String proxyUser = "H01234567890123P";
        final static String proxyPass = "0123456789012345";

        static OkHttpClient client = null;

        static {
            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));

            Authenticator proxyAuthenticator = new Authenticator() {
                public Request authenticate(Route route, Response response) {
                    String credential = Credentials.basic(proxyUser, proxyPass);
                    return response.request().newBuilder()
                            .header("Proxy-Authorization", credential)
                            .build();
                }
            };

            client = new OkHttpClient().newBuilder()
                    .connectTimeout(5, TimeUnit.SECONDS)
                    .readTimeout(5, TimeUnit.SECONDS)
                    .proxy(proxy)
                    .proxyAuthenticator(proxyAuthenticator)
                    .connectionPool(new ConnectionPool(5, 1, TimeUnit.SECONDS))
                    .build();
        }

        public static Response doGet() throws IOException {
            // 要访问的目标页面
            String targetUrl = "http://test.abuyun.com";

            Request request = new Request.Builder()
                    .url(targetUrl)
                    .build();
            Response response = client.newCall(request).execute();
            return response;
        }

        public static void main(String[] args) throws IOException {
            Response response1 = doGet();
            System.out.println("GET请求返回结果：");
            System.out.println(response1.body().string());
        }

    }

隧道验证方式

隧道验证方式