HTTP隧道 (经典版)
Java 接入指南

隧道验证方式

地址:http-cla.abuyun.com,端口:9030

隧道验证方式

HTTP隧道拥有两种授权模式:

  • 请求头(默认)

通过用户名/密码的形式进行身份认证,该认证信息最终会转换为『Proxy-Authorization』协议头跟随请求一起发出。

为便于部分语言进行接入,平台亦支持通过『Authorization』协议头进行隧道身份验证。

  • 绑定IP(暂未启用)

只须绑定用户发起请求的服务器IP即可。

一条代理隧道只能绑定一个IP,同一IP可以分别绑定到专业版、动态版、经典版代理隧道各一条。

              
    # -*-*-
    # 感谢骚男 『苗伦兽 (QQ: 519714803)』 提供的源代码
    # -*-*-

    import java.io.ByteArrayOutputStream;
    import java.io.InputStream;
    import java.net.Authenticator;
    import java.net.HttpURLConnection;
    import java.net.InetSocketAddress;
    import java.net.PasswordAuthentication;
    import java.net.Proxy;
    import java.net.URL;

    class ProxyAuthenticator extends Authenticator {
        private String user, password;

        public ProxyAuthenticator(String user, String password) {
            this.user     = user;
            this.password = password;
        }

        protected PasswordAuthentication getPasswordAuthentication() {
            return new PasswordAuthentication(user, password.toCharArray());
        }
    }

    /**
     * 注意:下面代码仅仅实现HTTP请求链接,每一次请求都是无状态保留的,仅仅是这次请求是更换IP的,如果下次请求的IP地址会改变
     * 如果是多线程访问的话,只要将下面的代码嵌入到你自己的业务逻辑里面,那么每次都会用新的IP进行访问,如果担心IP有重复,
     * 自己可以维护IP的使用情况,并做校验。
     * JDK 8u111版本后环境下:要访问的目标页面为HTTPS协议时,需修改“jdk.http.auth.tunneling.disabledSchemes”值
     */
    public class ProxyTest {
        public static void main(String args[]) throws Exception {
            // 要访问的目标页面
            String targetUrl = "http://test.abuyun.com";
            //JDK 8u111版本后,目标页面为HTTPS协议,启用proxy用户密码鉴权
            //System.setProperty("jdk.http.auth.tunneling.disabledSchemes","");

            //String targetUrl = "http://proxy.abuyun.com/switch-ip";
            //String targetUrl = "http://proxy.abuyun.com/current-ip";

            // 代理服务器
            String proxyServer = "http-cla.abuyun.com";
            int proxyPort      = 9030;

            // 代理隧道验证信息
            String proxyUser  = "H01234567890123C";
            String proxyPass  = "0123456789012345";

            try {
                URL url = new URL(targetUrl);

                Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));

                // 创建代理服务器地址对象
                InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
                // 创建HTTP类型代理对象
                Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);

                // 设置通过代理访问目标页面
                HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
                // 设置IP切换头
                connection.setRequestProperty("Proxy-Switch-Ip","yes");

                // 解析返回数据
                byte[] response = readStream(connection.getInputStream());

                System.out.println(new String(response));
            } catch (Exception e) {
                System.out.println(e.getLocalizedMessage());
            }
        }

        /**
         * 将输入流转换成字符串
         *
         * @param inStream
         * @return
         * @throws Exception
         */
        public static byte[] readStream(InputStream inStream) throws Exception {
            ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
            byte[] buffer = new byte[1024];
            int len = -1;

            while ((len = inStream.read(buffer)) != -1) {
                outSteam.write(buffer, 0, len);
            }
            outSteam.close();
            inStream.close();

            return outSteam.toByteArray();
        }
    }              
            
              
    import java.io.BufferedReader;
    import java.io.InputStreamReader;
    import java.io.IOException;
    import java.net.URI;
    import java.util.Arrays;
    import java.util.ArrayList;
    import java.util.HashSet;
    import java.util.List;
    import java.util.Set;

    import org.apache.http.Header;
    import org.apache.http.HeaderElement;
    import org.apache.http.HttpHost;
    import org.apache.http.auth.AuthScope;
    import org.apache.http.auth.UsernamePasswordCredentials;
    import org.apache.http.client.AuthCache;
    import org.apache.http.client.CredentialsProvider;
    import org.apache.http.client.HttpRequestRetryHandler;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.config.AuthSchemes;
    import org.apache.http.client.entity.GzipDecompressingEntity;
    import org.apache.http.client.entity.UrlEncodedFormEntity;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.methods.HttpPost;
    import org.apache.http.client.methods.HttpRequestBase;
    import org.apache.http.client.protocol.HttpClientContext;
    import org.apache.http.config.Registry;
    import org.apache.http.config.RegistryBuilder;
    import org.apache.http.conn.socket.ConnectionSocketFactory;
    import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
    import org.apache.http.conn.socket.PlainConnectionSocketFactory;
    import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
    import org.apache.http.impl.auth.BasicScheme;
    import org.apache.http.impl.client.BasicAuthCache;
    import org.apache.http.impl.client.BasicCredentialsProvider;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.impl.client.ProxyAuthenticationStrategy;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.message.BasicHeader;
    import org.apache.http.message.BasicNameValuePair;
    import org.apache.http.NameValuePair;
    import org.apache.http.util.EntityUtils;

    public class JavaHttpClient45Demo
    {
        // 代理服务器
        final static String proxyHost = "http-cla.abuyun.com";
        final static Integer proxyPort = 9030;

        // 代理隧道验证信息
        final static String proxyUser = "H01234567890123C";
        final static String proxyPass = "0123456789012345";

        // IP切换协议头
        final static String switchIpHeaderKey = "Proxy-Switch-Ip";
        final static String switchIpHeaderVal = "yes";

        private static PoolingHttpClientConnectionManager cm = null;
        private static HttpRequestRetryHandler httpRequestRetryHandler = null;
        private static HttpHost proxy = null;

        private static CredentialsProvider credsProvider = null;
        private static RequestConfig reqConfig = null;

        static {
            ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
            LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();

            Registry registry = RegistryBuilder.create()
                .register("http", plainsf)
                .register("https", sslsf)
                .build();

            cm = new PoolingHttpClientConnectionManager(registry);
            cm.setMaxTotal(20);
            cm.setDefaultMaxPerRoute(5);

            proxy = new HttpHost(proxyHost, proxyPort, "http");

            credsProvider = new BasicCredentialsProvider();
            credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPass));

            reqConfig = RequestConfig.custom()
                .setConnectionRequestTimeout(5000)
                .setConnectTimeout(5000)
                .setSocketTimeout(5000)
                .setExpectContinueEnabled(false)
                .setProxy(new HttpHost(proxyHost, proxyPort))
                .build();
        }

        public static void doRequest(HttpRequestBase httpReq) {
            CloseableHttpResponse httpResp = null;

            try {
                setHeaders(httpReq);

                httpReq.setConfig(reqConfig);

                CloseableHttpClient httpClient = HttpClients.custom()
                    .setConnectionManager(cm)
                    .setDefaultCredentialsProvider(credsProvider)
                    .build();

                AuthCache authCache = new BasicAuthCache();
                authCache.put(proxy, new BasicScheme());

                HttpClientContext localContext = HttpClientContext.create();
                localContext.setAuthCache(authCache);

                httpResp = httpClient.execute(httpReq, localContext);

                int statusCode = httpResp.getStatusLine().getStatusCode();

                System.out.println(statusCode);

                BufferedReader rd = new BufferedReader(new InputStreamReader(httpResp.getEntity().getContent()));

                String line = "";
                while((line = rd.readLine()) != null) {
                    System.out.println(line);
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                try {
                    if (httpResp != null) {
                        httpResp.close();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

        /**
         * 设置请求头
         *
         * @param httpReq
         */
        private static void setHeaders(HttpRequestBase httpReq) {
            httpReq.setHeader("Accept-Encoding", null);
            httpReq.setHeader(switchIpHeaderKey, switchIpHeaderVal);
        }

        public static void doPostRequest() {
            try {
                // 要访问的目标页面
                HttpPost httpPost = new HttpPost("https://test.abuyun.com");

                // 设置表单参数
                List params = new ArrayList();
                params.add(new BasicNameValuePair("method", "next"));
                params.add(new BasicNameValuePair("params", "{\"broker\":\"abuyun\":\"site\":\"https://www.abuyun.com\"}"));

                httpPost.setEntity(new UrlEncodedFormEntity(params, "utf-8"));

                doRequest(httpPost);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        public static void doGetRequest() {
            // 要访问的目标页面
            String targetUrl = "https://test.abuyun.com";
            //String targetUrl = "http://proxy.abuyun.com/switch-ip";
            //String targetUrl = "http://proxy.abuyun.com/current-ip";

            try {
                HttpGet httpGet = new HttpGet(targetUrl);

                doRequest(httpGet);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        public static void main(String[] args) {
            doGetRequest();

            //doPostRequest();
        }
    }              
            
              
    package htmlunit;

    import org.apache.http.auth.AuthScope;
    import org.apache.http.auth.UsernamePasswordCredentials;
    import org.apache.http.client.CredentialsProvider;
    import org.apache.http.impl.client.BasicCredentialsProvider;

    import com.gargoylesoftware.htmlunit.BrowserVersion;
    import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
    import com.gargoylesoftware.htmlunit.WebClient;
    import com.gargoylesoftware.htmlunit.html.HtmlPage;

    public class HtmlunitDemo {
        // 代理服务器
        final static String proxyHost = "http-cla.abuyun.com";
        final static Integer proxyPort = 9030;

        // 代理隧道验证信息
        final static String proxyUser = "H01234567890123C";
        final static String proxyPass = "0123456789012345";

        public static void main(String[] args) {
            // 第一步:创建凭证对象 credsProvider
            CredentialsProvider credsProvider = new BasicCredentialsProvider();
            credsProvider.setCredentials(
            //只对proxyHost:proxyPort使用此凭证
            new AuthScope(proxyHost, proxyPort),
            new UsernamePasswordCredentials(proxyUser, proxyPass));

            // 第二步:创建WebClient对象,设置代理服务器及端口
            WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);

            // 第三步:设置WebClient的凭证
            webClient.setCredentialsProvider(credsProvider);

            // 设置隧道代理到此结束,后面是请求的代码了
            webClient.setAjaxController(new NicelyResynchronizingAjaxController());
            webClient.getOptions().setJavaScriptEnabled(true);
            webClient.getOptions().setThrowExceptionOnScriptError(false);
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
            webClient.getOptions().setActiveXNative(false);
            webClient.getOptions().setCssEnabled(false);

            HtmlPage page = null;

            try {
                page = webClient.getPage("http://test.abuyun.com");
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                webClient.close();
            }

            webClient.waitForBackgroundJavaScript(30000);

            // 直接将加载完成的页面转换成xml格式的字符串
            String pageXml = page.asXml();

            System.out.println(pageXml);
        }
    }              
            
              
    # -*-*-
    # 感谢骚男 『不蛰不鸣』 提供的源代码
    # -*-*-

    import java.io.IOException;
    import java.net.Authenticator;
    import java.net.InetSocketAddress;
    import java.net.PasswordAuthentication;
    import java.net.Proxy;

    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;

    /***
     * 通过阿布云代理访问指定URL 内容
     * 此处 Jsoup Version 1.9.1
     * @author sun
     * JDK 8u111版本后环境下:要访问的目标页面为HTTPS协议时,需修改“jdk.http.auth.tunneling.disabledSchemes”值
     */
    public class ProxyDemo
    {
        // 代理隧道验证信息
        final static String ProxyUser = "H01234567890123C";
        final static String ProxyPass = "0123456789012345";

        // 代理服务器
        final static String ProxyHost = "http-cla.abuyun.com";
        final static Integer ProxyPort = 9030;

        // 设置IP切换头
        final static String ProxyHeadKey = "Proxy-Switch-Ip";
        final static String ProxyHeadVal = "yes";

        public static String getUrlProxyContent(String url)
        {
            Authenticator.setDefault(new Authenticator() {
                public PasswordAuthentication getPasswordAuthentication()
                {
                    return new PasswordAuthentication(ProxyUser, ProxyPass.toCharArray());
                }
            });

            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ProxyHost, ProxyPort));

            try
            {
                // 此处自己处理异常、其他参数等
                Document doc = Jsoup.connect(url)
                    .followRedirects(false)
                    .timeout(3000)
                    .header(ProxyHeadKey, ProxyHeadVal)
                    .proxy(proxy)
                    .validateTLSCertificates(false)
                    .get()
                ;

                if(doc != null) {
                    System.out.println(doc.body().html());
                }
            }
            catch (IOException e)
            {
                e.printStackTrace();
            }

            return null;
        }

        public static void main(String[] args) throws Exception
        {
            // 要访问的目标页面
            String targetUrl = "http://test.abuyun.com";
            //JDK 8u111版本后,目标页面为HTTPS协议,启用proxy用户密码鉴权
            //System.setProperty("jdk.http.auth.tunneling.disabledSchemes","");
            //String targetUrl = "http://proxy.abuyun.com/switch-ip";
            //String targetUrl = "http://proxy.abuyun.com/current-ip";

            getUrlProxyContent(targetUrl);
        }
    }              
            
              
    # -*-*-
    # 感谢骚男 『黑白之间 (QQ: 420269696)』 提供的源代码
    # -*-*-

    import java.io.IOException;
    import java.nio.charset.Charset;
    import java.util.ArrayList;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    import java.util.Set;

    import org.apache.commons.io.IOUtils;
    import org.apache.commons.lang.StringUtils;
    import org.apache.http.Header;
    import org.apache.http.HttpException;
    import org.apache.http.HttpHost;
    import org.apache.http.HttpRequest;
    import org.apache.http.HttpRequestInterceptor;
    import org.apache.http.HttpResponse;
    import org.apache.http.annotation.ThreadSafe;
    import org.apache.http.auth.AuthScope;
    import org.apache.http.auth.UsernamePasswordCredentials;
    import org.apache.http.client.AuthCache;
    import org.apache.http.client.CookieStore;
    import org.apache.http.client.CredentialsProvider;
    import org.apache.http.client.config.CookieSpecs;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.protocol.HttpClientContext;
    import org.apache.http.config.Registry;
    import org.apache.http.config.RegistryBuilder;
    import org.apache.http.config.SocketConfig;
    import org.apache.http.conn.socket.ConnectionSocketFactory;
    import org.apache.http.conn.socket.PlainConnectionSocketFactory;
    import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
    import org.apache.http.impl.auth.BasicScheme;
    import org.apache.http.impl.client.BasicAuthCache;
    import org.apache.http.impl.client.BasicCookieStore;
    import org.apache.http.impl.client.BasicCredentialsProvider;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
    import org.apache.http.impl.client.HttpClientBuilder;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.impl.cookie.BasicClientCookie;
    import org.apache.http.message.BasicHeader;
    import org.apache.http.protocol.HttpContext;
    import org.apache.http.util.EntityUtils;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;

    import us.codecraft.webmagic.Page;
    import us.codecraft.webmagic.Request;
    import us.codecraft.webmagic.Site;
    import us.codecraft.webmagic.Spider;
    import us.codecraft.webmagic.Task;
    import us.codecraft.webmagic.downloader.Downloader;
    import us.codecraft.webmagic.processor.PageProcessor;
    import us.codecraft.webmagic.selector.PlainText;
    import us.codecraft.webmagic.utils.UrlUtils;

    /**
     * webmagic 0.5.3版本接入阿布云代理需要自定义下载组件,0.5.4版本作者代理池功能进行了改进并且加入了代理身份认证,推荐使用0.5.4版本,
     * 或许不用自定义下载组件
     */
    public class WebMagicProxyDemo
    {
        public static void main(String[] args) {
            Spider.create(new PageProcessor() {
                @Override
                public void process(Page page) {
                    System.out.println(page.getHtml());
                }

                @Override
                public Site getSite() {
                    return Site.me();
                }
            }).setDownloader(new MyDownloader()).addUrl("https://test.abuyun.com").thread(1).run();
        }
    }

    @ThreadSafe
    class MyDownloader implements Downloader
    {
        // 代理服务器
        final static String proxyHost = "http-cla.abuyun.com";
        final static Integer proxyPort = 9030;

        private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();

        private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();

        @Override
        public void setThread(int threadNum) {}

        @Override
        public Page download(Request request, Task task) {
            Site site = task.getSite();
            Set<Integer> acceptStatCode = site.getAcceptStatCode();
            String charset = site.getCharset();
            CloseableHttpResponse httpResponse = null;
            int statusCode = 0;

            try {
                HttpHost target = new HttpHost(proxyHost, proxyPort, "http");
                site.setHttpProxy(target);
                request.putExtra(Request.PROXY, target);

                HttpClientContext localContext = HttpClientContext.create();
                AuthCache authCache = new BasicAuthCache();
                BasicScheme basicAuth = new BasicScheme();
                authCache.put(site.getHttpProxy(), basicAuth);
                localContext.setAuthCache(authCache);

                HttpGet httpGet = new HttpGet(request.getUrl());
                RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
                        .setConnectionRequestTimeout(site.getTimeOut()).setSocketTimeout(site.getTimeOut())
                        .setConnectTimeout(site.getTimeOut()).setCookieSpec(CookieSpecs.IGNORE_COOKIES);
                httpGet.setConfig(requestConfigBuilder.build());

                httpResponse = getHttpClient(site).execute(target, httpGet, localContext);

                statusCode = httpResponse.getStatusLine().getStatusCode();
                request.putExtra(Request.STATUS_CODE, statusCode);

                if (statusAccept(acceptStatCode, statusCode)) {
                    Page page = handleResponse(request, charset, httpResponse, task);
                    onSuccess(request);
                    return page;
                } else {
                    return null;
                }
            } catch (IOException e) {
                if (site.getCycleRetryTimes() > 0) {
                    return addToCycleRetry(request, site);
                }
                onError(request);
                return null;
            } finally {
                request.putExtra(Request.STATUS_CODE, statusCode);
                try {
                    if (httpResponse != null) {
                        EntityUtils.consume(httpResponse.getEntity());
                    }
                }
                catch (IOException e) {
                }
            }
        }

        private CloseableHttpClient getHttpClient(Site site) {
            String domain = site.getDomain();
            CloseableHttpClient httpClient = httpClients.get(domain);

            if (httpClient == null) {
                synchronized (this) {
                    httpClient = httpClients.get(domain);
                    if (httpClient == null) {
                        httpClient = httpClientGenerator.getClient(site);
                        httpClients.put(domain, httpClient);
                    }
                }
            }

            return httpClient;
        }

        protected boolean statusAccept(Set<Integer> acceptStatCode, int statusCode) {
            return acceptStatCode.contains(statusCode);
        }

        protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task)
                throws IOException {
            String content = getContent(charset, httpResponse);

            Page page = new Page();
            page.setRawText(content);
            page.setUrl(new PlainText(request.getUrl()));
            page.setRequest(request);
            page.setStatusCode(httpResponse.getStatusLine().getStatusCode());

            return page;
        }

        protected String getContent(String charset, HttpResponse httpResponse) throws IOException {
            if (charset == null) {
                byte[] contentBytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
                String htmlCharset = getHtmlCharset(httpResponse, contentBytes);

                if (htmlCharset != null) {
                    return new String(contentBytes, htmlCharset);
                } else {
                    return new String(contentBytes);
                }
            } else {
                return IOUtils.toString(httpResponse.getEntity().getContent(), charset);
            }
        }

        protected String getHtmlCharset(HttpResponse httpResponse, byte[] contentBytes) throws IOException {
            String charset = null;
            String value = httpResponse.getEntity().getContentType().getValue();
            charset = UrlUtils.getCharset(value);
            if (StringUtils.isNotBlank(charset)) {
                return charset;
            }

            Charset defaultCharset = Charset.defaultCharset();
            String content = new String(contentBytes, defaultCharset.name());
            if (StringUtils.isNotEmpty(content)) {
                Document document = Jsoup.parse(content);
                Elements links = document.select("meta");

                for (Element link : links) {
                    String metaContent = link.attr("content");
                    String metaCharset = link.attr("charset");

                    if (metaContent.indexOf("charset") != -1) {
                        metaContent = metaContent.substring(metaContent.indexOf("charset"), metaContent.length());
                        charset = metaContent.split("=")[1];
                        break;
                    } else if (StringUtils.isNotEmpty(metaCharset)) {
                        charset = metaCharset;
                        break;
                    }
                }
            }

            return charset;
        }

        protected void onSuccess(Request request) {}

        protected void onError(Request request) {}

        protected Page addToCycleRetry(Request request, Site site) {
            Page page = new Page();
            Object cycleTriedTimesObject = request.getExtra(Request.CYCLE_TRIED_TIMES);

            if (cycleTriedTimesObject == null) {
                page.addTargetRequest(request.setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, 1));
            } else {
                int cycleTriedTimes = (Integer) cycleTriedTimesObject;
                cycleTriedTimes++;
                if (cycleTriedTimes >= site.getCycleRetryTimes()) {
                    return null;
                }
                page.addTargetRequest(request.setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, cycleTriedTimes));
            }

            page.setNeedCycleRetry(true);

            return page;
        }
    }

    class HttpClientGenerator
    {
        private PoolingHttpClientConnectionManager connectionManager;

        // 代理隧道验证信息
        final static String proxyUser = "H01234567890123C";
        final static String proxyPass = "0123456789012345";

        public HttpClientGenerator() {
            Registry<ConnectionSocketFactory> reg = RegistryBuilder.<ConnectionSocketFactory> create()
                    .register("http", PlainConnectionSocketFactory.INSTANCE)
                    .register("https", SSLConnectionSocketFactory.getSocketFactory()).build();
            connectionManager = new PoolingHttpClientConnectionManager(reg);
            connectionManager.setDefaultMaxPerRoute(100);
        }

        public HttpClientGenerator setPoolSize(int poolSize) {
            connectionManager.setMaxTotal(poolSize);
            return this;
        }

        public CloseableHttpClient getClient(Site site) {
            return generateClient(site);
        }

        private CloseableHttpClient generateClient(Site site) {
            List<Header> list = new ArrayList<Header>();
            BasicHeader header = new BasicHeader("Proxy-Switch-Ip", "yes");
            list.add(header);
            if (site.getHeaders() != null) {
                for (Map.Entry<String, String> headerEntry : site.getHeaders().entrySet()) {
                    list.add(new BasicHeader(headerEntry.getKey(), headerEntry.getValue()));
                }
            }

            HttpHost target = site.getHttpProxy();
            CredentialsProvider credsProvider = new BasicCredentialsProvider();
            credsProvider.setCredentials(new AuthScope(target.getHostName(), target.getPort()),
                    new UsernamePasswordCredentials(proxyUser, proxyPass));

            HttpClientBuilder httpClientBuilder = HttpClients.custom().setConnectionManager(connectionManager)
                    .setDefaultCredentialsProvider(credsProvider).setDefaultHeaders(list);

            if (site != null && site.getUserAgent() != null) {
                httpClientBuilder.setUserAgent(site.getUserAgent());
            } else {
                httpClientBuilder.setUserAgent("");
            }

            if (site == null || site.isUseGzip()) {
                httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
                    public void process(final HttpRequest request, final HttpContext context)
                            throws HttpException, IOException {
                        if (!request.containsHeader("Accept-Encoding")) {
                            request.addHeader("Accept-Encoding", "gzip");
                        }
                    }
                });
            }

            SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
            httpClientBuilder.setDefaultSocketConfig(socketConfig);

            if (site != null) {
                httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
            }

            generateCookie(httpClientBuilder, site);

            return httpClientBuilder.build();
        }

        private void generateCookie(HttpClientBuilder httpClientBuilder, Site site) {
            CookieStore cookieStore = new BasicCookieStore();

            for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
                BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
                cookie.setDomain(site.getDomain());
                cookieStore.addCookie(cookie);
            }

            for (Map.Entry<String, Map<String, String>> domainEntry : site.getAllCookies().entrySet()) {
                for (Map.Entry<String, String> cookieEntry : domainEntry.getValue().entrySet()) {
                    BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
                    cookie.setDomain(domainEntry.getKey());
                    cookieStore.addCookie(cookie);
                }
            }

            httpClientBuilder.setDefaultCookieStore(cookieStore);
        }
    }              
            
体验服务
现在开始,体验阿布云大数据服务
计算、网络、大数据、人工智能,阿布云助您飞跃发展
  • · 功能和特性
  • · 价格和优惠
  • · 获取帮助
  • · 功能和特性
  • · 价格和优惠
  • · 获取帮助