HTTP隧道 （动态版）
Java 接入指南
地址：http-dyn.abuyun.com，端口：9020
HTTP隧道拥有两种授权模式：
通过用户名/密码的形式进行身份认证，该认证信息最终会转换为『Proxy-Authorization』协议头跟随请求一起发出。
为便于部分语言进行接入，平台亦支持通过『Authorization』协议头进行隧道身份验证。
只须绑定用户发起请求的服务器IP即可。
一条代理隧道只能绑定一个IP，同一IP可以分别绑定到专业版、动态版、经典版代理隧道各一条。
# -*-*-
# 感谢骚男 『苗伦兽 (QQ: 519714803)』 提供的源代码
# -*-*-
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.URL;
class ProxyAuthenticator extends Authenticator {
private String user, password;
public ProxyAuthenticator(String user, String password) {
this.user = user;
this.password = password;
}
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(user, password.toCharArray());
}
}
/**
* 注意：下面代码仅仅实现HTTP请求链接，每一次请求都是无状态保留的，仅仅是这次请求是更换IP的，如果下次请求的IP地址会改变
* 如果是多线程访问的话，只要将下面的代码嵌入到你自己的业务逻辑里面，那么每次都会用新的IP进行访问，如果担心IP有重复，
* 自己可以维护IP的使用情况，并做校验。
* JDK 8u111版本后环境下：要访问的目标页面为HTTPS协议时，需修改“jdk.http.auth.tunneling.disabledSchemes”值
*/
public class ProxyTest {
public static void main(String args[]) throws Exception {
// 要访问的目标页面
String targetUrl = "http://test.abuyun.com";
//JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
//System.setProperty("jdk.http.auth.tunneling.disabledSchemes","");
//String targetUrl = "http://proxy.abuyun.com/switch-ip";
//String targetUrl = "http://proxy.abuyun.com/current-ip";
// 代理服务器
String proxyServer = "http-dyn.abuyun.com";
int proxyPort = 9020;
// 代理隧道验证信息
String proxyUser = "H01234567890123D";
String proxyPass = "0123456789012345";
try {
URL url = new URL(targetUrl);
Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
// 设置通过代理访问目标页面
HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
// 设置IP切换头
connection.setRequestProperty("Proxy-Switch-Ip","yes");
// 解析返回数据
byte[] response = readStream(connection.getInputStream());
System.out.println(new String(response));
} catch (Exception e) {
System.out.println(e.getLocalizedMessage());
}
}
/**
* 将输入流转换成字符串
*
* @param inStream
* @return
* @throws Exception
*/
public static byte[] readStream(InputStream inStream) throws Exception {
ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int len = -1;
while ((len = inStream.read(buffer)) != -1) {
outSteam.write(buffer, 0, len);
}
outSteam.close();
inStream.close();
return outSteam.toByteArray();
}
}
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.ProxyAuthenticationStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.NameValuePair;
import org.apache.http.util.EntityUtils;
public class JavaHttpClient45Demo
{
// 代理服务器
final static String proxyHost = "http-dyn.abuyun.com";
final static Integer proxyPort = 9020;
// 代理隧道验证信息
final static String proxyUser = "H01234567890123D";
final static String proxyPass = "0123456789012345";
// IP切换协议头
final static String switchIpHeaderKey = "Proxy-Switch-Ip";
final static String switchIpHeaderVal = "yes";
private static PoolingHttpClientConnectionManager cm = null;
private static HttpRequestRetryHandler httpRequestRetryHandler = null;
private static HttpHost proxy = null;
private static CredentialsProvider credsProvider = null;
private static RequestConfig reqConfig = null;
static {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
Registry registry = RegistryBuilder.create()
.register("http", plainsf)
.register("https", sslsf)
.build();
cm = new PoolingHttpClientConnectionManager(registry);
cm.setMaxTotal(20);
cm.setDefaultMaxPerRoute(5);
proxy = new HttpHost(proxyHost, proxyPort, "http");
credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPass));
reqConfig = RequestConfig.custom()
.setConnectionRequestTimeout(5000)
.setConnectTimeout(5000)
.setSocketTimeout(5000)
.setExpectContinueEnabled(false)
.setProxy(new HttpHost(proxyHost, proxyPort))
.build();
}
public static void doRequest(HttpRequestBase httpReq) {
CloseableHttpResponse httpResp = null;
try {
setHeaders(httpReq);
httpReq.setConfig(reqConfig);
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setDefaultCredentialsProvider(credsProvider)
.build();
AuthCache authCache = new BasicAuthCache();
authCache.put(proxy, new BasicScheme());
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
httpResp = httpClient.execute(httpReq, localContext);
int statusCode = httpResp.getStatusLine().getStatusCode();
System.out.println(statusCode);
BufferedReader rd = new BufferedReader(new InputStreamReader(httpResp.getEntity().getContent()));
String line = "";
while((line = rd.readLine()) != null) {
System.out.println(line);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (httpResp != null) {
httpResp.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 设置请求头
*
* @param httpReq
*/
private static void setHeaders(HttpRequestBase httpReq) {
httpReq.setHeader("Accept-Encoding", null);
httpReq.setHeader(switchIpHeaderKey, switchIpHeaderVal);
}
public static void doPostRequest() {
try {
// 要访问的目标页面
HttpPost httpPost = new HttpPost("https://test.abuyun.com");
// 设置表单参数
List params = new ArrayList();
params.add(new BasicNameValuePair("method", "next"));
params.add(new BasicNameValuePair("params", "{\"broker\":\"abuyun\":\"site\":\"https://www.abuyun.com\"}"));
httpPost.setEntity(new UrlEncodedFormEntity(params, "utf-8"));
doRequest(httpPost);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void doGetRequest() {
// 要访问的目标页面
String targetUrl = "https://test.abuyun.com";
//String targetUrl = "http://proxy.abuyun.com/switch-ip";
//String targetUrl = "http://proxy.abuyun.com/current-ip";
try {
HttpGet httpGet = new HttpGet(targetUrl);
doRequest(httpGet);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
doGetRequest();
//doPostRequest();
}
}
package htmlunit;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class HtmlunitDemo {
// 代理服务器
final static String proxyHost = "http-dyn.abuyun.com";
final static Integer proxyPort = 9020;
// 代理隧道验证信息
final static String proxyUser = "H01234567890123D";
final static String proxyPass = "0123456789012345";
public static void main(String[] args) {
// 第一步：创建凭证对象 credsProvider
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
//只对proxyHost:proxyPort使用此凭证
new AuthScope(proxyHost, proxyPort),
new UsernamePasswordCredentials(proxyUser, proxyPass));
// 第二步：创建WebClient对象，设置代理服务器及端口
WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);
// 第三步：设置WebClient的凭证
webClient.setCredentialsProvider(credsProvider);
// 设置隧道代理到此结束，后面是请求的代码了
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
webClient.getOptions().setJavaScriptEnabled(true);
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
webClient.getOptions().setActiveXNative(false);
webClient.getOptions().setCssEnabled(false);
HtmlPage page = null;
try {
page = webClient.getPage("http://test.abuyun.com");
} catch (Exception e) {
e.printStackTrace();
} finally {
webClient.close();
}
webClient.waitForBackgroundJavaScript(30000);
// 直接将加载完成的页面转换成xml格式的字符串
String pageXml = page.asXml();
System.out.println(pageXml);
}
}
# -*-*-
# 感谢骚男 『不蛰不鸣』 提供的源代码
# -*-*-
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
/***
* 通过阿布云代理访问指定URL 内容
* 此处 Jsoup Version 1.9.1
* @author sun
* JDK 8u111版本后环境下：要访问的目标页面为HTTPS协议时，需修改“jdk.http.auth.tunneling.disabledSchemes”值
*/
public class ProxyDemo
{
// 代理隧道验证信息
final static String ProxyUser = "H01234567890123D";
final static String ProxyPass = "0123456789012345";
// 代理服务器
final static String ProxyHost = "http-dyn.abuyun.com";
final static Integer ProxyPort = 9020;
// 设置IP切换头
final static String ProxyHeadKey = "Proxy-Switch-Ip";
final static String ProxyHeadVal = "yes";
public static String getUrlProxyContent(String url)
{
Authenticator.setDefault(new Authenticator() {
public PasswordAuthentication getPasswordAuthentication()
{
return new PasswordAuthentication(ProxyUser, ProxyPass.toCharArray());
}
});
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ProxyHost, ProxyPort));
try
{
// 此处自己处理异常、其他参数等
Document doc = Jsoup.connect(url)
.followRedirects(false)
.timeout(3000)
.header(ProxyHeadKey, ProxyHeadVal)
.proxy(proxy)
.validateTLSCertificates(false)
.get()
;
if(doc != null) {
System.out.println(doc.body().html());
}
}
catch (IOException e)
{
e.printStackTrace();
}
return null;
}
public static void main(String[] args) throws Exception
{
// 要访问的目标页面
String targetUrl = "http://test.abuyun.com";
//JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
//System.setProperty("jdk.http.auth.tunneling.disabledSchemes","");
//String targetUrl = "http://proxy.abuyun.com/switch-ip";
//String targetUrl = "http://proxy.abuyun.com/current-ip";
getUrlProxyContent(targetUrl);
}
}
# -*-*-
# 感谢骚男 『黑白之间 (QQ: 420269696)』 提供的源代码
# -*-*-
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.annotation.ThreadSafe;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CookieStore;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.UrlUtils;
/**
* webmagic 0.5.3版本接入阿布云代理需要自定义下载组件，0.5.4版本作者代理池功能进行了改进并且加入了代理身份认证，推荐使用0.5.4版本，
* 或许不用自定义下载组件
*/
public class WebMagicProxyDemo
{
public static void main(String[] args) {
Spider.create(new PageProcessor() {
@Override
public void process(Page page) {
System.out.println(page.getHtml());
}
@Override
public Site getSite() {
return Site.me();
}
}).setDownloader(new MyDownloader()).addUrl("https://test.abuyun.com").thread(1).run();
}
}
@ThreadSafe
class MyDownloader implements Downloader
{
// 代理服务器
final static String proxyHost = "http-dyn.abuyun.com";
final static Integer proxyPort = 9020;
private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
@Override
public void setThread(int threadNum) {}
@Override
public Page download(Request request, Task task) {
Site site = task.getSite();
Set<Integer> acceptStatCode = site.getAcceptStatCode();
String charset = site.getCharset();
CloseableHttpResponse httpResponse = null;
int statusCode = 0;
try {
HttpHost target = new HttpHost(proxyHost, proxyPort, "http");
site.setHttpProxy(target);
request.putExtra(Request.PROXY, target);
HttpClientContext localContext = HttpClientContext.create();
AuthCache authCache = new BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
authCache.put(site.getHttpProxy(), basicAuth);
localContext.setAuthCache(authCache);
HttpGet httpGet = new HttpGet(request.getUrl());
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut()).setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut()).setCookieSpec(CookieSpecs.IGNORE_COOKIES);
httpGet.setConfig(requestConfigBuilder.build());
httpResponse = getHttpClient(site).execute(target, httpGet, localContext);
statusCode = httpResponse.getStatusLine().getStatusCode();
request.putExtra(Request.STATUS_CODE, statusCode);
if (statusAccept(acceptStatCode, statusCode)) {
Page page = handleResponse(request, charset, httpResponse, task);
onSuccess(request);
return page;
} else {
return null;
}
} catch (IOException e) {
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
onError(request);
return null;
} finally {
request.putExtra(Request.STATUS_CODE, statusCode);
try {
if (httpResponse != null) {
EntityUtils.consume(httpResponse.getEntity());
}
}
catch (IOException e) {
}
}
}
private CloseableHttpClient getHttpClient(Site site) {
String domain = site.getDomain();
CloseableHttpClient httpClient = httpClients.get(domain);
if (httpClient == null) {
synchronized (this) {
httpClient = httpClients.get(domain);
if (httpClient == null) {
httpClient = httpClientGenerator.getClient(site);
httpClients.put(domain, httpClient);
}
}
}
return httpClient;
}
protected boolean statusAccept(Set<Integer> acceptStatCode, int statusCode) {
return acceptStatCode.contains(statusCode);
}
protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task)
throws IOException {
String content = getContent(charset, httpResponse);
Page page = new Page();
page.setRawText(content);
page.setUrl(new PlainText(request.getUrl()));
page.setRequest(request);
page.setStatusCode(httpResponse.getStatusLine().getStatusCode());
return page;
}
protected String getContent(String charset, HttpResponse httpResponse) throws IOException {
if (charset == null) {
byte[] contentBytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
String htmlCharset = getHtmlCharset(httpResponse, contentBytes);
if (htmlCharset != null) {
return new String(contentBytes, htmlCharset);
} else {
return new String(contentBytes);
}
} else {
return IOUtils.toString(httpResponse.getEntity().getContent(), charset);
}
}
protected String getHtmlCharset(HttpResponse httpResponse, byte[] contentBytes) throws IOException {
String charset = null;
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
if (StringUtils.isNotBlank(charset)) {
return charset;
}
Charset defaultCharset = Charset.defaultCharset();
String content = new String(contentBytes, defaultCharset.name());
if (StringUtils.isNotEmpty(content)) {
Document document = Jsoup.parse(content);
Elements links = document.select("meta");
for (Element link : links) {
String metaContent = link.attr("content");
String metaCharset = link.attr("charset");
if (metaContent.indexOf("charset") != -1) {
metaContent = metaContent.substring(metaContent.indexOf("charset"), metaContent.length());
charset = metaContent.split("=")[1];
break;
} else if (StringUtils.isNotEmpty(metaCharset)) {
charset = metaCharset;
break;
}
}
}
return charset;
}
protected void onSuccess(Request request) {}
protected void onError(Request request) {}
protected Page addToCycleRetry(Request request, Site site) {
Page page = new Page();
Object cycleTriedTimesObject = request.getExtra(Request.CYCLE_TRIED_TIMES);
if (cycleTriedTimesObject == null) {
page.addTargetRequest(request.setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, 1));
} else {
int cycleTriedTimes = (Integer) cycleTriedTimesObject;
cycleTriedTimes++;
if (cycleTriedTimes >= site.getCycleRetryTimes()) {
return null;
}
page.addTargetRequest(request.setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, cycleTriedTimes));
}
page.setNeedCycleRetry(true);
return page;
}
}
class HttpClientGenerator
{
private PoolingHttpClientConnectionManager connectionManager;
// 代理隧道验证信息
final static String proxyUser = "H01234567890123D";
final static String proxyPass = "0123456789012345";
public HttpClientGenerator() {
Registry<ConnectionSocketFactory> reg = RegistryBuilder.<ConnectionSocketFactory> create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", SSLConnectionSocketFactory.getSocketFactory()).build();
connectionManager = new PoolingHttpClientConnectionManager(reg);
connectionManager.setDefaultMaxPerRoute(100);
}
public HttpClientGenerator setPoolSize(int poolSize) {
connectionManager.setMaxTotal(poolSize);
return this;
}
public CloseableHttpClient getClient(Site site) {
return generateClient(site);
}
private CloseableHttpClient generateClient(Site site) {
List<Header> list = new ArrayList<Header>();
BasicHeader header = new BasicHeader("Proxy-Switch-Ip", "yes");
list.add(header);
if (site.getHeaders() != null) {
for (Map.Entry<String, String> headerEntry : site.getHeaders().entrySet()) {
list.add(new BasicHeader(headerEntry.getKey(), headerEntry.getValue()));
}
}
HttpHost target = site.getHttpProxy();
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(target.getHostName(), target.getPort()),
new UsernamePasswordCredentials(proxyUser, proxyPass));
HttpClientBuilder httpClientBuilder = HttpClients.custom().setConnectionManager(connectionManager)
.setDefaultCredentialsProvider(credsProvider).setDefaultHeaders(list);
if (site != null && site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site == null || site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
if (site != null) {
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
}
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
private void generateCookie(HttpClientBuilder httpClientBuilder, Site site) {
CookieStore cookieStore = new BasicCookieStore();
for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
cookie.setDomain(site.getDomain());
cookieStore.addCookie(cookie);
}
for (Map.Entry<String, Map<String, String>> domainEntry : site.getAllCookies().entrySet()) {
for (Map.Entry<String, String> cookieEntry : domainEntry.getValue().entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
cookie.setDomain(domainEntry.getKey());
cookieStore.addCookie(cookie);
}
}
httpClientBuilder.setDefaultCookieStore(cookieStore);
}
}
