HTTP隧道 （动态版）
Java 接入指南
地址：http-dyn.abuyun.com，端口：9020
HTTP隧道拥有两种授权模式：
通过用户名/密码的形式进行身份认证，该认证信息最终会转换为『Proxy-Authorization』协议头跟随请求一起发出。
为便于部分语言进行接入，平台亦支持通过『Authorization』协议头进行隧道身份验证。
只须绑定用户发起请求的服务器IP即可。
一条代理隧道只能绑定一个IP，同一IP可以分别绑定到专业版、动态版、经典版代理隧道各一条。
# -*-*-
# 感谢骚男 『苗伦兽 (QQ: 519714803)』 提供的源代码
# -*-*-
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.URL;
class ProxyAuthenticator extends Authenticator {
private String user, password;
public ProxyAuthenticator(String user, String password) {
this.user = user;
this.password = password;
}
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(user, password.toCharArray());
}
}
/**
* 注意：下面代码仅仅实现HTTP请求链接，每一次请求都是无状态保留的，仅仅是这次请求是更换IP的，如果下次请求的IP地址会改变
* 如果是多线程访问的话，只要将下面的代码嵌入到你自己的业务逻辑里面，那么每次都会用新的IP进行访问，如果担心IP有重复，
* 自己可以维护IP的使用情况，并做校验。
*
* JDK 8u111版本后环境下：要访问的目标页面为HTTPS协议时，需修改“jdk.http.auth.tunneling.disabledSchemes”值
*/
public class ProxyTest {
public static void main(String args[]) throws Exception {
// 要访问的目标页面
String targetUrl = "http://test.abuyun.com";
// JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
// 代理服务器
String proxyServer = "http-dyn.abuyun.com";
int proxyPort = 9020;
// 代理隧道验证信息
String proxyUser = "H01234567890123D";
String proxyPass = "0123456789012345";
try {
URL url = new URL(targetUrl);
Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
// 设置通过代理访问目标页面
HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
// 解析返回数据
byte[] response = readStream(connection.getInputStream());
System.out.println(new String(response));
} catch (Exception e) {
System.out.println(e.getLocalizedMessage());
}
}
/**
* 将输入流转换成字符串
*
* @param inStream
* @return
* @throws Exception
*/
public static byte[] readStream(InputStream inStream) throws Exception {
ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int len = -1;
while ((len = inStream.read(buffer)) != -1) {
outSteam.write(buffer, 0, len);
}
outSteam.close();
inStream.close();
return outSteam.toByteArray();
}
}
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.ProxyAuthenticationStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.NameValuePair;
import org.apache.http.util.EntityUtils;
public class JavaHttpClient45Demo
{
// 代理服务器
final static String proxyHost = "http-dyn.abuyun.com";
final static Integer proxyPort = 9020;
// 代理隧道验证信息
final static String proxyUser = "H01234567890123D";
final static String proxyPass = "0123456789012345";
private static PoolingHttpClientConnectionManager cm = null;
private static HttpRequestRetryHandler httpRequestRetryHandler = null;
private static HttpHost proxy = null;
private static CredentialsProvider credsProvider = null;
private static RequestConfig reqConfig = null;
static {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
cm = new PoolingHttpClientConnectionManager(registry);
cm.setMaxTotal(20);
cm.setDefaultMaxPerRoute(5);
proxy = new HttpHost(proxyHost, proxyPort, "http");
credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPass));
reqConfig = RequestConfig.custom()
.setConnectionRequestTimeout(5000)
.setConnectTimeout(5000)
.setSocketTimeout(5000)
.setExpectContinueEnabled(false)
.setProxy(new HttpHost(proxyHost, proxyPort))
.build();
}
public static void doRequest(HttpRequestBase httpReq) {
CloseableHttpResponse httpResp = null;
try {
// JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
setHeaders(httpReq);
httpReq.setConfig(reqConfig);
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setDefaultCredentialsProvider(credsProvider)
.build();
AuthCache authCache = new BasicAuthCache();
authCache.put(proxy, new BasicScheme());
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
httpResp = httpClient.execute(httpReq, localContext);
int statusCode = httpResp.getStatusLine().getStatusCode();
System.out.println(statusCode);
BufferedReader rd = new BufferedReader(new InputStreamReader(httpResp.getEntity().getContent()));
String line = "";
while((line = rd.readLine()) != null) {
System.out.println(line);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (httpResp != null) {
httpResp.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 设置请求头
*
* @param httpReq
*/
private static void setHeaders(HttpRequestBase httpReq) {
httpReq.setHeader("Accept-Encoding", null);
httpReq.setHeader(switchIpHeaderKey, switchIpHeaderVal);
}
public static void doPostRequest() {
try {
// 要访问的目标页面
HttpPost httpPost = new HttpPost("https://test.abuyun.com");
// 设置表单参数
List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("method", "next"));
params.add(new BasicNameValuePair("params", "{\"broker\":\"abuyun\":\"site\":\"https://www.abuyun.com\"}"));
httpPost.setEntity(new UrlEncodedFormEntity(params, "utf-8"));
doRequest(httpPost);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void doGetRequest() {
// 要访问的目标页面
String targetUrl = "https://test.abuyun.com";
try {
HttpGet httpGet = new HttpGet(targetUrl);
doRequest(httpGet);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
doGetRequest();
//doPostRequest();
}
}
package htmlunit;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class HtmlunitDemo {
// 代理服务器
final static String proxyHost = "http-dyn.abuyun.com";
final static Integer proxyPort = 9020;
// 代理隧道验证信息
final static String proxyUser = "H01234567890123D";
final static String proxyPass = "0123456789012345";
public static void main(String[] args) {
// 第一步：创建凭证对象 credsProvider
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
//只对proxyHost:proxyPort使用此凭证
new AuthScope(proxyHost, proxyPort),
new UsernamePasswordCredentials(proxyUser, proxyPass));
// 第二步：创建WebClient对象，设置代理服务器及端口
WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);
// 第三步：设置WebClient的凭证
webClient.setCredentialsProvider(credsProvider);
// 设置隧道代理到此结束，后面是请求的代码了
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
webClient.getOptions().setJavaScriptEnabled(true);
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
webClient.getOptions().setActiveXNative(false);
webClient.getOptions().setCssEnabled(false);
HtmlPage page = null;
try {
page = webClient.getPage("http://test.abuyun.com");
} catch (Exception e) {
e.printStackTrace();
} finally {
webClient.close();
}
webClient.waitForBackgroundJavaScript(30000);
// 直接将加载完成的页面转换成xml格式的字符串
String pageXml = page.asXml();
System.out.println(pageXml);
}
}
# -*-*-
# 感谢骚男 『不蛰不鸣』 提供的源代码
# -*-*-
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
/***
* 通过阿布云代理访问指定URL 内容
* 此处 Jsoup Version 1.9.1
* @author sun
* JDK 8u111版本后环境下：要访问的目标页面为HTTPS协议时，需修改“jdk.http.auth.tunneling.disabledSchemes”值
*/
public class ProxyDemo
{
// 代理隧道验证信息
final static String ProxyUser = "H01234567890123D";
final static String ProxyPass = "0123456789012345";
// 代理服务器
final static String ProxyHost = "http-dyn.abuyun.com";
final static Integer ProxyPort = 9020;
public static String getUrlProxyContent(String url)
{
Authenticator.setDefault(new Authenticator() {
public PasswordAuthentication getPasswordAuthentication()
{
return new PasswordAuthentication(ProxyUser, ProxyPass.toCharArray());
}
});
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ProxyHost, ProxyPort));
try
{
// 此处自己处理异常、其他参数等
Document doc = Jsoup.connect(url)
.followRedirects(false)
.timeout(3000)
.header(ProxyHeadKey, ProxyHeadVal)
.proxy(proxy)
// 忽略证书校验
.validateTLSCertificates(false)
.get()
;
if(doc != null) {
System.out.println(doc.body().html());
}
}
catch (IOException e)
{
e.printStackTrace();
}
return null;
}
public static void main(String[] args) throws Exception
{
// 要访问的目标页面
String targetUrl = "http://test.abuyun.com";
// JDK 8u111版本后，目标页面为HTTPS协议，启用proxy用户密码鉴权
System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
getUrlProxyContent(targetUrl);
}
}
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
/**
* 基于WebMagic 0.7.3版本，关于WebMagic代理详细设置请看WebMagic官方文档
**/
public class WebMagicProxyDemo implements PageProcessor {
private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
@Override
public void process(Page page) {
page.putField("html", page.getHtml());
}
@Override
public Site getSite() {
return site;
}
public static void main(String[] args) {
// 代理服务器
final String proxyHost = "http-dyn.abuyun.com";
final Integer proxyPort = 9020;
// 代理隧道验证信息
final String proxyUser = "H01234567890123D";
final String proxyPass = "0123456789012345";
final String targetUrl ="http://test.abuyun.com/";
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy(proxyHost,proxyPort,proxyUser,proxyPass)));
Spider.create(new WebMagic73ProxyDemo()).setDownloader(httpClientDownloader).addUrl(targetUrl).thread(5).run();
}
}
