JAVA

:::warning[通过代理访问HTTP2网站]
需要保证JDK的版本支持HTTP2网站的访问，java9已经以上才能完整支持
:::

:::danger[407错误]
// Change in Java 8 Update 111 以上版本需要下面代码 System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "false"); System.setProperty("jdk.http.auth.proxying.disabledSchemes", "false");
:::

import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;

import java.io.IOException;

public class Main {
    # 代理服务器(产品官网 www.duoip.cn)
    private static final String PROXY_HOST = "tunnel.hahado.cn";
    private static final int PROXY_PORT = 31111;

    public static void main(String[] args) {
        HttpClient client = new HttpClient();
        HttpMethod method = new GetMethod("https://httpbin.org/ip");

        HostConfiguration config = client.getHostConfiguration();
        config.setProxy(PROXY_HOST, PROXY_PORT);

        client.getParams().setAuthenticationPreemptive(true);

        String username = "haABCCKJ";
        String password = "712323";
        Credentials credentials = new UsernamePasswordCredentials(username, password);
        AuthScope authScope = new AuthScope(PROXY_HOST, PROXY_PORT);

        client.getState().setProxyCredentials(authScope, credentials);

        try {
            client.executeMethod(method);

            if (method.getStatusCode() == HttpStatus.SC_OK) {
                String response = method.getResponseBodyAsString();
                System.out.println("Response = " + response);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            method.releaseConnection();
        }
    }
}

 

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.ProxyAuthenticationStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.NameValuePair;
import org.apache.http.util.EntityUtils;

public class Demo
{
    // 代理服务器(产品官网 www.duoip.cn)
    final static String proxyHost = "tunnel.hahado.cn";
    final static Integer proxyPort = 31000;

    // 代理验证信息
    final static String proxyUser = "username";
    final static String proxyPass = "password";




    private static PoolingHttpClientConnectionManager cm = null;
    private static HttpRequestRetryHandler httpRequestRetryHandler = null;
    private static HttpHost proxy = null;

    private static CredentialsProvider credsProvider = null;
    private static RequestConfig reqConfig = null;

    static {
        ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
        LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();

        Registry registry = RegistryBuilder.create()
            .register("http", plainsf)
            .register("https", sslsf)
            .build();

        cm = new PoolingHttpClientConnectionManager(registry);
        cm.setMaxTotal(20);
        cm.setDefaultMaxPerRoute(5);

        proxy = new HttpHost(proxyHost, proxyPort, "http");

        credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPass));

        reqConfig = RequestConfig.custom()
            .setConnectionRequestTimeout(5000)
            .setConnectTimeout(5000)
            .setSocketTimeout(5000)
            .setExpectContinueEnabled(false)
            .setProxy(new HttpHost(proxyHost, proxyPort))
            .build();
    }

    public static void doRequest(HttpRequestBase httpReq) {
        CloseableHttpResponse httpResp = null;

        try {
            setHeaders(httpReq);

            httpReq.setConfig(reqConfig);

            CloseableHttpClient httpClient = HttpClients.custom()
                .setConnectionManager(cm)
                .setDefaultCredentialsProvider(credsProvider)
                .build();

            //设置TCP keep alive，访问https网站时保持IP不切换
            // SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setSoTimeout(3600000).build(); 
            // CloseableHttpClient httpClient =  HttpClients.custom()
            //    .setConnectionManager(cm)
            //    .setDefaultCredentialsProvider(credsProvider)
            //    .setDefaultSocketConfig(socketConfig)
            //    .build();


            AuthCache authCache = new BasicAuthCache();
            authCache.put(proxy, new BasicScheme());
            // 如果遇到407,可以设置代理认证 Proxy-Authenticate
            // authCache.put(proxy, new BasicScheme(ChallengeState.PROXY));

            HttpClientContext localContext = HttpClientContext.create();
            localContext.setAuthCache(authCache);

            httpResp = httpClient.execute(httpReq, localContext);

            int statusCode = httpResp.getStatusLine().getStatusCode();

            System.out.println(statusCode);

            BufferedReader rd = new BufferedReader(new InputStreamReader(httpResp.getEntity().getContent()));

            String line = "";
            while((line = rd.readLine()) != null) {
                System.out.println(line);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (httpResp != null) {
                    httpResp.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 设置请求头
     *
     * @param httpReq
     */
    private static void setHeaders(HttpRequestBase httpReq) {

        // 设置Proxy-Tunnel
        // Random random = new Random();
        // int tunnel = random.nextInt(10000);
        // httpReq.setHeader("Proxy-Tunnel", String.valueOf(tunnel));

        httpReq.setHeader("Accept-Encoding", null);

    }


    public static void doGetRequest() {
        // 要访问的目标页面
        String targetUrl = "https://httpbin.org/ip";


        try {
            HttpGet httpGet = new HttpGet(targetUrl);

            doRequest(httpGet);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        doGetRequest();


    }
}

:::warning[JSoup无法使用Keep-alive]
JSoup默认会关闭连接访问HTTP网站请通过设置相同Proxy-Tunnel来保持相同的外网IP. 访问HTTPS网站请使用其他库,保持相同的外网IP.
:::

import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;


public class Demo
{
    // 代理验证信息
    final static String ProxyUser = "username";
    final static String ProxyPass = "password";

    // 代理服务器(产品官网 www.duoip.cn)
    final static String ProxyHost = "tunnel.hahado.cn";
    final static Integer ProxyPort = 31111;

    // 设置IP切换头
    final static String ProxyHeadKey = "Proxy-Tunnel";


    public static String getUrlProxyContent(String url)
    {
        Authenticator.setDefault(new Authenticator() {
            public PasswordAuthentication getPasswordAuthentication()
            {
                return new PasswordAuthentication(ProxyUser, ProxyPass.toCharArray());
            }
        });
        // 设置Proxy-Tunnel
        Random random = new Random();
        int tunnel = random.nextInt(10000);
        String ProxyHeadVal = String.valueOf(tunnel);

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ProxyHost, ProxyPort));

        try
        {
            // 处理异常、其他参数
            Document doc = Jsoup.connect(url).timeout(3000).header(ProxyHeadKey, ProxyHeadVal).proxy(proxy).get();

            if(doc != null) {
                System.out.println(doc.body().html());
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }

        return null;
    }

    public static void main(String[] args) throws Exception
    {
        // 要访问的目标页面
        String targetUrl = "http://httpbin.org/ip";


        getUrlProxyContent(targetUrl);
    }
}

import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

public class Demo {

    public static void main(String[] args) {

        try{

            // 代理服务器(产品官网 www.duoip.cn)
            final static String ProxyHost = "tunnel.hahado.cn";
            final static String ProxyPort = "31111";

            System.setProperty("http.proxyHost", ProxyHost);
            System.setProperty("https.proxyHost", ProxyHost);

            System.setProperty("http.proxyPort", ProxyPort);
            System.setProperty("https.proxyPort", ProxyPort);

            // 代理验证信息
            final static String ProxyUser = "username";
            final static String ProxyPass = "password";

            System.setProperty("http.proxyUser", ProxyUser);
            System.setProperty("http.proxyPassword", ProxyPass);

            System.setProperty("https.proxyUser", ProxyUser);
            System.setProperty("https.proxyPassword", ProxyPass);




            // 设置IP切换头
            final static String ProxyHeadKey = "Proxy-Tunnel";

            // 设置Proxy-Tunnel
            Random random = new Random();
            int tunnel = random.nextInt(10000);
            String ProxyHeadVal = String.valueOf(tunnel);



            // 处理异常、其他参数
            Document doc = Jsoup.connect(url).timeout(3000).header(ProxyHeadKey, ProxyHeadVal).get();

            if(doc != null) {
                System.out.println(doc.body().html());
            }

        }catch (IOException e)
        {
            e.printStackTrace();
        }

    }
}

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.URL;
import java.util.Random;

class ProxyAuthenticator extends Authenticator {
    private String user, password;

    public ProxyAuthenticator(String user, String password) {
        this.user     = user;
        this.password = password;
    }

    protected PasswordAuthentication getPasswordAuthentication() {
        return new PasswordAuthentication(user, password.toCharArray());
    }
}

/**
 * 注意：下面代码仅仅实现HTTP请求链接，每一次请求都是无状态保留的，仅仅是这次请求是更换IP的，如果下次请求的IP地址会改变
 * 如果是多线程访问的话，只要将下面的代码嵌入到你自己的业务逻辑里面，那么每次都会用新的IP进行访问，如果担心IP有重复，
 * 自己可以维护IP的使用情况，并做校验。
 */
public class Demo {
    public static void main(String args[]) throws Exception {
        // Change in Java 8 Update 111 以上版本需要下面代码
        // System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "false");
        // System.setProperty("jdk.http.auth.proxying.disabledSchemes", "false");

        // 要访问的目标页面
        String targetUrl = "http://httpbin.org/ip";


        // 代理服务器(产品官网 www.duoip.cn)
        String proxyServer = "tunnel.hahado.cn";
        int proxyPort      = 31111;

        // 代理验证信息
        String proxyUser  = "username";
        String proxyPass  = "password";

        try {
            URL url = new URL(targetUrl);

            Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));

            // 创建代理服务器地址对象
            InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
            // 创建HTTP类型代理对象
            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);

            // 设置通过代理访问目标页面
            HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);

            // 设置KeepAlive
            // connection.setRequestProperty("Connection", "keep-alive");
            // connection.setRequestProperty("Keep-Alive", "timeout=5, max=100");                

            // 设置Proxy-Tunnel
            // Random random = new Random();
            // int tunnel = random.nextInt(10000);
            // connection.setRequestProperty("Proxy-Tunnel",String.valueOf(tunnel));

            // 解析返回数据
            byte[] response = readStream(connection.getInputStream());

            System.out.println(new String(response));
        } catch (Exception e) {
            System.out.println(e.getLocalizedMessage());
        }
    }

    /**
     * 将输入流转换成字符串
     *
     * @param inStream
     * @return
     * @throws Exception
     */
    public static byte[] readStream(InputStream inStream) throws Exception {
        ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
        byte[] buffer = new byte[1024];
        int len = -1;

        while ((len = inStream.read(buffer)) != -1) {
            outSteam.write(buffer, 0, len);
        }
        outSteam.close();
        inStream.close();

        return outSteam.toByteArray();
    }
}

package htmlunit;

import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class HtmlunitDemo {
    // 代理服务器(产品官网 www.duoip.cn)
    final static String proxyHost = "tunnel.hahado.cn";
    final static Integer proxyPort = 31111;

    // 代理验证信息
    final static String proxyUser = "USERNAME";
    final static String proxyPass = "PASSWORD";

    public static void main(String[] args) {

        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(

        new AuthScope(proxyHost, proxyPort),
        new UsernamePasswordCredentials(proxyUser, proxyPass));


        WebClient webClient = new WebClient(BrowserVersion.CHROME,proxyHost, proxyPort);


        webClient.setCredentialsProvider(credsProvider);


        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setActiveXNative(false);
        webClient.getOptions().setCssEnabled(false);

        HtmlPage page = null;

        try {
            page = webClient.getPage("http://httpbin.org/ip");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            webClient.close();
        }

        webClient.waitForBackgroundJavaScript(30000);


        String pageXml = page.asXml();

        System.out.println(pageXml);
    }
}

    import okhttp3.*;

    import java.io.IOException;
    import java.net.InetSocketAddress;
    import java.net.Proxy;
    import java.util.concurrent.TimeUnit;

    public class OkHttp {

        // 代理服务器(产品官网 www.duoip.cn)
        final static String proxyHost = "tunnel.hahado.cn";
        final static Integer proxyPort = 31111;

        // 代理验证信息
        final static String proxyUser = "USERNAME";
        final static String proxyPass = "PASSWORD";

        static OkHttpClient client = null;

        static {
            Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));

            Authenticator proxyAuthenticator = new Authenticator() {
                public Request authenticate(Route route, Response response) {
                    String credential = Credentials.basic(proxyUser, proxyPass);
                    return response.request().newBuilder()
                            .header("Proxy-Authorization", credential)
                            .build();
                }
            };

            client = new OkHttpClient().newBuilder()
                    .connectTimeout(5, TimeUnit.SECONDS)
                    .readTimeout(5, TimeUnit.SECONDS)
                    .proxy(proxy)
                    .proxyAuthenticator(proxyAuthenticator)
                    .connectionPool(new ConnectionPool(5, 1, TimeUnit.SECONDS))
                    .build();
        }

        public static Response doGet() throws IOException {
            // 要访问的目标页面
            String targetUrl = "http://httpbin.org/ip";

            Request request = new Request.Builder()
                    .url(targetUrl)
                    .build();
            Response response = client.newCall(request).execute();
            return response;
        }

        public static void main(String[] args) throws IOException {
            Response response1 = doGet();
            System.out.println("GET请求返回结果：");
            System.out.println(response1.body().string());
        }

    }

最后修改时间： 2 个月前