HTTPCLIENT-2381 Enable automatic mapping

httpclient5/pom.xml

@@ -118,6 +118,11 @@
       <artifactId>zstd-jni</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.stefanbirkner</groupId>
+      <artifactId>system-lambda</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/HttpClientBuilder.java

@@ -233,6 +233,7 @@ private ExecInterceptorEntry(
     private boolean authCachingDisabled;
     private boolean connectionStateDisabled;
     private boolean defaultUserAgentDisabled;
+    private boolean proxyAutodetectionDisabled;
     private ProxySelector proxySelector;
 
     private List<Closeable> closeables;
@@ -791,6 +792,19 @@ public final HttpClientBuilder disableDefaultUserAgent() {
         return this;
     }
 
+    /**
+     * Disables automatic proxy detection for clients created by this builder.
+     * <p>
+     * When disabled, and unless an explicit proxy or route planner is configured,
+     * the builder falls back to {@link org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner}.
+     * </p>
+     * @return this instance.
+     */
+    public final HttpClientBuilder disableProxyAutodetection() {
+        this.proxyAutodetectionDisabled = true;
+        return this;
+    }
+
     /**
      * Sets the {@link ProxySelector} that will be used to select the proxies
      * to be used for establishing HTTP connections. If a non-null proxy selector is set,
@@ -838,6 +852,7 @@ protected Function<HttpContext, HttpClientContext> contextAdaptor() {
     }
 
     public CloseableHttpClient build() {
+
         // Create main request executor
         // We copy the instance fields to avoid changing them, and rename to avoid accidental use of the wrong version
         HttpRequestExecutor requestExecCopy = this.requestExec;
@@ -1011,11 +1026,11 @@ public CloseableHttpClient build() {
             }
             if (proxy != null) {
                 routePlannerCopy = new DefaultProxyRoutePlanner(proxy, schemePortResolverCopy);
-            } else if (this.proxySelector != null) {
-                routePlannerCopy = new SystemDefaultRoutePlanner(schemePortResolverCopy, this.proxySelector);
-            } else if (systemProperties) {
-                final ProxySelector defaultProxySelector = AccessController.doPrivileged((PrivilegedAction<ProxySelector>) ProxySelector::getDefault);
-                routePlannerCopy = new SystemDefaultRoutePlanner(schemePortResolverCopy, defaultProxySelector);
+            } else if (!this.proxyAutodetectionDisabled) {
+                final ProxySelector effectiveSelector = this.proxySelector != null
+                        ? this.proxySelector
+                        : AccessController.doPrivileged((PrivilegedAction<ProxySelector>) ProxySelector::getDefault);
+                routePlannerCopy = new SystemDefaultRoutePlanner(schemePortResolverCopy, effectiveSelector);
             } else {
                 routePlannerCopy = new DefaultRoutePlanner(schemePortResolverCopy);
             }

httpclient5/src/main/java/org/apache/hc/client5/http/impl/routing/SystemDefaultRoutePlanner.java

@@ -27,12 +27,16 @@
 
 package org.apache.hc.client5.http.impl.routing;
 
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.List;
+import java.util.Locale;
 
 import org.apache.hc.client5.http.SchemePortResolver;
 import org.apache.hc.core5.annotation.Contract;
@@ -85,7 +89,7 @@ protected HttpHost determineProxy(final HttpHost target, final HttpContext conte
         }
         if (proxySelectorInstance == null) {
             //The proxy selector can be "unset", so we must be able to deal with a null selector
-            return null;
+            return determineEnvProxy(target); // === env-fallback ===
         }
         final List<Proxy> proxies = proxySelectorInstance.select(targetURI);
         final Proxy p = chooseProxy(proxies);
@@ -100,8 +104,79 @@ protected HttpHost determineProxy(final HttpHost target, final HttpContext conte
             result = new HttpHost(null, isa.getAddress(), isa.getHostString(), isa.getPort());
         }
 
+        if (result == null) {
+            result = determineEnvProxy(target);
+        }
+
         return result;
     }
+    private static HttpHost determineEnvProxy(final HttpHost target) {
+        final boolean secure = "https".equalsIgnoreCase(target.getSchemeName());
+        HttpHost proxy = proxyFromEnv(secure ? "HTTPS_PROXY" : "HTTP_PROXY");
+        if (proxy == null && !secure) {
+            proxy = proxyFromEnv("HTTPS_PROXY"); // reuse HTTPS proxy for HTTP if only that exists
+        }
+        if (proxy != null && !isNoProxy(target)) {
+            return proxy;
+        }
+        return null;
+    }
+
+    private static HttpHost proxyFromEnv(final String var) {
+        String val = getenv(var);
+        if (val == null || val.isEmpty()) {
+            val = getenv(var.toLowerCase(Locale.ROOT));
+        }
+        if (val == null || val.isEmpty()) {
+            return null;
+        }
+        if (!val.contains("://")) {
+            val = "http://" + val;
+        }
+        try {
+            final URI uri = new URI(val);
+            final String host = uri.getHost();
+            final int port = uri.getPort() != -1
+                    ? uri.getPort()
+                    : ("https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80);
+            return new HttpHost(uri.getScheme(), InetAddress.getByName(host), port);
+        } catch (final Exception ignore) {
+            return null;
+        }
+    }
+
+    private static boolean isNoProxy(final HttpHost target) {
+        String list = getenv("NO_PROXY");
+        if (list == null || list.isEmpty()) {
+            list = getenv("no_proxy");
+        }
+        if (list == null || list.isEmpty()) {
+            return false;
+        }
+        final String host = target.getHostName().toLowerCase(Locale.ROOT);
+        final String hostPort = host + (target.getPort() != -1 ? ":" + target.getPort() : "");
+        for (String rule : list.split(",")) {
+            rule = rule.trim().toLowerCase(Locale.ROOT);
+            if (rule.isEmpty()) {
+                continue;
+            }
+            if (rule.equals(host) || rule.equals(hostPort)) {
+                return true; // exact
+            }
+            if (rule.startsWith("*.") && host.endsWith(rule.substring(1))) {
+                return true; // *.example.com
+            }
+            if (rule.endsWith("/16") && host.startsWith(rule.substring(0, rule.length() - 3))) {
+                return true; // cidr /16
+            }
+        }
+        return false;
+    }
+
+    private static String getenv(final String key) {
+        return AccessController.doPrivileged(
+                (PrivilegedAction<String>) () -> System.getenv(key));
+    }
 
     private Proxy chooseProxy(final List<Proxy> proxies) {
         Proxy result = null;

httpclient5/src/test/java/org/apache/hc/client5/http/impl/classic/TestHttpClientBuilder.java

@@ -27,12 +27,22 @@
 package org.apache.hc.client5.http.impl.classic;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.ProxySelector;
 
 import org.apache.hc.client5.http.classic.ExecChain;
 import org.apache.hc.client5.http.classic.ExecChainHandler;
+import org.apache.hc.client5.http.impl.routing.DefaultProxyRoutePlanner;
+import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner;
+import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner;
+import org.apache.hc.client5.http.protocol.HttpClientContext;
+import org.apache.hc.client5.http.routing.HttpRoutePlanner;
 import org.apache.hc.core5.http.ClassicHttpRequest;
 import org.apache.hc.core5.http.ClassicHttpResponse;
 import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 class TestHttpClientBuilder {
@@ -66,4 +76,87 @@ public ClassicHttpResponse execute(
             return chain.proceed(request, scope);
         }
     }
+
+    @Test
+    void testDefaultUsesSystemDefaultRoutePlanner() throws Exception {
+        try (final InternalHttpClient client = (InternalHttpClient) HttpClients.custom().build()) {
+            final Object planner = getPrivateField(client, "routePlanner");
+            Assertions.assertNotNull(planner);
+            Assertions.assertInstanceOf(SystemDefaultRoutePlanner.class, planner, "Default should be SystemDefaultRoutePlanner (auto-detect proxies)");
+        }
+    }
+
+    @Test
+    void testDisableProxyAutodetectionFallsBackToDefaultRoutePlanner() throws Exception {
+        try (final InternalHttpClient client = (InternalHttpClient) HttpClients.custom()
+                .disableProxyAutodetection()
+                .build()) {
+            final Object planner = getPrivateField(client, "routePlanner");
+            Assertions.assertNotNull(planner);
+            Assertions.assertInstanceOf(DefaultRoutePlanner.class, planner, "disableProxyAutodetection() should restore DefaultRoutePlanner");
+        }
+    }
+
+    @Test
+    void testExplicitProxyWinsOverAutodetection() throws Exception {
+        try (final InternalHttpClient client = (InternalHttpClient) HttpClients.custom()
+                .setProxy(new HttpHost("http", "proxy.local", 8080))
+                .build()) {
+            final Object planner = getPrivateField(client, "routePlanner");
+            Assertions.assertNotNull(planner);
+            Assertions.assertInstanceOf(DefaultProxyRoutePlanner.class, planner, "Explicit proxy must take precedence");
+        }
+    }
+
+    @Test
+    void testCustomRoutePlannerIsRespected() throws Exception {
+        final HttpRoutePlanner custom = new HttpRoutePlanner() {
+            @Override
+            public org.apache.hc.client5.http.HttpRoute determineRoute(
+                    final HttpHost host, final HttpContext context) {
+                // trivial, never used in this test
+                return new org.apache.hc.client5.http.HttpRoute(host);
+            }
+        };
+        try (final InternalHttpClient client = (InternalHttpClient) HttpClients.custom()
+                .setRoutePlanner(custom)
+                .build()) {
+            final Object planner = getPrivateField(client, "routePlanner");
+            Assertions.assertSame(custom, planner, "Custom route planner must be used as-is");
+        }
+    }
+
+    @Test
+    void testProvidedProxySelectorIsUsedBySystemDefaultRoutePlanner() throws Exception {
+        class TouchProxySelector extends ProxySelector {
+            volatile boolean touched = false;
+            @Override
+            public java.util.List<java.net.Proxy> select(final java.net.URI uri) {
+                touched = true;
+                return java.util.Collections.singletonList(java.net.Proxy.NO_PROXY);
+            }
+            @Override
+            public void connectFailed(final java.net.URI uri, final java.net.SocketAddress sa, final IOException ioe) { }
+        }
+        final TouchProxySelector selector = new TouchProxySelector();
+
+        try (final InternalHttpClient client = (InternalHttpClient) HttpClients.custom()
+                .setProxySelector(selector)
+                .build()) {
+            final Object planner = getPrivateField(client, "routePlanner");
+            Assertions.assertInstanceOf(SystemDefaultRoutePlanner.class, planner);
+
+            // Call determineRoute on the planner directly to avoid making a real request
+            final SystemDefaultRoutePlanner sdrp = (SystemDefaultRoutePlanner) planner;
+            sdrp.determineRoute(new HttpHost("http", "example.com", 80), HttpClientContext.create());
+
+            Assertions.assertTrue(selector.touched, "Provided ProxySelector should be consulted");
+        }
+    }
+
+    private static Object getPrivateField(final Object target, final String name) throws Exception {
+        final Field f = target.getClass().getDeclaredField(name);
+        f.setAccessible(true);
+        return f.get(target);
+    }
 }

httpclient5/src/test/java/org/apache/hc/client5/http/impl/routing/TestSystemDefaultRoutePlanner.java

@@ -27,12 +27,15 @@
 
 package org.apache.hc.client5.http.impl.routing;
 
+import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable;
+
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.hc.client5.http.HttpRoute;
@@ -42,9 +45,12 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledForJreRange;
+import org.junit.jupiter.api.condition.JRE;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
+
 /**
  * Tests for {@link SystemDefaultRoutePlanner}.
  */
@@ -91,8 +97,8 @@ void testDirectDefaultPort() throws Exception {
     @Test
     void testProxy() throws Exception {
 
-        final InetAddress ia = InetAddress.getByAddress(new byte[] {
-            (byte)127, (byte)0, (byte)0, (byte)1
+        final InetAddress ia = InetAddress.getByAddress(new byte[]{
+                (byte) 127, (byte) 0, (byte) 0, (byte) 1
         });
         final InetSocketAddress isa1 = new InetSocketAddress(ia, 11111);
         final InetSocketAddress isa2 = new InetSocketAddress(ia, 22222);
@@ -113,4 +119,52 @@ void testProxy() throws Exception {
         Assertions.assertEquals(isa1.getPort(), route.getProxyHost().getPort());
     }
 
+    @EnabledForJreRange(max = JRE.JAVA_15)
+    @Test
+    void testEnvHttpProxy() throws Exception {
+        withEnvironmentVariable("HTTP_PROXY", "http://proxy.acme.local:8080")
+                .execute(() -> {
+                    Mockito.when(proxySelector.select(ArgumentMatchers.any()))
+                            .thenReturn(Collections.singletonList(Proxy.NO_PROXY));
+
+                    final HttpHost target = new HttpHost("http", "example.com", 80);
+                    final HttpRoute route = routePlanner.determineRoute(
+                            target, HttpClientContext.create());
+
+                    Assertions.assertNull(route.getProxyHost());
+                });
+    }
+    @EnabledForJreRange(max = JRE.JAVA_15)
+    @Test
+    void testEnvHttpsProxy() throws Exception {
+        withEnvironmentVariable("HTTPS_PROXY", "http://secure.proxy:8443")
+                .execute(() -> {
+                    Mockito.when(proxySelector.select(ArgumentMatchers.any()))
+                            .thenReturn(Collections.singletonList(Proxy.NO_PROXY));
+
+                    final HttpHost target = new HttpHost("https", "secure.example", 443);
+                    final HttpRoute route = routePlanner.determineRoute(
+                            target, HttpClientContext.create());
+
+                    Assertions.assertNull(route.getProxyHost());
+                });
+    }
+    @EnabledForJreRange(max = JRE.JAVA_15)
+    @Test
+    void testEnvNoProxyExcludesHost() throws Exception {
+        withEnvironmentVariable("HTTP_PROXY", "http://proxy:3128")
+                .and("NO_PROXY", "localhost,127.0.0.1")
+                .execute(() -> {
+                    Mockito.when(proxySelector.select(ArgumentMatchers.any()))
+                            .thenReturn(Collections.singletonList(Proxy.NO_PROXY));
+
+                    final HttpHost target = new HttpHost("http", "localhost", 80);
+                    final HttpRoute route = routePlanner.determineRoute(
+                            target, HttpClientContext.create());
+
+                    Assertions.assertNull(route.getProxyHost());
+                });
+    }
+
+
 }

pom.xml

@@ -80,6 +80,7 @@
     <hc.animal-sniffer.signature.ignores>javax.net.ssl.SSLEngine,javax.net.ssl.SSLParameters,java.nio.ByteBuffer,java.nio.CharBuffer</hc.animal-sniffer.signature.ignores>
     <commons.compress.version>1.27.1</commons.compress.version>
     <zstd.jni.version>1.5.7-4</zstd.jni.version>
+    <stefanbirkner.version>1.2.1</stefanbirkner.version>
   </properties>
 
   <dependencyManagement>
@@ -216,6 +217,11 @@
         <artifactId>zstd-jni</artifactId>
         <version>${zstd.jni.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.github.stefanbirkner</groupId>
+        <artifactId>system-lambda</artifactId>
+        <version>${stefanbirkner.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>