feat: bypass the proxy for the hosts listed in nonProxyHosts

changes.md

@@ -186,6 +186,9 @@ Changes log
         - HTTP/2 and HTTP/3 requests advertise `te: trailers`; stream header names are normalized
           to lowercase at the HTTP boundary.
         - Added the `Protocol.STDIO` pseudo-protocol constant for standard input/output connectors.
+        - The HTTP client connects directly to the hosts listed in the "nonProxyHosts" parameter
+          or the "http.nonProxyHosts" system property, without proxy. As with the JDK, the
+          loopback addresses are excluded by default. Issue #401.
         - Added an optional origin matcher to CORS (`CorsService`, `CorsFilter`, `setAllowedOriginMatcher(Predicate<String>)`) to accept origins that are not listed, for example any sub-domain. The response always contains the exact request origin. Issue #1611.
         - Added the `SameSite` attribute to cookie settings (`CookieSetting.get/setSameSite()`,
           `SameSite` enum), read and written in `Set-Cookie`.
@@ -207,6 +210,9 @@ Changes log
         - The HTTP client uses the "https.proxyHost" and "https.proxyPort" system properties for
           HTTPS requests, instead of "http.proxyHost" and "http.proxyPort", like the JDK. The
           "proxyHost" and "proxyPort" parameters still apply to both. Issue #1570.
+        - In the "HTTP2" transport mode, the HTTP client uses HTTP/1.1 for cleartext requests
+          sent through a forward proxy, instead of HTTP/2 with prior knowledge that HTTP proxies
+          generally refuse. Issue #1571.
         - The `Max-Age` and `Path` attributes of a cookie setting are no longer quoted. Issue #1334.
         - `Restlet.handle()` no longer fails with a `NullPointerException` when the Restlet has no context and cannot be started.
         - `CookieAuthenticator` no longer lets the next restlet overwrite the redirection sent after

org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java

@@ -239,12 +239,12 @@ String getTunnelProxyIdentity() {
 
     /** Indicates whether the exchange goes through a CONNECT tunnel of the HTTP proxy. */
     private boolean isTunneled() {
-        return secure && (helper.getProxy(true) != null);
+        return secure && (helper.getProxy(true, targetHost) != null);
     }
 
     /** Indicates whether the exchange is sent to an HTTP forward proxy. */
     private boolean isForwardProxied() {
-        return !secure && (helper.getProxy(false) != null);
+        return !secure && (helper.getProxy(false, targetHost) != null);
     }
 
     /**

org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java

@@ -55,6 +55,7 @@
 import java.security.GeneralSecurityException;
 import java.util.Deque;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
@@ -134,7 +135,8 @@
  * <td>String</td>
  * <td>HTTP1_1</td>
  * <td>Indicate the HTTP client transport mode among the following options: "HTTP1_1", "HTTP2"
- * (HTTP/2 only: negotiated with ALPN over TLS, prior knowledge in cleartext), "DYNAMIC" (HTTP/2
+ * (HTTP/2 only: negotiated with ALPN over TLS, prior knowledge in cleartext, except through a
+ * forward proxy where HTTP/1.1 is used), "DYNAMIC" (HTTP/2
  * when negotiated with ALPN over TLS, HTTP/1.1 otherwise), "HTTP3" (<b>experimental</b>: HTTP/3
  * over QUIC for HTTPS URIs without proxy, HTTP/1.1 otherwise; requires the optional
  * {@code io.netty:netty-codec-http3} dependency with its native QUIC library, and the {@link
@@ -166,6 +168,15 @@
  * <td>The max size in bytes of the response status line and headers.</td>
  * </tr>
  * <tr>
+ * <td>nonProxyHosts</td>
+ * <td>String</td>
+ * <td>System property "http.nonProxyHosts" or "localhost|127.*|[::1]|0.0.0.0|[::0]"</td>
+ * <td>The hosts to connect to directly, without proxy, separated by "|". A host name can start or
+ * end with a "*" wildcard, e.g. "localhost|*.example.com|192.168.*", and is matched ignoring
+ * case. As with the JDK, the loopback addresses are excluded by default, and an empty value
+ * excludes no host.</td>
+ * </tr>
+ * <tr>
  * <td>proxyHost</td>
  * <td>String</td>
  * <td>System property "http.proxyHost" for HTTP requests, "https.proxyHost" for HTTPS
@@ -209,6 +220,9 @@
     private static final AttributeKey<Promise<String>> READY =
             AttributeKey.valueOf(NettyClientHelper.class, "ready");
 
+    /** The hosts connected to directly by default, as with the JDK. */
+    private static final String DEFAULT_NON_PROXY_HOSTS = "localhost|127.*|[::1]|0.0.0.0|[::0]";
+
     /** The pool owning a connection. */
     private static final AttributeKey<OriginPool> POOL =
             AttributeKey.valueOf(NettyClientHelper.class, "pool");
@@ -316,7 +330,7 @@
         }
 
         boolean secure = "https".equalsIgnoreCase(scheme);
-        boolean forwardProxy = !secure && (getProxy(false) != null);
+        boolean forwardProxy = !secure && (getProxy(false, ref.getHostDomain()) != null);
         String target = ReferenceUtils.format(request.getResourceRef(), forwardProxy, request);
         return new NettyClientCall(this, request.getMethod().toString(), ref, target, secure);
     }
@@ -471,7 +485,7 @@
     Channel acquire(NettyClientCall call) throws IOException {
         if (call.isConfidential()
                 && getTransportMode() == TransportMode.HTTP3
-                && getProxy(true) == null) {
+                && getProxy(true, call.getTargetHost()) == null) {
             return acquireHttp3(call);
         }
 
@@ -666,7 +680,7 @@
         boolean acquired = true;
         try {
             final SslContext ssl = pool.secure ? getSslContext() : null;
-            final InetSocketAddress proxyRef = getProxy(pool.secure);
+            final InetSocketAddress proxyRef = getProxy(pool.secure, pool.host);
             final InetSocketAddress proxy =
                     (proxyRef == null)
                             ? null
@@ -686,11 +700,7 @@
                                         @Override
                                         protected void initChannel(SocketChannel ch) {
                                             initConnection(
-                                                    ch,
-                                                    pool,
-                                                    ssl,
-                                                    pool.secure ? proxy : null,
-                                                    proxyAuthorization);
+                                                    ch, pool, ssl, proxy, proxyAuthorization);
                                         }
                                     });
 
@@ -795,7 +805,7 @@
             SocketChannel ch,
             OriginPool pool,
             SslContext ssl,
-            InetSocketAddress tunnelProxy,
+            InetSocketAddress proxy,
             String proxyAuthorization) {
         final Promise<String> ready = ch.eventLoop().newPromise();
         ch.attr(READY).set(ready);
@@ -810,16 +820,16 @@
         }
         p.addLast("writability", Writability.HANDLER);
 
-        if (tunnelProxy == null) {
-            initProtocols(ch, pool, ssl, ready);
+        if ((proxy == null) || !pool.secure) {
+            initProtocols(ch, pool, ssl, ready, proxy != null);
             return;
         }
 
         HttpHeaders connectHeaders = new DefaultHttpHeaders();
         if (proxyAuthorization != null) {
             connectHeaders.set(HttpHeaderNames.PROXY_AUTHORIZATION, proxyAuthorization);
         }
-        HttpProxyHandler proxyHandler = new HttpProxyHandler(tunnelProxy, connectHeaders);
+        HttpProxyHandler proxyHandler = new HttpProxyHandler(proxy, connectHeaders);
         proxyHandler.setConnectTimeoutMillis(getConnectTimeout());
         p.addLast("proxy", proxyHandler);
         p.addLast(
@@ -840,7 +850,7 @@
                 .addListener(
                         f -> {
                             if (f.isSuccess()) {
-                                initProtocols(ch, pool, ssl, ready);
+                                initProtocols(ch, pool, ssl, ready, false);
                             } else {
                                 ready.tryFailure(f.cause());
                             }
@@ -854,9 +864,14 @@
      * @param pool The connection pool of the origin.
      * @param ssl The SSL context, or null for a cleartext connection.
      * @param ready The promise completed with the negotiated protocol.
+     * @param forwardProxy True for a cleartext connection to a forward proxy.
      */
     private void initProtocols(
-            SocketChannel ch, OriginPool pool, SslContext ssl, Promise<String> ready) {
+            SocketChannel ch,
+            OriginPool pool,
+            SslContext ssl,
+            Promise<String> ready,
+            boolean forwardProxy) {
         ChannelPipeline p = ch.pipeline();
         final TransportMode mode = getTransportMode();
         if (ssl != null) {
@@ -897,10 +912,19 @@
                             ctx.close();
                         }
                     });
-        } else if (mode == TransportMode.HTTP2) {
+        } else if (mode == TransportMode.HTTP2 && !forwardProxy) {
             addHttp2Handlers(p);
             ready.trySuccess(ApplicationProtocolNames.HTTP_2);
         } else {
+            if (mode == TransportMode.HTTP2) {
+                // HTTP proxies generally don't accept HTTP/2 with prior knowledge.
+                getLogger()
+                        .fine(
+                                "Using HTTP/1.1 through the forward proxy to "
+                                        + pool.host
+                                        + ":"
+                                        + pool.port);
+            }
             addHttp1Handlers(p);
             ready.trySuccess(ApplicationProtocolNames.HTTP_1_1);
         }
@@ -1131,16 +1155,72 @@
      * server.
      *
      * @param secure True for an HTTPS request, false for an HTTP request.
+     * @param targetHost The host name of the origin server.
      * @return The unresolved address of the HTTP proxy, or null.
      */
-    InetSocketAddress getProxy(boolean secure) {
+    InetSocketAddress getProxy(boolean secure, String targetHost) {
         String host = secure ? getHttpsProxyHost() : getProxyHost();
-        return (host == null)
+        return ((host == null) || isNonProxyHost(targetHost))
                 ? null
                 : InetSocketAddress.createUnresolved(
                         host, secure ? getHttpsProxyPort() : getProxyPort());
     }
 
+    /**
+     * Returns the hosts to connect to directly, without proxy, separated by "|": the
+     * "nonProxyHosts" parameter, or the "http.nonProxyHosts" system property, the loopback
+     * addresses otherwise.
+     *
+     * @return The hosts to connect to directly, possibly empty.
+     */
+    public String getNonProxyHosts() {
+        return getHelpedParameters()
+                .getFirstValue(
+                        "nonProxyHosts",
+                        System.getProperty("http.nonProxyHosts", DEFAULT_NON_PROXY_HOSTS));
+    }
+
+    /**
+     * Indicates whether a host must be connected to directly, without proxy.
+     *
+     * @param host The host name.
+     * @return True if the host matches one of the {@link #getNonProxyHosts()} patterns.
+     */
+    boolean isNonProxyHost(String host) {
+        String patterns = getNonProxyHosts();
+        if ((patterns == null) || (host == null)) {
+            return false;
+        }
+
+        String name = host.toLowerCase(Locale.ROOT);
+        for (String pattern : patterns.split("\\|")) {
+            if (matchesHost(pattern.trim().toLowerCase(Locale.ROOT), name)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /** Matches a host name against a pattern with an optional leading or trailing wildcard. */
+    private static boolean matchesHost(String pattern, String host) {
+        if (pattern.isEmpty()) {
+            return false;
+        } else if ("*".equals(pattern)) {
+            return true;
+        }
+        boolean leading = pattern.startsWith("*");
+        boolean trailing = pattern.endsWith("*");
+        String core = pattern.substring(leading ? 1 : 0, pattern.length() - (trailing ? 1 : 0));
+        if (leading && trailing) {
+            return host.contains(core);
+        } else if (leading) {
+            return host.endsWith(core);
+        } else if (trailing) {
+            return host.startsWith(core);
+        }
+        return host.equals(core);
+    }
+
     /**
      * Returns the host name of the HTTP proxy for HTTPS requests, if specified: the "proxyHost"
      * parameter, or the "https.proxyHost" system property.