diff --git a/changes.md b/changes.md index 89954d9b6b..b8192b962a 100644 --- a/changes.md +++ b/changes.md @@ -186,6 +186,9 @@ Changes log - HTTP/2 and HTTP/3 requests advertise `te: trailers`; stream header names are normalized to lowercase at the HTTP boundary. - Added the `Protocol.STDIO` pseudo-protocol constant for standard input/output connectors. + - The HTTP client connects directly to the hosts listed in the "nonProxyHosts" parameter + or the "http.nonProxyHosts" system property, without proxy. As with the JDK, the + loopback addresses are excluded by default. Issue #401. - Added an optional origin matcher to CORS (`CorsService`, `CorsFilter`, `setAllowedOriginMatcher(Predicate)`) to accept origins that are not listed, for example any sub-domain. The response always contains the exact request origin. Issue #1611. - Added the `SameSite` attribute to cookie settings (`CookieSetting.get/setSameSite()`, `SameSite` enum), read and written in `Set-Cookie`. @@ -207,6 +210,9 @@ Changes log - The HTTP client uses the "https.proxyHost" and "https.proxyPort" system properties for HTTPS requests, instead of "http.proxyHost" and "http.proxyPort", like the JDK. The "proxyHost" and "proxyPort" parameters still apply to both. Issue #1570. + - In the "HTTP2" transport mode, the HTTP client uses HTTP/1.1 for cleartext requests + sent through a forward proxy, instead of HTTP/2 with prior knowledge that HTTP proxies + generally refuse. Issue #1571. - The `Max-Age` and `Path` attributes of a cookie setting are no longer quoted. Issue #1334. - `Restlet.handle()` no longer fails with a `NullPointerException` when the Restlet has no context and cannot be started. - `CookieAuthenticator` no longer lets the next restlet overwrite the redirection sent after diff --git a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java index 5cc3fabbd4..2ee520217f 100644 --- a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java +++ b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java @@ -239,12 +239,12 @@ String getTunnelProxyIdentity() { /** Indicates whether the exchange goes through a CONNECT tunnel of the HTTP proxy. */ private boolean isTunneled() { - return secure && (helper.getProxy(true) != null); + return secure && (helper.getProxy(true, targetHost) != null); } /** Indicates whether the exchange is sent to an HTTP forward proxy. */ private boolean isForwardProxied() { - return !secure && (helper.getProxy(false) != null); + return !secure && (helper.getProxy(false, targetHost) != null); } /** diff --git a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java index bb64bfde51..7170fd436d 100644 --- a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java +++ b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java @@ -55,6 +55,7 @@ import java.security.GeneralSecurityException; import java.util.Deque; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; @@ -134,7 +135,8 @@ * String * HTTP1_1 * Indicate the HTTP client transport mode among the following options: "HTTP1_1", "HTTP2" - * (HTTP/2 only: negotiated with ALPN over TLS, prior knowledge in cleartext), "DYNAMIC" (HTTP/2 + * (HTTP/2 only: negotiated with ALPN over TLS, prior knowledge in cleartext, except through a + * forward proxy where HTTP/1.1 is used), "DYNAMIC" (HTTP/2 * when negotiated with ALPN over TLS, HTTP/1.1 otherwise), "HTTP3" (experimental: HTTP/3 * over QUIC for HTTPS URIs without proxy, HTTP/1.1 otherwise; requires the optional * {@code io.netty:netty-codec-http3} dependency with its native QUIC library, and the {@link @@ -166,6 +168,15 @@ * The max size in bytes of the response status line and headers. * * + * nonProxyHosts + * String + * System property "http.nonProxyHosts" or "localhost|127.*|[::1]|0.0.0.0|[::0]" + * The hosts to connect to directly, without proxy, separated by "|". A host name can start or + * end with a "*" wildcard, e.g. "localhost|*.example.com|192.168.*", and is matched ignoring + * case. As with the JDK, the loopback addresses are excluded by default, and an empty value + * excludes no host. + * + * * proxyHost * String * System property "http.proxyHost" for HTTP requests, "https.proxyHost" for HTTPS @@ -209,6 +220,9 @@ public abstract class NettyClientHelper extends ClientHelper { private static final AttributeKey> READY = AttributeKey.valueOf(NettyClientHelper.class, "ready"); + /** The hosts connected to directly by default, as with the JDK. */ + private static final String DEFAULT_NON_PROXY_HOSTS = "localhost|127.*|[::1]|0.0.0.0|[::0]"; + /** The pool owning a connection. */ private static final AttributeKey POOL = AttributeKey.valueOf(NettyClientHelper.class, "pool"); @@ -316,7 +330,7 @@ NettyClientCall create(Request request) { } boolean secure = "https".equalsIgnoreCase(scheme); - boolean forwardProxy = !secure && (getProxy(false) != null); + boolean forwardProxy = !secure && (getProxy(false, ref.getHostDomain()) != null); String target = ReferenceUtils.format(request.getResourceRef(), forwardProxy, request); return new NettyClientCall(this, request.getMethod().toString(), ref, target, secure); } @@ -471,7 +485,7 @@ private static void resetResponse(Response response, Request next) { Channel acquire(NettyClientCall call) throws IOException { if (call.isConfidential() && getTransportMode() == TransportMode.HTTP3 - && getProxy(true) == null) { + && getProxy(true, call.getTargetHost()) == null) { return acquireHttp3(call); } @@ -666,7 +680,7 @@ private Channel connect(final OriginPool pool, final String proxyAuthorization) boolean acquired = true; try { final SslContext ssl = pool.secure ? getSslContext() : null; - final InetSocketAddress proxyRef = getProxy(pool.secure); + final InetSocketAddress proxyRef = getProxy(pool.secure, pool.host); final InetSocketAddress proxy = (proxyRef == null) ? null @@ -686,11 +700,7 @@ private Channel connect(final OriginPool pool, final String proxyAuthorization) @Override protected void initChannel(SocketChannel ch) { initConnection( - ch, - pool, - ssl, - pool.secure ? proxy : null, - proxyAuthorization); + ch, pool, ssl, proxy, proxyAuthorization); } }); @@ -795,7 +805,7 @@ private void initConnection( SocketChannel ch, OriginPool pool, SslContext ssl, - InetSocketAddress tunnelProxy, + InetSocketAddress proxy, String proxyAuthorization) { final Promise ready = ch.eventLoop().newPromise(); ch.attr(READY).set(ready); @@ -810,8 +820,8 @@ private void initConnection( } p.addLast("writability", Writability.HANDLER); - if (tunnelProxy == null) { - initProtocols(ch, pool, ssl, ready); + if ((proxy == null) || !pool.secure) { + initProtocols(ch, pool, ssl, ready, proxy != null); return; } @@ -819,7 +829,7 @@ private void initConnection( if (proxyAuthorization != null) { connectHeaders.set(HttpHeaderNames.PROXY_AUTHORIZATION, proxyAuthorization); } - HttpProxyHandler proxyHandler = new HttpProxyHandler(tunnelProxy, connectHeaders); + HttpProxyHandler proxyHandler = new HttpProxyHandler(proxy, connectHeaders); proxyHandler.setConnectTimeoutMillis(getConnectTimeout()); p.addLast("proxy", proxyHandler); p.addLast( @@ -840,7 +850,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { .addListener( f -> { if (f.isSuccess()) { - initProtocols(ch, pool, ssl, ready); + initProtocols(ch, pool, ssl, ready, false); } else { ready.tryFailure(f.cause()); } @@ -854,9 +864,14 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { * @param pool The connection pool of the origin. * @param ssl The SSL context, or null for a cleartext connection. * @param ready The promise completed with the negotiated protocol. + * @param forwardProxy True for a cleartext connection to a forward proxy. */ private void initProtocols( - SocketChannel ch, OriginPool pool, SslContext ssl, Promise ready) { + SocketChannel ch, + OriginPool pool, + SslContext ssl, + Promise ready, + boolean forwardProxy) { ChannelPipeline p = ch.pipeline(); final TransportMode mode = getTransportMode(); if (ssl != null) { @@ -897,10 +912,19 @@ protected void handshakeFailure( ctx.close(); } }); - } else if (mode == TransportMode.HTTP2) { + } else if (mode == TransportMode.HTTP2 && !forwardProxy) { addHttp2Handlers(p); ready.trySuccess(ApplicationProtocolNames.HTTP_2); } else { + if (mode == TransportMode.HTTP2) { + // HTTP proxies generally don't accept HTTP/2 with prior knowledge. + getLogger() + .fine( + "Using HTTP/1.1 through the forward proxy to " + + pool.host + + ":" + + pool.port); + } addHttp1Handlers(p); ready.trySuccess(ApplicationProtocolNames.HTTP_1_1); } @@ -1131,16 +1155,72 @@ public int getMaxResponseHeaderSize() { * server. * * @param secure True for an HTTPS request, false for an HTTP request. + * @param targetHost The host name of the origin server. * @return The unresolved address of the HTTP proxy, or null. */ - InetSocketAddress getProxy(boolean secure) { + InetSocketAddress getProxy(boolean secure, String targetHost) { String host = secure ? getHttpsProxyHost() : getProxyHost(); - return (host == null) + return ((host == null) || isNonProxyHost(targetHost)) ? null : InetSocketAddress.createUnresolved( host, secure ? getHttpsProxyPort() : getProxyPort()); } + /** + * Returns the hosts to connect to directly, without proxy, separated by "|": the + * "nonProxyHosts" parameter, or the "http.nonProxyHosts" system property, the loopback + * addresses otherwise. + * + * @return The hosts to connect to directly, possibly empty. + */ + public String getNonProxyHosts() { + return getHelpedParameters() + .getFirstValue( + "nonProxyHosts", + System.getProperty("http.nonProxyHosts", DEFAULT_NON_PROXY_HOSTS)); + } + + /** + * Indicates whether a host must be connected to directly, without proxy. + * + * @param host The host name. + * @return True if the host matches one of the {@link #getNonProxyHosts()} patterns. + */ + boolean isNonProxyHost(String host) { + String patterns = getNonProxyHosts(); + if ((patterns == null) || (host == null)) { + return false; + } + + String name = host.toLowerCase(Locale.ROOT); + for (String pattern : patterns.split("\\|")) { + if (matchesHost(pattern.trim().toLowerCase(Locale.ROOT), name)) { + return true; + } + } + return false; + } + + /** Matches a host name against a pattern with an optional leading or trailing wildcard. */ + private static boolean matchesHost(String pattern, String host) { + if (pattern.isEmpty()) { + return false; + } else if ("*".equals(pattern)) { + return true; + } + boolean leading = pattern.startsWith("*"); + boolean trailing = pattern.endsWith("*"); + String core = pattern.substring(leading ? 1 : 0, pattern.length() - (trailing ? 1 : 0)); + if (leading && trailing) { + return host.contains(core); + } else if (leading) { + return host.endsWith(core); + } else if (trailing) { + return host.startsWith(core); + } + return host.equals(core); + } + /** * Returns the host name of the HTTP proxy for HTTPS requests, if specified: the "proxyHost" * parameter, or the "https.proxyHost" system property. diff --git a/org.restlet/src/test/java/org/restlet/engine/connector/ProxyTestCase.java b/org.restlet/src/test/java/org/restlet/engine/connector/ProxyTestCase.java index 819e820708..25ae39907c 100644 --- a/org.restlet/src/test/java/org/restlet/engine/connector/ProxyTestCase.java +++ b/org.restlet/src/test/java/org/restlet/engine/connector/ProxyTestCase.java @@ -115,9 +115,7 @@ void forwardsHttpRequestsWithAbsoluteUris() throws Exception { server = new Server(new Context(), Protocol.HTTP, 0, HELLO); server.start(); - Context context = new Context(); - context.getParameters().add("proxyHost", "localhost"); - context.getParameters().add("proxyPort", Integer.toString(proxy.port())); + Context context = proxyContext(); client = new Client(context, Protocol.HTTP); client.start(); @@ -133,6 +131,23 @@ void forwardsHttpRequestsWithAbsoluteUris() throws Exception { assertEquals("GET " + uri + " HTTP/1.1", proxy.requestLines.getFirst()); } + @Test + void forwardsCleartextHttp2RequestsWithHttp1() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, HELLO); + server.start(); + + Context context = proxyContext(); + context.getParameters().add("httpClientTransportMode", "HTTP2"); + client = new Client(context, Protocol.HTTP); + client.start(); + + String uri = "http://localhost:" + server.getActualPort() + "/h2"; + Response response = client.handle(new Request(Method.GET, uri)); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertEquals("hello /h2", response.getEntityAsText()); + assertEquals(List.of("GET " + uri + " HTTP/1.1"), proxy.requestLines); + } + @Test void tunnelsHttpsRequestsWithConnect() throws Exception { startHttpsServer(HELLO); @@ -261,7 +276,13 @@ private static ChallengeResponse rawCredentials(String token) { void usesHttpsProxySystemPropertiesForHttpsRequests() throws Exception { startHttpsServer(HELLO); withSystemProperties( - Map.of("https.proxyHost", "localhost", "https.proxyPort", proxyPort()), + Map.of( + "https.proxyHost", + "localhost", + "https.proxyPort", + proxyPort(), + "http.nonProxyHosts", + ""), () -> { client = newHttpsClient(false); Response response = @@ -281,7 +302,13 @@ void usesHttpsProxySystemPropertiesForHttpsRequests() throws Exception { void ignoresHttpProxySystemPropertiesForHttpsRequests() throws Exception { startHttpsServer(HELLO); withSystemProperties( - Map.of("http.proxyHost", "localhost", "http.proxyPort", proxyPort()), + Map.of( + "http.proxyHost", + "localhost", + "http.proxyPort", + proxyPort(), + "http.nonProxyHosts", + ""), () -> { client = newHttpsClient(false); Response response = @@ -299,7 +326,13 @@ void ignoresHttpProxySystemPropertiesForHttpsRequests() throws Exception { void doesNotSendProxyCredentialsToOriginServerWithoutProxy() throws Exception { startHttpsServer(PROXY_AUTHORIZATION_ECHO); withSystemProperties( - Map.of("http.proxyHost", "localhost", "http.proxyPort", proxyPort()), + Map.of( + "http.proxyHost", + "localhost", + "http.proxyPort", + proxyPort(), + "http.nonProxyHosts", + ""), () -> { client = newHttpsClient(false); Request request = @@ -316,6 +349,91 @@ void doesNotSendProxyCredentialsToOriginServerWithoutProxy() throws Exception { assertTrue(proxy.requestLines.isEmpty(), proxy.requestLines.toString()); } + @Test + void bypassesProxyForNonProxyHosts() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, HELLO); + server.start(); + + Context context = new Context(); + context.getParameters().add("proxyHost", "localhost"); + context.getParameters().add("proxyPort", proxyPort()); + context.getParameters().add("nonProxyHosts", "example.org | LOCAL*"); + client = new Client(context, Protocol.HTTP); + client.start(); + + Response response = + client.handle( + new Request( + Method.GET, "http://localhost:" + server.getActualPort() + "/d")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertEquals("hello /d", response.getEntityAsText()); + assertTrue(proxy.requestLines.isEmpty(), proxy.requestLines.toString()); + } + + @Test + void bypassesProxyForNonProxyHostsSystemProperty() throws Exception { + startHttpsServer(HELLO); + withSystemProperties( + Map.of( + "https.proxyHost", + "localhost", + "https.proxyPort", + proxyPort(), + "http.nonProxyHosts", + "*.example.org|*host"), + () -> { + client = newHttpsClient(false); + Response response = + client.handle( + new Request( + Method.GET, + "https://localhost:" + server.getActualPort() + "/s")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + }); + + assertTrue(proxy.requestLines.isEmpty(), proxy.requestLines.toString()); + } + + @Test + void bypassesProxyForLoopbackHostsByDefault() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, HELLO); + server.start(); + + Context context = new Context(); + context.getParameters().add("proxyHost", "localhost"); + context.getParameters().add("proxyPort", proxyPort()); + client = new Client(context, Protocol.HTTP); + client.start(); + + for (String host : List.of("localhost", "127.0.0.1")) { + Response response = + client.handle( + new Request( + Method.GET, + "http://" + host + ":" + server.getActualPort() + "/l")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + } + assertTrue(proxy.requestLines.isEmpty(), proxy.requestLines.toString()); + } + + @Test + void usesProxyForOtherHosts() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, HELLO); + server.start(); + + Context context = new Context(); + context.getParameters().add("proxyHost", "localhost"); + context.getParameters().add("proxyPort", proxyPort()); + context.getParameters().add("nonProxyHosts", "*.example.org|local|127.*"); + client = new Client(context, Protocol.HTTP); + client.start(); + + String uri = "http://localhost:" + server.getActualPort() + "/p"; + Response response = client.handle(new Request(Method.GET, uri)); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertEquals(List.of("GET " + uri + " HTTP/1.1"), proxy.requestLines); + } + @Test void reportsUnreachableProxy() throws Exception { server = new Server(new Context(), Protocol.HTTP, 0, HELLO); @@ -328,6 +446,7 @@ void reportsUnreachableProxy() throws Exception { Context context = new Context(); context.getParameters().add("proxyHost", "localhost"); context.getParameters().add("proxyPort", Integer.toString(closedPort)); + context.getParameters().add("nonProxyHosts", ""); context.getParameters().add("connectTimeout", "2000"); client = new Client(context, Protocol.HTTP); client.start(); @@ -363,14 +482,10 @@ private Client newHttpsClient(String... parameters) throws Exception { private Client newHttpsClient(boolean proxyParameters, String... parameters) throws Exception { Path keystore = keystore(); - Context context = new Context(); + Context context = proxyParameters ? proxyContext() : new Context(); for (int i = 0; i < parameters.length; i += 2) { context.getParameters().add(parameters[i], parameters[i + 1]); } - if (proxyParameters) { - context.getParameters().add("proxyHost", "localhost"); - context.getParameters().add("proxyPort", proxyPort()); - } context.getParameters().add("truststorePath", keystore.toString()); context.getParameters().add("truststorePassword", "testtest"); context.getParameters().add("trustStoreType", "PKCS12"); @@ -379,6 +494,15 @@ private Client newHttpsClient(boolean proxyParameters, String... parameters) thr return result; } + /** Returns a context using the test proxy, including for the loopback addresses. */ + private Context proxyContext() { + Context context = new Context(); + context.getParameters().add("proxyHost", "localhost"); + context.getParameters().add("proxyPort", proxyPort()); + context.getParameters().add("nonProxyHosts", ""); + return context; + } + private String proxyPort() { return Integer.toString(proxy.port()); }