This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new a04c532650 Core: Enable HTTP proxy support for the client used by REST
Catalog (#12406)
a04c532650 is described below
commit a04c532650156c0e3cb0de56cee48ddfb8e8fd2a
Author: Akhil Lawrence <[email protected]>
AuthorDate: Wed May 7 04:23:57 2025 +0530
Core: Enable HTTP proxy support for the client used by REST Catalog (#12406)
---
.../java/org/apache/iceberg/rest/HTTPClient.java | 34 +++++++++++++
.../org/apache/iceberg/rest/TestHTTPClient.java | 59 ++++++++++++++++++++++
2 files changed, 93 insertions(+)
diff --git a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
index 2f465e0450..532fff2401 100644
--- a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
+++ b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
@@ -27,9 +27,12 @@ import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;
import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
@@ -50,6 +53,7 @@ import org.apache.iceberg.IcebergBuild;
import org.apache.iceberg.exceptions.RESTException;
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.rest.HTTPRequest.HTTPMethod;
import org.apache.iceberg.rest.auth.AuthSession;
@@ -72,6 +76,10 @@ public class HTTPClient extends BaseHTTPClient {
static final int REST_MAX_CONNECTIONS_DEFAULT = 100;
static final String REST_MAX_CONNECTIONS_PER_ROUTE =
"rest.client.connections-per-route";
static final int REST_MAX_CONNECTIONS_PER_ROUTE_DEFAULT = 100;
+ static final String REST_PROXY_HOSTNAME = "rest.client.proxy.hostname";
+ static final String REST_PROXY_PORT = "rest.client.proxy.port";
+ static final String REST_PROXY_USERNAME = "rest.client.proxy.username";
+ static final String REST_PROXY_PASSWORD = "rest.client.proxy.password";
@VisibleForTesting
static final String REST_CONNECTION_TIMEOUT_MS =
"rest.client.connection-timeout-ms";
@@ -443,6 +451,32 @@ public class HTTPClient extends BaseHTTPClient {
withHeader(CLIENT_VERSION_HEADER, IcebergBuild.fullVersion());
withHeader(CLIENT_GIT_COMMIT_SHORT_HEADER,
IcebergBuild.gitCommitShortId());
+ String proxyHostname =
+ PropertyUtil.propertyAsString(properties,
HTTPClient.REST_PROXY_HOSTNAME, null);
+
+ Integer proxyPort =
+ PropertyUtil.propertyAsNullableInt(properties,
HTTPClient.REST_PROXY_PORT);
+
+ if (!Strings.isNullOrEmpty(proxyHostname) && proxyPort != null) {
+ withProxy(proxyHostname, proxyPort);
+
+ String proxyUsername =
+ PropertyUtil.propertyAsString(properties,
HTTPClient.REST_PROXY_USERNAME, null);
+
+ String proxyPassword =
+ PropertyUtil.propertyAsString(properties,
HTTPClient.REST_PROXY_PASSWORD, null);
+
+ if (!Strings.isNullOrEmpty(proxyUsername) &&
!Strings.isNullOrEmpty(proxyPassword)) {
+ // currently only basic auth is supported
+ BasicCredentialsProvider credentialProvider = new
BasicCredentialsProvider();
+ credentialProvider.setCredentials(
+ new AuthScope(proxyHostname, proxyPort),
+ new UsernamePasswordCredentials(proxyUsername,
proxyPassword.toCharArray()));
+
+ withProxyCredentialsProvider(credentialProvider);
+ }
+ }
+
if (this.proxyCredentialsProvider != null) {
Preconditions.checkNotNull(
proxy, "Invalid http client proxy for proxy credentials provider:
null");
diff --git a/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java
b/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java
index 760ed840de..33984307b8 100644
--- a/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java
+++ b/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java
@@ -174,6 +174,65 @@ public class TestHTTPClient {
.hasMessage("Invalid hostname for http client proxy: null");
}
+ @Test
+ public void testClientWithProxyProps() throws IOException {
+ int proxyPort = 1070;
+ try (ClientAndServer proxyServer = startClientAndServer(proxyPort);
+ RESTClient clientWithProxy =
+ HTTPClient.builder(
+ ImmutableMap.of(
+ HTTPClient.REST_PROXY_HOSTNAME,
+ "localhost",
+ HTTPClient.REST_PROXY_PORT,
+ String.valueOf(proxyPort)))
+ .uri(URI)
+ .withAuthSession(AuthSession.EMPTY)
+ .build()) {
+ String path = "v1/config";
+ HttpRequest mockRequest =
+ request("/" +
path).withMethod(HttpMethod.HEAD.name().toUpperCase(Locale.ROOT));
+ HttpResponse mockResponse = response().withStatusCode(200);
+ proxyServer.when(mockRequest).respond(mockResponse);
+ clientWithProxy.head(path, ImmutableMap.of(), (onError) -> {});
+ proxyServer.verify(mockRequest, VerificationTimes.exactly(1));
+ }
+ }
+
+ @Test
+ public void testClientWithAuthProxyProps() throws IOException {
+ int proxyPort = 1070;
+ String authorizedUsername = "test-username";
+ String authorizedPassword = "test-password";
+ try (ClientAndServer proxyServer =
+ startClientAndServer(
+ new Configuration()
+ .proxyAuthenticationUsername(authorizedUsername)
+ .proxyAuthenticationPassword(authorizedPassword),
+ proxyPort);
+ RESTClient clientWithProxy =
+ HTTPClient.builder(
+ ImmutableMap.of(
+ HTTPClient.REST_PROXY_HOSTNAME,
+ "localhost",
+ HTTPClient.REST_PROXY_PORT,
+ String.valueOf(proxyPort),
+ HTTPClient.REST_PROXY_USERNAME,
+ authorizedUsername,
+ HTTPClient.REST_PROXY_PASSWORD,
+ authorizedPassword))
+ .uri(URI)
+ .withAuthSession(AuthSession.EMPTY)
+ .build()) {
+ String path = "v1/config";
+ HttpRequest mockRequest =
+ request("/" +
path).withMethod(HttpMethod.HEAD.name().toUpperCase(Locale.ROOT));
+ HttpResponse mockResponse = response().withStatusCode(200);
+ proxyServer.when(mockRequest).respond(mockResponse);
+ clientWithProxy.head(path, ImmutableMap.of(), (onError) -> {});
+ proxyServer.verify(mockRequest, VerificationTimes.exactly(1));
+ }
+ }
+
@Test
public void testProxyAuthenticationFailure() throws IOException {
int proxyPort = 1050;
