This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 5e08f886f3 Make proxy endpoint configurable for s3 Http clients
(#10332)
5e08f886f3 is described below
commit 5e08f886f367e548834afe1d43f99e0cb8d3ef5e
Author: Yufei Gu <[email protected]>
AuthorDate: Tue May 14 10:29:24 2024 -0700
Make proxy endpoint configurable for s3 Http clients (#10332)
---
.../apache/iceberg/aws/ApacheHttpClientConfigurations.java | 10 ++++++++++
.../java/org/apache/iceberg/aws/HttpClientProperties.java | 7 +++++++
.../iceberg/aws/UrlConnectionHttpClientConfigurations.java | 10 ++++++++++
.../apache/iceberg/aws/TestHttpClientConfigurations.java | 13 +++++++++++++
docs/docs/aws.md | 9 +++++----
5 files changed, 45 insertions(+), 4 deletions(-)
diff --git
a/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
b/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
index 04f5b911ff..95fe34b742 100644
---
a/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
+++
b/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
@@ -18,12 +18,14 @@
*/
package org.apache.iceberg.aws;
+import java.net.URI;
import java.time.Duration;
import java.util.Map;
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.util.PropertyUtil;
import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
class ApacheHttpClientConfigurations {
private Long connectionTimeoutMs;
@@ -35,6 +37,7 @@ class ApacheHttpClientConfigurations {
private Integer maxConnections;
private Boolean tcpKeepAliveEnabled;
private Boolean useIdleConnectionReaperEnabled;
+ private String proxyEndpoint;
private ApacheHttpClientConfigurations() {}
@@ -72,6 +75,9 @@ class ApacheHttpClientConfigurations {
this.useIdleConnectionReaperEnabled =
PropertyUtil.propertyAsNullableBoolean(
httpClientProperties,
HttpClientProperties.APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED);
+ this.proxyEndpoint =
+ PropertyUtil.propertyAsString(
+ httpClientProperties, HttpClientProperties.PROXY_ENDPOINT, null);
}
@VisibleForTesting
@@ -103,6 +109,10 @@ class ApacheHttpClientConfigurations {
if (useIdleConnectionReaperEnabled != null) {
apacheHttpClientBuilder.useIdleConnectionReaper(useIdleConnectionReaperEnabled);
}
+ if (proxyEndpoint != null) {
+ apacheHttpClientBuilder.proxyConfiguration(
+
ProxyConfiguration.builder().endpoint(URI.create(proxyEndpoint)).build());
+ }
}
public static ApacheHttpClientConfigurations create(Map<String, String>
properties) {
diff --git a/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
b/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
index 2a5ca2ece8..3e615cad70 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
@@ -52,6 +52,13 @@ public class HttpClientProperties implements Serializable {
public static final String CLIENT_TYPE_URLCONNECTION = "urlconnection";
public static final String CLIENT_TYPE_DEFAULT = CLIENT_TYPE_APACHE;
+
+ /**
+ * Used to configure the proxy endpoint. Used by both {@link
+ *
software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient.Builder} and
{@link
+ * software.amazon.awssdk.http.apache.ApacheHttpClient.Builder}
+ */
+ public static final String PROXY_ENDPOINT = "http-client.proxy-endpoint";
/**
* Used to configure the connection timeout in milliseconds for {@link
*
software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient.Builder}.
This flag only
diff --git
a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
index e71b02caef..ff8dafcf06 100644
---
a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
+++
b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
@@ -18,17 +18,20 @@
*/
package org.apache.iceberg.aws;
+import java.net.URI;
import java.time.Duration;
import java.util.Map;
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.util.PropertyUtil;
import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder;
+import software.amazon.awssdk.http.urlconnection.ProxyConfiguration;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
class UrlConnectionHttpClientConfigurations {
private Long httpClientUrlConnectionConnectionTimeoutMs;
private Long httpClientUrlConnectionSocketTimeoutMs;
+ private String proxyEndpoint;
private UrlConnectionHttpClientConfigurations() {}
@@ -46,6 +49,9 @@ class UrlConnectionHttpClientConfigurations {
this.httpClientUrlConnectionSocketTimeoutMs =
PropertyUtil.propertyAsNullableLong(
httpClientProperties,
HttpClientProperties.URLCONNECTION_SOCKET_TIMEOUT_MS);
+ this.proxyEndpoint =
+ PropertyUtil.propertyAsString(
+ httpClientProperties, HttpClientProperties.PROXY_ENDPOINT, null);
}
@VisibleForTesting
@@ -59,6 +65,10 @@ class UrlConnectionHttpClientConfigurations {
urlConnectionHttpClientBuilder.socketTimeout(
Duration.ofMillis(httpClientUrlConnectionSocketTimeoutMs));
}
+ if (proxyEndpoint != null) {
+ urlConnectionHttpClientBuilder.proxyConfiguration(
+
ProxyConfiguration.builder().endpoint(URI.create(proxyEndpoint)).build());
+ }
}
public static UrlConnectionHttpClientConfigurations create(
diff --git
a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
index 17ac7ca728..0f96ac0f6c 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
@@ -24,6 +24,7 @@ import
org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
public class TestHttpClientConfigurations {
@@ -34,6 +35,7 @@ public class TestHttpClientConfigurations {
properties.put(HttpClientProperties.URLCONNECTION_CONNECTION_TIMEOUT_MS,
"80");
properties.put(HttpClientProperties.APACHE_SOCKET_TIMEOUT_MS, "100");
properties.put(HttpClientProperties.APACHE_CONNECTION_TIMEOUT_MS, "200");
+ properties.put(HttpClientProperties.PROXY_ENDPOINT, "http://proxy:8080";);
UrlConnectionHttpClientConfigurations
urlConnectionHttpClientConfigurations =
UrlConnectionHttpClientConfigurations.create(properties);
UrlConnectionHttpClient.Builder urlConnectionHttpClientBuilder =
@@ -46,6 +48,9 @@ public class TestHttpClientConfigurations {
Mockito.verify(spyUrlConnectionHttpClientBuilder).socketTimeout(Duration.ofMillis(90));
Mockito.verify(spyUrlConnectionHttpClientBuilder).connectionTimeout(Duration.ofMillis(80));
+ Mockito.verify(spyUrlConnectionHttpClientBuilder)
+ .proxyConfiguration(
+
Mockito.any(software.amazon.awssdk.http.urlconnection.ProxyConfiguration.class));
}
@Test
@@ -64,6 +69,9 @@ public class TestHttpClientConfigurations {
.connectionTimeout(Mockito.any(Duration.class));
Mockito.verify(spyUrlConnectionHttpClientBuilder, Mockito.never())
.socketTimeout(Mockito.any(Duration.class));
+ Mockito.verify(spyUrlConnectionHttpClientBuilder, Mockito.never())
+ .proxyConfiguration(
+
Mockito.any(software.amazon.awssdk.http.urlconnection.ProxyConfiguration.class));
}
@Test
@@ -80,6 +88,7 @@ public class TestHttpClientConfigurations {
properties.put(HttpClientProperties.APACHE_MAX_CONNECTIONS, "104");
properties.put(HttpClientProperties.APACHE_TCP_KEEP_ALIVE_ENABLED, "true");
properties.put(HttpClientProperties.APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED,
"false");
+ properties.put(HttpClientProperties.PROXY_ENDPOINT, "http://proxy:8080";);
ApacheHttpClientConfigurations apacheHttpClientConfigurations =
ApacheHttpClientConfigurations.create(properties);
ApacheHttpClient.Builder apacheHttpClientBuilder =
ApacheHttpClient.builder();
@@ -96,6 +105,8 @@ public class TestHttpClientConfigurations {
Mockito.verify(spyApacheHttpClientBuilder).maxConnections(104);
Mockito.verify(spyApacheHttpClientBuilder).tcpKeepAlive(true);
Mockito.verify(spyApacheHttpClientBuilder).useIdleConnectionReaper(false);
+ Mockito.verify(spyApacheHttpClientBuilder)
+ .proxyConfiguration(Mockito.any(ProxyConfiguration.class));
}
@Test
@@ -123,5 +134,7 @@ public class TestHttpClientConfigurations {
Mockito.verify(spyApacheHttpClientBuilder,
Mockito.never()).tcpKeepAlive(Mockito.anyBoolean());
Mockito.verify(spyApacheHttpClientBuilder, Mockito.never())
.useIdleConnectionReaper(Mockito.anyBoolean());
+ Mockito.verify(spyApacheHttpClientBuilder, Mockito.never())
+ .proxyConfiguration(Mockito.any(ProxyConfiguration.class));
}
}
diff --git a/docs/docs/aws.md b/docs/docs/aws.md
index 2bd6636670..b2f3241654 100644
--- a/docs/docs/aws.md
+++ b/docs/docs/aws.md
@@ -578,11 +578,12 @@ In contrast, URL Connection HTTP Client optimizes for
minimum dependencies and s
For more details of configuration, see sections [URL Connection HTTP Client
Configurations](#url-connection-http-client-configurations) and [Apache HTTP
Client Configurations](#apache-http-client-configurations).
-Configure the following property to set the type of HTTP client:
+Configurations for the HTTP client can be set via catalog properties. Below is
an overview of available configurations:
-| Property | Default | Description
|
-|------------------|---------|------------------------------------------------------------------------------------------------------------|
-| http-client.type | apache | Types of HTTP Client. <br/> `urlconnection`:
URL Connection HTTP Client <br/> `apache`: Apache HTTP Client |
+| Property | Default | Description
|
+|----------------------------|---------|------------------------------------------------------------------------------------------------------------|
+| http-client.type | apache | Types of HTTP Client. <br/>
`urlconnection`: URL Connection HTTP Client <br/> `apache`: Apache HTTP Client |
+| http-client.proxy-endpoint | null | An optional proxy endpoint to use for
the HTTP client. |
#### URL Connection HTTP Client Configurations
