This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5ea8dd7a31 [ASTERIXDB-3619][CONF] Add cloud properties to configure
HTTP client used by cloud client
5ea8dd7a31 is described below
commit 5ea8dd7a3193019b2ed0ffe52381b74ab33a6b43
Author: Ali Alsuliman <[email protected]>
AuthorDate: Tue Jun 3 17:48:30 2025 -0700
[ASTERIXDB-3619][CONF] Add cloud properties to configure HTTP client used
by cloud client
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add the following cloud properties:
CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS.
CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT.
- Apply the HTTP configurations to the (non-Crt) S3AsyncClient similar to
the S3Client:
- Now S3AsyncClient will use the default 1000 instead of 50 for
the number of concurrent connections.
- Allow configuring the HTTP connection acquire timeout:
- Default is increased to 2 minutes.
- Allow configuring the max HTTP pending connections:
- Default is 10000 (the same as S3 SDK default).
Ext-ref: MB-67039
Change-Id: I25ac7b00fea6433f0cb72b6bbd0f00b5356addb9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19870
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
---
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 ++
.../cluster_state_1_full.1.regexadm | 2 ++
.../cluster_state_1_less.1.regexadm | 2 ++
.../cloud/clients/aws/s3/S3ClientConfig.java | 21 ++++++++++++++---
.../cloud/clients/aws/s3/S3CloudClient.java | 9 ++++++++
.../cloud/clients/aws/s3/S3ParallelDownloader.java | 13 +++++++++++
.../asterix/common/config/CloudProperties.java | 26 +++++++++++++++++-----
7 files changed, 67 insertions(+), 8 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index d71912c06a..7ea1cb4d9d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -17,7 +17,9 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.http.connection.acquire.timeout" : 120,
"cloud.requests.max.http.connections" : 1000,
+ "cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index f16ae94cc7..e5e0da6a31 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -17,7 +17,9 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.http.connection.acquire.timeout" : 120,
"cloud.requests.max.http.connections" : 1000,
+ "cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 9871dfc85a..a218009f46 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -17,7 +17,9 @@
"cloud.max.read.requests.per.second" : 1500,
"cloud.max.write.requests.per.second" : 250,
"cloud.profiler.log.interval" : 5,
+ "cloud.requests.http.connection.acquire.timeout" : 120,
"cloud.requests.max.http.connections" : 1000,
+ "cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
"cloud.storage.bucket" : "",
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index e0449b6b89..025af0809f 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -42,6 +42,8 @@ public final class S3ClientConfig {
private final int readMaxRequestsPerSeconds;
private final int writeMaxRequestsPerSeconds;
private final int requestsMaxHttpConnections;
+ private final int requestsMaxPendingHttpConnections;
+ private final int requestsHttpConnectionAcquireTimeout;
private final boolean forcePathStyle;
private final boolean disableSslVerify;
private final boolean storageListEventuallyConsistent;
@@ -49,13 +51,14 @@ public final class S3ClientConfig {
public S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize) {
this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
writeBufferSize, 1, 0, 0, 0, false, false,
- false);
+ false, 0, 0);
}
private S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize, long
tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
int readMaxRequestsPerSeconds, int requestsMaxHttpConnections,
boolean forcePathStyle,
- boolean disableSslVerify, boolean storageListEventuallyConsistent)
{
+ boolean disableSslVerify, boolean storageListEventuallyConsistent,
int requestsMaxPendingHttpConnections,
+ int requestsHttpConnectionAcquireTimeout) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -66,6 +69,8 @@ public final class S3ClientConfig {
this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds;
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
this.requestsMaxHttpConnections = requestsMaxHttpConnections;
+ this.requestsMaxPendingHttpConnections =
requestsMaxPendingHttpConnections;
+ this.requestsHttpConnectionAcquireTimeout =
requestsHttpConnectionAcquireTimeout;
this.forcePathStyle = forcePathStyle;
this.disableSslVerify = disableSslVerify;
this.storageListEventuallyConsistent = storageListEventuallyConsistent;
@@ -78,7 +83,9 @@ public final class S3ClientConfig {
cloudProperties.getTokenAcquireTimeout(),
cloudProperties.getWriteMaxRequestsPerSecond(),
cloudProperties.getReadMaxRequestsPerSecond(),
cloudProperties.getRequestsMaxHttpConnections(),
cloudProperties.isStorageForcePathStyle(),
cloudProperties.isStorageDisableSSLVerify(),
- cloudProperties.isStorageListEventuallyConsistent());
+ cloudProperties.isStorageListEventuallyConsistent(),
+ cloudProperties.getRequestsMaxPendingHttpConnections(),
+ cloudProperties.getRequestsHttpConnectionAcquireTimeout());
}
public static S3ClientConfig of(Map<String, String> configuration, int
writeBufferSize) {
@@ -140,6 +147,14 @@ public final class S3ClientConfig {
return requestsMaxHttpConnections;
}
+ public int getRequestsMaxPendingHttpConnections() {
+ return requestsMaxPendingHttpConnections;
+ }
+
+ public int getRequestsHttpConnectionAcquireTimeout() {
+ return requestsHttpConnectionAcquireTimeout;
+ }
+
public boolean isDisableSslVerify() {
return disableSslVerify;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 8a28f53edf..81b96d7862 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.ByteBuffer;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -352,6 +353,14 @@ public final class S3CloudClient implements ICloudClient {
customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS,
config.getRequestsMaxHttpConnections());
}
+ if (config.getRequestsMaxPendingHttpConnections() > 0) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_PENDING_CONNECTION_ACQUIRES,
+ config.getRequestsMaxPendingHttpConnections());
+ }
+ if (config.getRequestsHttpConnectionAcquireTimeout() > 0) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_ACQUIRE_TIMEOUT,
+
Duration.ofSeconds(config.getRequestsHttpConnectionAcquireTimeout()));
+ }
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.endpointOverride(URI.create(config.getEndpoint()));
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index b5259e9974..4d27c5a8c5 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -20,6 +20,7 @@ package org.apache.asterix.cloud.clients.aws.s3;
import java.io.IOException;
import java.net.URI;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -193,6 +194,18 @@ class S3ParallelDownloader implements IParallelDownloader {
if (config.isDisableSslVerify()) {
customHttpConfigBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES,
true);
}
+ if (config.getRequestsMaxHttpConnections() > 0) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS,
+ config.getRequestsMaxHttpConnections());
+ }
+ if (config.getRequestsMaxPendingHttpConnections() > 0) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_PENDING_CONNECTION_ACQUIRES,
+ config.getRequestsMaxPendingHttpConnections());
+ }
+ if (config.getRequestsHttpConnectionAcquireTimeout() > 0) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.CONNECTION_ACQUIRE_TIMEOUT,
+
Duration.ofSeconds(config.getRequestsHttpConnectionAcquireTimeout()));
+ }
SdkAsyncHttpClient nettyHttpClient =
NettyNioAsyncHttpClient.builder().buildWithDefaults(customHttpConfigBuilder.build());
builder.httpClient(nettyHttpClient);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 4354e637ac..87eace3f34 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -66,6 +66,8 @@ public class CloudProperties extends AbstractProperties {
StorageUtil.getIntSizeInBytes(8,
StorageUtil.StorageUnit.MEGABYTE)),
CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50),
CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS(POSITIVE_INTEGER, 1000),
+ CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS(POSITIVE_INTEGER, 10000),
+ CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT(POSITIVE_INTEGER, 120),
CLOUD_STORAGE_FORCE_PATH_STYLE(BOOLEAN, false),
CLOUD_STORAGE_DISABLE_SSL_VERIFY(BOOLEAN, false),
CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT(BOOLEAN, false);
@@ -101,6 +103,8 @@ public class CloudProperties extends AbstractProperties {
case CLOUD_WRITE_BUFFER_SIZE:
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
+ case CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS:
+ case CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT:
case CLOUD_STORAGE_FORCE_PATH_STYLE:
case CLOUD_STORAGE_DISABLE_SSL_VERIFY:
case CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT:
@@ -131,7 +135,7 @@ public class CloudProperties extends AbstractProperties {
+ " request to open it. 'selective' caching will
act as the 'lazy' policy; however, "
+ " it allows to use the local disk(s) as a cache,
where pages and indexes can be "
+ " cached or evicted according to the pressure
imposed on the local disks."
- + " (default: 'lazy')";
+ + " (default: 'selective')";
case CLOUD_STORAGE_ALLOCATION_PERCENTAGE:
return "The percentage of the total disk space that should
be allocated for data storage when the"
+ " 'selective' caching policy is used. The
remaining will act as a buffer for "
@@ -157,9 +161,8 @@ public class CloudProperties extends AbstractProperties {
+ " CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE."
+ " (default: 0. I.e.,
CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE will be used by default)";
case CLOUD_PROFILER_LOG_INTERVAL:
- return "The waiting time (in minutes) to log cloud request
statistics (default: 0, which means"
- + " the profiler is disabled by default). The
minimum is 1 minute."
- + " NOTE: Enabling the profiler could perturb the
performance of cloud requests";
+ return "The waiting time (in minutes) to log cloud request
statistics. The minimum is 1 minute."
+ + " Note: by default, the logging is disabled.
Enabling it could perturb the performance of cloud requests";
case CLOUD_ACQUIRE_TOKEN_TIMEOUT:
return "The waiting time (in milliseconds) if a requesting
thread failed to acquire a token if the"
+ " rate limit of cloud requests exceeded
(default: 100, min: 1, and max: 5000)";
@@ -172,7 +175,12 @@ public class CloudProperties extends AbstractProperties {
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
return "The number of cloud reads for re-evaluating an
eviction plan. (default: 50)";
case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
- return "The maximum number of HTTP connections to use for
cloud requests per node. (default: 1000)";
+ return "The maximum number of HTTP connections to use
concurrently for cloud requests per node. (default: 1000)";
+ case CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS:
+ return "The maximum number of HTTP connections allowed to
wait for a connection per node. (default: 10000)";
+ case CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT:
+ return "The waiting time (in seconds) to acquire an HTTP
connection before failing the request."
+ + " (default: 120 seconds)";
case CLOUD_STORAGE_FORCE_PATH_STYLE:
return "Indicates whether or not to force path style when
accessing the cloud storage. (default:"
+ " false)";
@@ -282,6 +290,14 @@ public class CloudProperties extends AbstractProperties {
return accessor.getInt(Option.CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS);
}
+ public int getRequestsMaxPendingHttpConnections() {
+ return
accessor.getInt(Option.CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS);
+ }
+
+ public int getRequestsHttpConnectionAcquireTimeout() {
+ return
accessor.getInt(Option.CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT);
+ }
+
public boolean isStorageForcePathStyle() {
return accessor.getBoolean(Option.CLOUD_STORAGE_FORCE_PATH_STYLE);
}
