This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0b011da678d81f9dac405f7a115ec66e13294faa Author: Ali Alsuliman <[email protected]> AuthorDate: Mon Apr 7 17:00:48 2025 -0700 [ASTERIXDB-3592][OTH] Increase S3 client HTTP pool size - user model changes: no - storage format changes: no - interface changes: no Details: Increase S3 client HTTP pool size from 50 to 1000. Ext-ref: MB-66048 Change-Id: I4ac5bdc21d9577513f777b3fedf6ffb96b03754a Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19628 Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Murtadha Hubail <[email protected]> Integration-Tests: Murtadha Hubail <[email protected]> --- .../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 + .../api/cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 + .../api/cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 + asterixdb/asterix-cloud/pom.xml | 4 ++++ .../apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java | 12 +++++++++--- .../apache/asterix/cloud/clients/aws/s3/S3CloudClient.java | 5 +++++ .../org/apache/asterix/common/config/CloudProperties.java | 10 +++++++++- asterixdb/pom.xml | 5 +++++ 8 files changed, 35 insertions(+), 4 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index c6d7ba9de5..29061ec7c3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -17,6 +17,7 @@ "cloud.max.read.requests.per.second" : 1500, "cloud.max.write.requests.per.second" : 250, "cloud.profiler.log.interval" : 5, + "cloud.requests.max.http.connections" : 1000, "cloud.storage.allocation.percentage" : 0.8, "cloud.storage.anonymous.auth" : false, "cloud.storage.bucket" : "", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 562e195afc..f2ea15baad 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -17,6 +17,7 @@ "cloud.max.read.requests.per.second" : 1500, "cloud.max.write.requests.per.second" : 250, "cloud.profiler.log.interval" : 5, + "cloud.requests.max.http.connections" : 1000, "cloud.storage.allocation.percentage" : 0.8, "cloud.storage.anonymous.auth" : false, "cloud.storage.bucket" : "", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 132fa5b281..685d28b7bc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -17,6 +17,7 @@ "cloud.max.read.requests.per.second" : 1500, "cloud.max.write.requests.per.second" : 250, "cloud.profiler.log.interval" : 5, + "cloud.requests.max.http.connections" : 1000, "cloud.storage.allocation.percentage" : 0.8, "cloud.storage.anonymous.auth" : false, "cloud.storage.bucket" : "", diff --git a/asterixdb/asterix-cloud/pom.xml b/asterixdb/asterix-cloud/pom.xml index 6f952b088d..419ac4d762 100644 --- a/asterixdb/asterix-cloud/pom.xml +++ b/asterixdb/asterix-cloud/pom.xml @@ -246,6 +246,10 @@ <groupId>software.amazon.awssdk.crt</groupId> <artifactId>aws-crt</artifactId> </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>apache-client</artifactId> + </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java index 0b9b15c26e..20727dea3d 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java @@ -41,15 +41,16 @@ public final class S3ClientConfig { private final long tokenAcquireTimeout; private final int readMaxRequestsPerSeconds; private final int writeMaxRequestsPerSeconds; + private final int requestsMaxHttpConnections; public S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth, long profilerLogInterval, int writeBufferSize) { - this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0); + this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0, 0); } private S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth, long profilerLogInterval, int writeBufferSize, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds, - int readMaxRequestsPerSeconds) { + int readMaxRequestsPerSeconds, int requestsMaxHttpConnections) { this.region = Objects.requireNonNull(region, "region"); this.endpoint = endpoint; this.prefix = Objects.requireNonNull(prefix, "prefix"); @@ -59,6 +60,7 @@ public final class S3ClientConfig { this.tokenAcquireTimeout = tokenAcquireTimeout; this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds; this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds; + this.requestsMaxHttpConnections = requestsMaxHttpConnections; } public static S3ClientConfig of(CloudProperties cloudProperties) { @@ -66,7 +68,7 @@ public final class S3ClientConfig { cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth(), cloudProperties.getProfilerLogInterval(), cloudProperties.getWriteBufferSize(), cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(), - cloudProperties.getReadMaxRequestsPerSecond()); + cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getRequestsMaxHttpConnections()); } public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) { @@ -124,6 +126,10 @@ public final class S3ClientConfig { return readMaxRequestsPerSeconds; } + public int getRequestsMaxHttpConnections() { + return requestsMaxHttpConnections; + } + private boolean isS3Mock() { return endpoint != null && !endpoint.isEmpty(); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java index 224ede46ae..01a8b0244b 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java @@ -63,6 +63,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; @@ -325,6 +326,10 @@ public final class S3CloudClient implements ICloudClient { S3ClientBuilder builder = S3Client.builder(); builder.credentialsProvider(config.createCredentialsProvider()); builder.region(Region.of(config.getRegion())); + if (config.getRequestsMaxHttpConnections() > 0) { + builder.httpClientBuilder( + ApacheHttpClient.builder().maxConnections(config.getRequestsMaxHttpConnections())); + } if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) { URI uri; try { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java index 7b6fb6e004..1e3fe75033 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java @@ -64,7 +64,8 @@ public class CloudProperties extends AbstractProperties { CLOUD_WRITE_BUFFER_SIZE( getRangedIntegerType(5, Integer.MAX_VALUE), StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE)), - CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50); + CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50), + CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS(POSITIVE_INTEGER, 1000); private final IOptionType interpreter; private final Object defaultValue; @@ -96,6 +97,7 @@ public class CloudProperties extends AbstractProperties { case CLOUD_MAX_READ_REQUESTS_PER_SECOND: case CLOUD_WRITE_BUFFER_SIZE: case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD: + case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS: return Section.COMMON; default: return Section.NC; @@ -163,6 +165,8 @@ public class CloudProperties extends AbstractProperties { return "The write buffer size in bytes. (default: 8MB, min: 5MB)"; case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD: return "The number of cloud reads for re-evaluating an eviction plan. (default: 50)"; + case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS: + return "The maximum number of HTTP connections to use for cloud requests per node. (default: 1000)"; default: throw new IllegalStateException("NYI: " + this); } @@ -258,4 +262,8 @@ public class CloudProperties extends AbstractProperties { public int getEvictionPlanReevaluationThreshold() { return accessor.getInt(Option.CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD); } + + public int getRequestsMaxHttpConnections() { + return accessor.getInt(Option.CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS); + } } diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index e90283ddd1..35f9219b01 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -1633,6 +1633,11 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>apache-client</artifactId> + <version>${awsjavasdk.version}</version> + </dependency> <dependency> <groupId>software.amazon.awssdk</groupId> <artifactId>http-client-spi</artifactId>
