>From Ritik Raj <[email protected]>:
Ritik Raj has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20510?usp=email )
Change subject: [NO ISSUE][CLOUD] Introduced config MaxConcurrentRequests for
Azure
......................................................................
[NO ISSUE][CLOUD] Introduced config MaxConcurrentRequests for Azure
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-68999
Change-Id: If17c939416587329f98521ec74e4ba2fb685fbdc
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
M
asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
M
asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
8 files changed, 32 insertions(+), 13 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/10/20510/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
index f79a7ea..30bb2d9 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
@@ -31,7 +31,7 @@
import org.junit.runners.Parameterized.Parameters;
/**
- * Runs the cluster state runtime tests with with full parallelism on node
controllers.
+ * Runs the cluster state runtime tests with full parallelism on node
controllers.
*/
@RunWith(Parameterized.class)
public class ClusterStateExecutionFullParallelismTest {
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index eb32737..5ba7e09 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -22,6 +22,7 @@
"cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
+ "cloud.storage.az.blob.max.concurrent.requests" : 20,
"cloud.storage.bucket" : "",
"cloud.storage.cache.policy" : "selective",
"cloud.storage.debug.mode.enabled" : false,
@@ -34,6 +35,7 @@
"cloud.storage.list.eventually.consistent" : false,
"cloud.storage.prefix" : "",
"cloud.storage.region" : "",
+ "cloud.storage.s3.enable.crt.client" : true,
"cloud.storage.scheme" : "",
"cloud.storage.sweep.threshold.percentage" : 0.9,
"cloud.write.buffer.size" : 8388608,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 99fcba0..cd54262 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -22,6 +22,7 @@
"cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
+ "cloud.storage.az.blob.max.concurrent.requests" : 20,
"cloud.storage.bucket" : "",
"cloud.storage.cache.policy" : "selective",
"cloud.storage.debug.mode.enabled" : false,
@@ -34,6 +35,7 @@
"cloud.storage.list.eventually.consistent" : false,
"cloud.storage.prefix" : "",
"cloud.storage.region" : "",
+ "cloud.storage.s3.enable.crt.client" : true,
"cloud.storage.scheme" : "",
"cloud.storage.sweep.threshold.percentage" : 0.9,
"cloud.write.buffer.size" : 8388608,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 98d1a44..a65783a 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -22,6 +22,7 @@
"cloud.requests.max.pending.http.connections" : 10000,
"cloud.storage.allocation.percentage" : 0.8,
"cloud.storage.anonymous.auth" : false,
+ "cloud.storage.az.blob.max.concurrent.requests" : 20,
"cloud.storage.bucket" : "",
"cloud.storage.cache.policy" : "selective",
"cloud.storage.debug.mode.enabled" : false,
@@ -34,6 +35,7 @@
"cloud.storage.list.eventually.consistent" : false,
"cloud.storage.prefix" : "",
"cloud.storage.region" : "",
+ "cloud.storage.s3.enable.crt.client" : true,
"cloud.storage.scheme" : "",
"cloud.storage.sweep.threshold.percentage" : 0.9,
"cloud.write.buffer.size" : 8388608,
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
index 31b160d..b154fae 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
@@ -29,9 +29,6 @@
import com.azure.identity.DefaultAzureCredentialBuilder;
public class AzBlobStorageClientConfig {
- // Ref:
https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id
- static final int MAX_CONCURRENT_REQUESTS = 20;
-
private final int writeBufferSize;
private final String region;
private final String endpoint;
@@ -44,15 +41,18 @@
private final int writeMaxRequestsPerSeconds;
private final int readMaxRequestsPerSeconds;
private final boolean storageDisableSSLVerify;
+ private final int maxConcurrentRequests;
public AzBlobStorageClientConfig(String region, String endpoint, String
prefix, boolean anonymousAuth,
- long profilerLogInterval, String bucket, int writeBufferSize) {
- this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
bucket, 1, 0, 0, writeBufferSize, false);
+ long profilerLogInterval, String bucket, int writeBufferSize, int
maxConcurrentRequests) {
+ this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
bucket, 1, 0, 0, writeBufferSize, false,
+ maxConcurrentRequests);
}
public AzBlobStorageClientConfig(String region, String endpoint, String
prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, long tokenAcquireTimeout,
int writeMaxRequestsPerSeconds,
- int readMaxRequestsPerSeconds, int writeBufferSize, boolean
storageDisableSSLVerify) {
+ int readMaxRequestsPerSeconds, int writeBufferSize, boolean
storageDisableSSLVerify,
+ int maxConcurrentRequests) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -64,6 +64,7 @@
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
this.writeBufferSize = writeBufferSize;
this.storageDisableSSLVerify = storageDisableSSLVerify;
+ this.maxConcurrentRequests = maxConcurrentRequests;
}
public static AzBlobStorageClientConfig of(CloudProperties
cloudProperties) {
@@ -72,7 +73,7 @@
cloudProperties.getProfilerLogInterval(),
cloudProperties.getStorageBucket(),
cloudProperties.getTokenAcquireTimeout(),
cloudProperties.getWriteMaxRequestsPerSecond(),
cloudProperties.getReadMaxRequestsPerSecond(),
cloudProperties.getWriteBufferSize(),
- cloudProperties.isStorageDisableSSLVerify());
+ cloudProperties.isStorageDisableSSLVerify(),
cloudProperties.getMaxAzBlobConcurrentRequests());
}
public static AzBlobStorageClientConfig of(Map<String, String>
configuration, int writeBufferSize) {
@@ -86,9 +87,10 @@
String region = "";
String prefix = "";
boolean anonymousAuth = false;
+ int maxConcurrentRequests = 20;
return new AzBlobStorageClientConfig(region, endPoint, prefix,
anonymousAuth, profilerLogInterval, bucket,
- writeBufferSize);
+ writeBufferSize, maxConcurrentRequests);
}
public String getRegion() {
@@ -138,4 +140,8 @@
public int getWriteBufferSize() {
return writeBufferSize;
}
+
+ public int getMaxConcurrentRequests() {
+ return maxConcurrentRequests;
+ }
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index 0bd16a8..cfb8e63 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -19,8 +19,6 @@
package org.apache.asterix.cloud.clients.azure.blobstorage;
-import static
org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageClientConfig.MAX_CONCURRENT_REQUESTS;
-
import java.io.ByteArrayOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -292,7 +290,7 @@
}
try {
- Flux.fromIterable(deleteMonos).flatMap(mono -> mono,
MAX_CONCURRENT_REQUESTS).then().block();
+ Flux.fromIterable(deleteMonos).flatMap(mono -> mono,
config.getMaxConcurrentRequests()).then().block();
} catch (Exception ex) {
throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE,
"DELETE", ex, paths.toString());
}
diff --git
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
index 44b4856..749f62e 100644
---
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
+++
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
@@ -117,7 +117,7 @@
URI blobStore = URI.create(blobServiceClient.getAccountUrl());
String endpoint = blobStore.getScheme() + "://" +
blobStore.getAuthority() + "/devstoreaccount1";
AzBlobStorageClientConfig config = new
AzBlobStorageClientConfig(MOCK_SERVER_REGION, endpoint, "", false, 0,
- PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize, true);
+ PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize, true, 20);
CLOUD_CLIENT = new AzBlobStorageCloudClient(config,
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 392282f..98d5137 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -73,6 +73,7 @@
CLOUD_STORAGE_FORCE_PATH_STYLE(BOOLEAN, false),
CLOUD_STORAGE_DISABLE_SSL_VERIFY(BOOLEAN, false),
CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT(BOOLEAN, false),
+ CLOUD_STORAGE_AZ_BLOB_MAX_CONCURRENT_REQUESTS(POSITIVE_INTEGER, 20),
CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT(BOOLEAN,
(Function<IApplicationConfig, Boolean>) app -> {
String endpoint = app.getString(CLOUD_STORAGE_ENDPOINT);
return endpoint == null || endpoint.isEmpty();
@@ -119,6 +120,8 @@
case CLOUD_STORAGE_FORCE_PATH_STYLE:
case CLOUD_STORAGE_DISABLE_SSL_VERIFY:
case CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT:
+ case CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT:
+ case CLOUD_STORAGE_AZ_BLOB_MAX_CONCURRENT_REQUESTS:
return Section.COMMON;
default:
return Section.NC;
@@ -194,6 +197,8 @@
+ "after they are deleted";
case CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT:
return "Indicates whether or not to use the AWS CRT S3
client for async requests";
+ case CLOUD_STORAGE_AZ_BLOB_MAX_CONCURRENT_REQUESTS:
+ return "The maximum number of concurrent requests to Azure
Blob Storage";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -305,6 +310,10 @@
return
accessor.getInt(Option.CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS);
}
+ public int getMaxAzBlobConcurrentRequests() {
+ return
accessor.getInt(Option.CLOUD_STORAGE_AZ_BLOB_MAX_CONCURRENT_REQUESTS);
+ }
+
public int getRequestsHttpConnectionAcquireTimeout() {
return
accessor.getInt(Option.CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT);
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20510?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: phoenix
Gerrit-Change-Id: If17c939416587329f98521ec74e4ba2fb685fbdc
Gerrit-Change-Number: 20510
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <[email protected]>
