This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e5014515f6 [ASTERIXDB-3598][CONF] Add cloud storage configuration
e5014515f6 is described below
commit e5014515f64e7a7601e0999740c7abc8d4fa0fd9
Author: Murtadha Hubail <[email protected]>
AuthorDate: Sun Apr 13 16:19:32 2025 +0300
[ASTERIXDB-3598][CONF] Add cloud storage configuration
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Add an option to disable SSL verification on the cloud storage
to be used for testing purposes.
- Add an to force path style access on cloud storage (ASTERIXDB-3599)
Change-Id: I932b85cea015daa91b512b0219d10d4b9075ca9b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19648
Integration-Tests: Jenkins <[email protected]>
Tested-by: Murtadha Hubail <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Reviewed-by: Michael Blow <[email protected]>
---
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 ++
.../cluster_state_1_full.1.regexadm | 2 ++
.../cluster_state_1_less.1.regexadm | 2 ++
asterixdb/asterix-cloud/pom.xml | 5 +++++
.../cloud/clients/aws/s3/S3ClientConfig.java | 20 ++++++++++++++---
.../cloud/clients/aws/s3/S3CloudClient.java | 24 ++++++++++++---------
.../cloud/clients/aws/s3/S3ParallelDownloader.java | 25 ++++++++++++++--------
.../asterix/common/config/CloudProperties.java | 20 ++++++++++++++++-
8 files changed, 77 insertions(+), 23 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 29061ec7c3..025334892d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -24,8 +24,10 @@
"cloud.storage.cache.policy" : "selective",
"cloud.storage.debug.mode.enabled" : false,
"cloud.storage.debug.sweep.threshold.size" : 1073741824,
+ "cloud.storage.disable.ssl.verify" : false,
"cloud.storage.disk.monitor.interval" : 120,
"cloud.storage.endpoint" : "",
+ "cloud.storage.force.path.style" : false,
"cloud.storage.index.inactive.duration.threshold" : 360,
"cloud.storage.prefix" : "",
"cloud.storage.region" : "",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index f2ea15baad..14a4710c7d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -24,8 +24,10 @@
"cloud.storage.cache.policy" : "selective",
"cloud.storage.debug.mode.enabled" : false,
"cloud.storage.debug.sweep.threshold.size" : 1073741824,
+ "cloud.storage.disable.ssl.verify" : false,
"cloud.storage.disk.monitor.interval" : 120,
"cloud.storage.endpoint" : "",
+ "cloud.storage.force.path.style" : false,
"cloud.storage.index.inactive.duration.threshold" : 360,
"cloud.storage.prefix" : "",
"cloud.storage.region" : "",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 685d28b7bc..8e9b1879fd 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -24,8 +24,10 @@
"cloud.storage.cache.policy" : "selective",
"cloud.storage.debug.mode.enabled" : false,
"cloud.storage.debug.sweep.threshold.size" : 1073741824,
+ "cloud.storage.disable.ssl.verify" : false,
"cloud.storage.disk.monitor.interval" : 120,
"cloud.storage.endpoint" : "",
+ "cloud.storage.force.path.style" : false,
"cloud.storage.index.inactive.duration.threshold" : 360,
"cloud.storage.prefix" : "",
"cloud.storage.region" : "",
diff --git a/asterixdb/asterix-cloud/pom.xml b/asterixdb/asterix-cloud/pom.xml
index 419ac4d762..efe8673954 100644
--- a/asterixdb/asterix-cloud/pom.xml
+++ b/asterixdb/asterix-cloud/pom.xml
@@ -250,6 +250,11 @@
<groupId>software.amazon.awssdk</groupId>
<artifactId>apache-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>netty-nio-client</artifactId>
+ <version>${awsjavasdk.version}</version>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index 20727dea3d..5414099017 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -42,15 +42,18 @@ public final class S3ClientConfig {
private final int readMaxRequestsPerSeconds;
private final int writeMaxRequestsPerSeconds;
private final int requestsMaxHttpConnections;
+ private final boolean forcePathStyle;
+ private final boolean disableSslVerify;
public S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize) {
- this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
writeBufferSize, 1, 0, 0, 0);
+ this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
writeBufferSize, 1, 0, 0, 0, false, false);
}
private S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth,
long profilerLogInterval, int writeBufferSize, long
tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
- int readMaxRequestsPerSeconds, int requestsMaxHttpConnections) {
+ int readMaxRequestsPerSeconds, int requestsMaxHttpConnections,
boolean forcePathStyle,
+ boolean disableSslVerify) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -61,6 +64,8 @@ public final class S3ClientConfig {
this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds;
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
this.requestsMaxHttpConnections = requestsMaxHttpConnections;
+ this.forcePathStyle = forcePathStyle;
+ this.disableSslVerify = disableSslVerify;
}
public static S3ClientConfig of(CloudProperties cloudProperties) {
@@ -68,7 +73,8 @@ public final class S3ClientConfig {
cloudProperties.getStoragePrefix(),
cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getProfilerLogInterval(),
cloudProperties.getWriteBufferSize(),
cloudProperties.getTokenAcquireTimeout(),
cloudProperties.getWriteMaxRequestsPerSecond(),
- cloudProperties.getReadMaxRequestsPerSecond(),
cloudProperties.getRequestsMaxHttpConnections());
+ cloudProperties.getReadMaxRequestsPerSecond(),
cloudProperties.getRequestsMaxHttpConnections(),
+ cloudProperties.isStorageForcePathStyle(),
cloudProperties.isStorageDisableSSLVerify());
}
public static S3ClientConfig of(Map<String, String> configuration, int
writeBufferSize) {
@@ -130,6 +136,14 @@ public final class S3ClientConfig {
return requestsMaxHttpConnections;
}
+ public boolean isDisableSslVerify() {
+ return disableSslVerify;
+ }
+
+ public boolean isForcePathStyle() {
+ return forcePathStyle;
+ }
+
private boolean isS3Mock() {
return endpoint != null && !endpoint.isEmpty();
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 01a8b0244b..e4ba4b41a6 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -26,7 +26,6 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
@@ -63,6 +62,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.SdkHttpConfigurationOption;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
@@ -79,6 +80,7 @@ import
software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Error;
import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.utils.AttributeMap;
@ThreadSafe
public final class S3CloudClient implements ICloudClient {
@@ -326,19 +328,21 @@ public final class S3CloudClient implements ICloudClient {
S3ClientBuilder builder = S3Client.builder();
builder.credentialsProvider(config.createCredentialsProvider());
builder.region(Region.of(config.getRegion()));
+ builder.forcePathStyle(config.isForcePathStyle());
+
+ AttributeMap.Builder customHttpConfigBuilder = AttributeMap.builder();
if (config.getRequestsMaxHttpConnections() > 0) {
- builder.httpClientBuilder(
-
ApacheHttpClient.builder().maxConnections(config.getRequestsMaxHttpConnections()));
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.MAX_CONNECTIONS,
+ config.getRequestsMaxHttpConnections());
}
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
- URI uri;
- try {
- uri = new URI(config.getEndpoint());
- } catch (URISyntaxException ex) {
- throw new IllegalArgumentException(ex);
- }
- builder.endpointOverride(uri);
+ builder.endpointOverride(URI.create(config.getEndpoint()));
+ }
+ if (config.isDisableSslVerify()) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES,
true);
}
+ SdkHttpClient httpClient =
ApacheHttpClient.builder().buildWithDefaults(customHttpConfigBuilder.build());
+ builder.httpClient(httpClient);
return builder.build();
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index 2eb9f0908d..b5259e9974 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -37,6 +37,9 @@ import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.util.annotations.ThreadSafe;
+import software.amazon.awssdk.http.SdkHttpConfigurationOption;
+import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
@@ -50,6 +53,7 @@ import
software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;
import software.amazon.awssdk.transfer.s3.model.FailedFileDownload;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
+import software.amazon.awssdk.utils.AttributeMap;
@ThreadSafe
class S3ParallelDownloader implements IParallelDownloader {
@@ -170,24 +174,28 @@ class S3ParallelDownloader implements IParallelDownloader
{
}
private static S3AsyncClient createAsyncClient(S3ClientConfig config) {
- if (config.isLocalS3Provider()) {
- // CRT client is not supported by S3Mock
- return createS3AsyncClient(config);
- } else {
- // CRT could provide a better performance when used with an actual
S3
+ // CRT client is not supported by all local S3 providers, but provides
a better performance with AWS S3
+ if (!config.isLocalS3Provider()) {
return createS3CrtAsyncClient(config);
}
+ return createS3AsyncClient(config);
}
private static S3AsyncClient createS3AsyncClient(S3ClientConfig config) {
S3AsyncClientBuilder builder = S3AsyncClient.builder();
builder.credentialsProvider(config.createCredentialsProvider());
builder.region(Region.of(config.getRegion()));
-
+ builder.forcePathStyle(config.isForcePathStyle());
+ AttributeMap.Builder customHttpConfigBuilder = AttributeMap.builder();
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.endpointOverride(URI.create(config.getEndpoint()));
}
-
+ if (config.isDisableSslVerify()) {
+
customHttpConfigBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES,
true);
+ }
+ SdkAsyncHttpClient nettyHttpClient =
+
NettyNioAsyncHttpClient.builder().buildWithDefaults(customHttpConfigBuilder.build());
+ builder.httpClient(nettyHttpClient);
return builder.build();
}
@@ -195,11 +203,10 @@ class S3ParallelDownloader implements IParallelDownloader
{
S3CrtAsyncClientBuilder builder = S3AsyncClient.crtBuilder();
builder.credentialsProvider(config.createCredentialsProvider());
builder.region(Region.of(config.getRegion()));
-
+ builder.forcePathStyle(config.isForcePathStyle());
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.endpointOverride(URI.create(config.getEndpoint()));
}
-
return builder.build();
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 1e3fe75033..1af4824267 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -65,7 +65,9 @@ public class CloudProperties extends AbstractProperties {
getRangedIntegerType(5, Integer.MAX_VALUE),
StorageUtil.getIntSizeInBytes(8,
StorageUtil.StorageUnit.MEGABYTE)),
CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD(POSITIVE_INTEGER, 50),
- CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS(POSITIVE_INTEGER, 1000);
+ CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS(POSITIVE_INTEGER, 1000),
+ CLOUD_STORAGE_FORCE_PATH_STYLE(BOOLEAN, false),
+ CLOUD_STORAGE_DISABLE_SSL_VERIFY(BOOLEAN, false);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -98,6 +100,8 @@ public class CloudProperties extends AbstractProperties {
case CLOUD_WRITE_BUFFER_SIZE:
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
+ case CLOUD_STORAGE_FORCE_PATH_STYLE:
+ case CLOUD_STORAGE_DISABLE_SSL_VERIFY:
return Section.COMMON;
default:
return Section.NC;
@@ -167,6 +171,12 @@ public class CloudProperties extends AbstractProperties {
return "The number of cloud reads for re-evaluating an
eviction plan. (default: 50)";
case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
return "The maximum number of HTTP connections to use for
cloud requests per node. (default: 1000)";
+ case CLOUD_STORAGE_FORCE_PATH_STYLE:
+ return "Indicates whether or not to force path style when
accessing the cloud storage. (default:"
+ + " false)";
+ case CLOUD_STORAGE_DISABLE_SSL_VERIFY:
+ return "Indicates whether or not to disable SSL
certificate verification on the cloud storage. "
+ + "(default: false)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -266,4 +276,12 @@ public class CloudProperties extends AbstractProperties {
public int getRequestsMaxHttpConnections() {
return accessor.getInt(Option.CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS);
}
+
+ public boolean isStorageForcePathStyle() {
+ return accessor.getBoolean(Option.CLOUD_STORAGE_FORCE_PATH_STYLE);
+ }
+
+ public boolean isStorageDisableSSLVerify() {
+ return accessor.getBoolean(Option.CLOUD_STORAGE_DISABLE_SSL_VERIFY);
+ }
}
