This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9e278ea217 [ASTERIXDB-3549][EXT]: Add support to enable cross-region
for AWS S3 external collections
9e278ea217 is described below
commit 9e278ea21746d9284c941c879b8bca812c45f199
Author: Hussain Towaileb <[email protected]>
AuthorDate: Thu Jan 16 09:33:20 2025 +0300
[ASTERIXDB-3549][EXT]: Add support to enable cross-region for AWS S3
external collections
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
cross-region support allows the AWS SDK to automatically
detect the correct region and cache it if an incorrect
region is specified.
Ext-ref: MB-64169
Change-Id: I3d552457cfa77aa2abe4a2ecb7d77d2d36cb70d3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19320
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Tested-by: Hussain Towaileb <[email protected]>
---
.../boolean-value/external_dataset.000.ddl.sqlpp | 38 ++++++++++++++++++++++
.../boolean-value/external_dataset.099.ddl.sqlpp | 20 ++++++++++++
.../external_dataset.000.ddl.sqlpp | 38 ++++++++++++++++++++++
.../external_dataset.099.ddl.sqlpp | 20 ++++++++++++
.../runtimets/testsuite_external_dataset_s3.xml | 11 +++++++
.../asterix/external/util/aws/s3/S3AuthUtils.java | 13 ++++++++
.../asterix/external/util/aws/s3/S3Constants.java | 1 +
7 files changed, 141 insertions(+)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/boolean-value/external_dataset.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/boolean-value/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000000..426de41076
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/boolean-value/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-1"),
+ ("crossRegion"="true"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="playground"),
+ ("definition"="json-data/reviews/single-line/json"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/boolean-value/external_dataset.099.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/boolean-value/external_dataset.099.ddl.sqlpp
new file mode 100644
index 0000000000..548e63267e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/boolean-value/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/non-boolean-value/external_dataset.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/non-boolean-value/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000000..f63bc9800b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/non-boolean-value/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-1"),
+ ("crossRegion"="non-boolean-value"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="playground"),
+ ("definition"="json-data/reviews/single-line/json"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/non-boolean-value/external_dataset.099.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/non-boolean-value/external_dataset.099.ddl.sqlpp
new file mode 100644
index 0000000000..548e63267e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/cross-region/non-boolean-value/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 8a5151c4b2..b34dba93a5 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -1078,6 +1078,17 @@
<expected-error>Provided S3 region is not supported:
'some-new-region'</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="cross-region/boolean-value">
+ <output-dir compare="Text">cross-region/boolean-value</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="cross-region/non-boolean-value">
+ <output-dir compare="Text">cross-region/non-boolean-value</output-dir>
+ <expected-error>c</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/s3">
<compilation-unit name="anonymous_no_auth">
<output-dir compare="Text">anonymous_no_auth</output-dir>
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 2ed16a19ec..57889eacbb 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@ -28,6 +28,7 @@ import static
org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTa
import static
org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
import static
org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
+import static
org.apache.asterix.external.util.aws.s3.S3Constants.CROSS_REGION_FIELD_NAME;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_EXPIRED_TOKEN;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
@@ -134,10 +135,12 @@ public class S3AuthUtils {
String serviceEndpoint =
configuration.get(SERVICE_END_POINT_FIELD_NAME);
Region region = validateAndGetRegion(regionId);
+ boolean crossRegion =
validateAndGetCrossRegion(configuration.get(CROSS_REGION_FIELD_NAME));
AwsCredentialsProvider credentialsProvider =
buildCredentialsProvider(appCtx, configuration);
S3ClientBuilder builder = S3Client.builder();
builder.region(region);
+ builder.crossRegionAccessEnabled(crossRegion);
builder.credentialsProvider(credentialsProvider);
// Validate the service endpoint if present
@@ -212,6 +215,16 @@ public class S3AuthUtils {
}
}
+ public static boolean validateAndGetCrossRegion(String crossRegion) throws
CompilationException {
+ if (crossRegion == null) {
+ return false;
+ }
+ if (!"true".equalsIgnoreCase(crossRegion) &&
!"false".equalsIgnoreCase(crossRegion)) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE,
CROSS_REGION_FIELD_NAME, "true, false");
+ }
+ return Boolean.parseBoolean(crossRegion);
+ }
+
private static boolean noAuth(Map<String, String> configuration) {
return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME,
ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
SESSION_TOKEN_FIELD_NAME) == null;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index 95c5040c40..a67cd64f5f 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -25,6 +25,7 @@ public class S3Constants {
// Authentication specific parameters
public static final String REGION_FIELD_NAME = "region";
+ public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
public static final String SECRET_ACCESS_KEY_FIELD_NAME =
"secretAccessKey";
