This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0381a66782891e2abd6bf9290fff007d2e363b0f Author: Hussain Towaileb <[email protected]> AuthorDate: Sun Feb 7 18:08:06 2021 +0300 [ASTERIXDB-2826][EXT]: S3 allow using custom non-S3 regions - user model changes: no - storage format changes: no - interface changes: no Details: - Currently, if an external dataset is created using a non-S3 region, querying the external dataset fails. This change allows using non-S3 regions which is possible for other S3-compatible storages to have. Change-Id: If7f89eb2adbf9fb9a00d83c339099da05ab0ad06 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9843 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- .../non-s3-region/external_dataset.000.ddl.sqlpp | 41 ++++++++++++++++++++++ .../non-s3-region/external_dataset.099.ddl.sqlpp | 20 +++++++++++ .../runtimets/testsuite_external_dataset.xml | 5 +++ .../asterix/external/util/ExternalDataUtils.java | 13 +------ 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.000.ddl.sqlpp new file mode 100644 index 0000000..3b22f11 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.000.ddl.sqlpp @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Creating an external dataset should not fail when non-s3-region is used (some-new-region) + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +drop dataset test if exists; +create external dataset test(test) using S3 ( + ("accessKeyId"="dummyAccessKey"), + ("secretAccessKey"="dummySecretKey"), + ("region"="some-new-region"), + ("serviceEndpoint"="http://localhost:8001";), + ("container"="playground"), + ("definition"="json-data/reviews/single-line/json"), + ("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.099.ddl.sqlpp new file mode 100644 index 0000000..548e632 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.099.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml index 87ba14d..9cded35 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml @@ -119,6 +119,11 @@ <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> + <test-case FilePath="external-dataset"> + <compilation-unit name="aws/s3/non-s3-region"> + <output-dir compare="Text">aws/s3/non-s3-region</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="include-exclude"> <test-case FilePath="external-dataset"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8206d4c..c0a7a4d 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -32,7 +32,6 @@ import java.util.Arrays; import java.util.EnumMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.regex.Matcher; import org.apache.asterix.common.exceptions.AsterixException; @@ -628,17 +627,7 @@ public class ExternalDataUtils { // Credentials AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey); builder.credentialsProvider(StaticCredentialsProvider.create(credentials)); - - // Validate the region - List<Region> supportedRegions = S3Client.serviceMetadata().regions(); - Optional<Region> selectedRegion = - supportedRegions.stream().filter(region -> region.id().equalsIgnoreCase(regionId)).findFirst(); - - if (!selectedRegion.isPresent()) { - throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, - String.format("region %s is not supported", regionId)); - } - builder.region(selectedRegion.get()); + builder.region(Region.of(regionId)); // Validate the service endpoint if present if (serviceEndpoint != null) {
