This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c70687f57f1bf93727b746a6bb787e374dfec0f4 Author: Hussain Towaileb <[email protected]> AuthorDate: Thu Apr 15 17:47:31 2021 +0300 [ASTERIXDB-2874][EXT] S3 external datasets: add support to anonymous authentication - user model changes: no - storage format changes: no - interface changes: no Details: - Added support to creating external datasets with no credentials (anonymous). Change-Id: I9b0b83ac51b4c262b52d657f5cdcf6c0f818451e Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11023 Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Till Westmann <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- .../s3/anonymous_no_auth/test.000.ddl.sqlpp | 36 +++++++++++++++++ .../s3/anonymous_no_auth/test.001.ddl.sqlpp | 36 +++++++++++++++++ .../s3/anonymous_no_auth/test.002.ddl.sqlpp | 35 +++++++++++++++++ .../s3/anonymous_no_auth/test.099.ddl.sqlpp | 20 ++++++++++ .../runtimets/testsuite_external_dataset_s3.xml | 7 ++++ .../asterix/common/exceptions/ErrorCode.java | 1 + .../src/main/resources/asx_errormsg/en.properties | 1 + .../asterix/external/util/ExternalDataUtils.java | 45 +++++++++++++++++----- 8 files changed, 172 insertions(+), 9 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp new file mode 100644 index 0000000..13cfe8a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.000.ddl.sqlpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +drop dataset test if exists; +CREATE EXTERNAL DATASET test(test) USING S3 ( +("accessKeyId"="dummyAccessKey"), +("region"="us-west-2"), +("serviceEndpoint"="http://localhost:8001";), +("container"="playground"), +("definition"="json-data/reviews/single-line/json"), +("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp new file mode 100644 index 0000000..b8d0945 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.001.ddl.sqlpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +drop dataset test if exists; +CREATE EXTERNAL DATASET test(test) USING S3 ( +("secretAccessKey"="dummySecretKey"), +("region"="us-west-2"), +("serviceEndpoint"="http://localhost:8001";), +("container"="playground"), +("definition"="json-data/reviews/single-line/json"), +("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp new file mode 100644 index 0000000..9eda057 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.002.ddl.sqlpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +drop dataset test if exists; +CREATE EXTERNAL DATASET test(test) USING S3 ( +("region"="us-west-2"), +("serviceEndpoint"="http://localhost:8001";), +("container"="playground"), +("definition"="json-data"), +("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp new file mode 100644 index 0000000..548e632 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/anonymous_no_auth/test.099.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 6557230..9bc463c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -148,6 +148,13 @@ <output-dir compare="Text">non-s3-region</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/s3"> + <compilation-unit name="anonymous_no_auth"> + <output-dir compare="Text">anonymous_no_auth</output-dir> + <expected-error>ASX3119: Parameter 'secretAccessKey' is required if 'accessKeyId' is provided</expected-error> + <expected-error>ASX3119: Parameter 'accessKeyId' is required if 'secretAccessKey' is provided</expected-error> + </compilation-unit> + </test-case> <test-case FilePath="external-dataset/common"> <compilation-unit name="query-with-limit-plan"> <placeholder name="adapter" value="S3" /> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 210ce0f..7e32e0b 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -348,6 +348,7 @@ public enum ErrorCode implements IError { INPUT_DECODE_FAILURE(3116), FAILED_TO_PARSE_MALFORMED_LOG_RECORD(3117), ACTIVE_ENTITY_NOT_RUNNING(3118), + REQUIRED_PARAM_IF_PARAM_IS_PRESENT(3119), // Lifecycle management errors DUPLICATE_PARTITION_ID(4000), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 93fa828..345b9d8 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -353,6 +353,7 @@ 3116 = Failed to decode input 3117 = Failed to parse record, malformed log record 3118 = Active Entity %1$s is not running (it is %2$s) +3119 = Parameter '%1$s' is required if '%2$s' is provided # Lifecycle management errors 4000 = Partition id %1$s for node %2$s already in use by node %3$s diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 2c29e60..f5c62a5 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -18,7 +18,10 @@ */ package org.apache.asterix.external.util; +import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT; +import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED; +import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME; @@ -84,8 +87,9 @@ import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobItem; import com.azure.storage.blob.models.ListBlobsOptions; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.exception.SdkException; @@ -726,8 +730,8 @@ public class ExternalDataUtils { */ public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException { // TODO(Hussain): Need to ensure that all required parameters are present in a previous step - String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME); - String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME); + String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME); + String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME); String sessionToken = configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME); String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME); String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME); @@ -735,14 +739,23 @@ public class ExternalDataUtils { S3ClientBuilder builder = S3Client.builder(); // Credentials - AwsCredentials credentials; - if (sessionToken != null) { - credentials = AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken); + AwsCredentialsProvider credentialsProvider; + + // No auth required + if (accessKeyId == null) { + credentialsProvider = AnonymousCredentialsProvider.create(); } else { - credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey); + // auth required, check for temporary or permanent credentials + if (sessionToken != null) { + credentialsProvider = StaticCredentialsProvider + .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken)); + } else { + credentialsProvider = + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey)); + } } - builder.credentialsProvider(StaticCredentialsProvider.create(credentials)); + builder.credentialsProvider(credentialsProvider); builder.region(Region.of(regionId)); // Validate the service endpoint if present @@ -777,10 +790,24 @@ public class ExternalDataUtils { throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT); } + // Both parameters should be passed, or neither should be passed (for anonymous/no auth) + String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME); + String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME); + if (accessKeyId == null || secretAccessKey == null) { + // If one is passed, the other is required + if (accessKeyId != null) { + throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME, + ACCESS_KEY_ID_FIELD_NAME); + } else if (secretAccessKey != null) { + throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME, + SECRET_ACCESS_KEY_FIELD_NAME); + } + } + validateIncludeExclude(configuration); // Check if the bucket is present - S3Client s3Client = buildAwsS3Client(configuration);; + S3Client s3Client = buildAwsS3Client(configuration); S3Response response; boolean useOldApi = false; String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
