This is an automated email from the ASF dual-hosted git repository. htowaileb pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a9806215a4defc7d34c00b8473afd61099df4cbd Author: Hussain Towaileb <[email protected]> AuthorDate: Thu Jul 1 00:55:25 2021 +0300 [ASTERIXDB-2919] Azure Blob external datasets: add support to anonymous authentication - user model changes: no - storage format changes: no - interface changes: no Details: - Added support to creating external datasets with no credentials (anonymous). Change-Id: I19a161a3613371572fa26c786bab690e332fdd08 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11983 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- .../test/external_dataset/ExternalDatasetTestUtils.java | 4 ++-- .../microsoft/AzureBlobStorageExternalDatasetTest.java | 7 +++++++ .../test.000.ddl.sqlpp | 3 +-- .../test.099.ddl.sqlpp | 0 .../test.000.ddl.sqlpp | 3 +-- .../test.001.query.sqlpp} | 3 ++- .../test.099.ddl.sqlpp | 0 .../test.000.ddl.sqlpp | 0 .../test.001.query.sqlpp} | 3 ++- .../test.099.ddl.sqlpp | 0 .../result.001.adm | 1 + .../testsuite_external_dataset_azure_blob_storage.xml | 17 ++++++++++++++--- .../org/apache/asterix/common/exceptions/ErrorCode.java | 1 + .../src/main/resources/asx_errormsg/en.properties | 1 + .../apache/asterix/external/util/ExternalDataUtils.java | 13 +++++++------ 15 files changed, 39 insertions(+), 17 deletions(-) diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java index 3c129b4..c064281 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java @@ -18,7 +18,7 @@ */ package org.apache.asterix.test.external_dataset; -import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.*; +import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER; import java.io.BufferedWriter; import java.io.FileWriter; @@ -344,4 +344,4 @@ public class ExternalDatasetTestUtils { mixedDataLoader.upload(MIXED_DEFINITION + "json/NO-EXTENSION/" + "goodbye-world-2019", "{\"id\":" + 5 + "}"); mixedDataLoader.upload(MIXED_DEFINITION + "json/NO-EXTENSION/" + "goodbye-world-2020", "{\"id\":" + 6 + "}"); } -} \ No newline at end of file +} diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java index 8372d87..2d41686 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java @@ -66,11 +66,13 @@ import org.junit.runners.Parameterized.Parameters; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.PublicAccessType; import com.azure.storage.common.sas.AccountSasPermission; import com.azure.storage.common.sas.AccountSasResourceType; import com.azure.storage.common.sas.AccountSasService; import com.azure.storage.common.sas.AccountSasSignatureValues; +// TODO(Hussain): Need to run the test manually to ensure new tests (anonymous access) are working fine @Ignore @RunWith(Parameterized.class) @FixMethodOrder(MethodSorters.NAME_ASCENDING) @@ -100,6 +102,7 @@ public class AzureBlobStorageExternalDatasetTest { private static final String PLAYGROUND_CONTAINER = "playground"; private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude"; + private static final String PUBLIC_ACCESS_CONTAINER = "public-access-container"; // requires no authentication private static final String JSON_DEFINITION = "json-data/reviews/"; private static final String CSV_DEFINITION = "csv-data/reviews/"; private static final String TSV_DEFINITION = "tsv-data/reviews/"; @@ -116,6 +119,7 @@ public class AzureBlobStorageExternalDatasetTest { + "BlobEndpoint=" + BLOB_SERVICE_ENDPOINT + "/devstoreaccount1;"; private static BlobServiceClient blobServiceClient; private static BlobContainerClient playgroundContainer; + private static BlobContainerClient publicAccessContainer; protected TestCaseContext tcCtx; @@ -195,6 +199,8 @@ public class AzureBlobStorageExternalDatasetTest { LOGGER.info("creating container " + PLAYGROUND_CONTAINER); playgroundContainer = blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER); + publicAccessContainer = blobServiceClient.createBlobContainer(PUBLIC_ACCESS_CONTAINER); + publicAccessContainer.setAccessPolicy(PublicAccessType.CONTAINER, null); LOGGER.info("container " + PLAYGROUND_CONTAINER + " created successfully"); LOGGER.info("Adding JSON files"); @@ -342,6 +348,7 @@ public class AzureBlobStorageExternalDatasetTest { // Load the data playgroundContainer.getBlobClient(basePath + finalFileName).uploadFromFile(filePath.toString()); + publicAccessContainer.getBlobClient(basePath + finalFileName).uploadFromFile(filePath.toString()); if (copyToSubLevels) { playgroundContainer.getBlobClient(basePath + "level1a/" + finalFileName) .uploadFromFile(filePath.toString()); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp similarity index 93% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp index 43db107..d6bc862 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp @@ -25,10 +25,9 @@ drop type test if exists; create type test as open { }; -// bad case: no auth method is provided +// bad case: no auth method and no endpoint provided drop dataset test if exists; CREATE EXTERNAL DATASET test(test) USING AZUREBLOB ( -("blobEndpoint"="%azureblob-endpoint%"), ("container"="playground"), ("definition"="json-data/reviews/single-line/json"), ("format"="json") diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp similarity index 100% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp similarity index 94% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp index 43db107..598831e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp @@ -25,11 +25,10 @@ drop type test if exists; create type test as open { }; -// bad case: no auth method is provided drop dataset test if exists; CREATE EXTERNAL DATASET test(test) USING AZUREBLOB ( ("blobEndpoint"="%azureblob-endpoint%"), -("container"="playground"), +("container"="public-access"), ("definition"="json-data/reviews/single-line/json"), ("format"="json") ); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp similarity index 94% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp index 548e632..8ec9cc0 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp @@ -17,4 +17,5 @@ * under the License. */ -drop dataverse test if exists; \ No newline at end of file +use test; +select count(*) `count` from test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.099.ddl.sqlpp similarity index 100% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.099.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp similarity index 94% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp index 548e632..8ec9cc0 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp @@ -17,4 +17,5 @@ * under the License. */ -drop dataverse test if exists; \ No newline at end of file +use test; +select count(*) `count` from test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.099.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.099.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm new file mode 100644 index 0000000..187a8cb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm @@ -0,0 +1 @@ +{ "count": 100 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index 1e302da..27ba148 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -68,9 +68,20 @@ </compilation-unit> </test-case> <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> - <compilation-unit name="invalid-no-auth"> - <output-dir compare="Text">invalid-no-auth</output-dir> - <expected-error>ASX1134: No authentication parameters provided</expected-error> + <compilation-unit name="anonymous-no-auth-public-access-allowed"> + <output-dir compare="Text">anonymous-no-auth-public-access-allowed</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="anonymous-no-auth-public-access-not-allowed"> + <output-dir compare="Text">anonymous-no-auth-public-access-not-allowed</output-dir> + <expected-error>Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="anonymous-no-auth-no-endpoint"> + <output-dir compare="Text">anonymous-no-auth-no-endpoint</output-dir> + <expected-error>ASX1151: No authentication credentials provided, 'blobEndpoint' field is required for anonymous access</expected-error> </compilation-unit> </test-case> </test-group> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 4078ea2..e80dbfb 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -235,6 +235,7 @@ public enum ErrorCode implements IError { CANNOT_DROP_OBJECT_DEPENDENT_EXISTS(1148), ILLEGAL_FUNCTION_RECURSION(1149), ILLEGAL_FUNCTION_USE(1150), + NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS(1151), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 18898bb..cd4a289 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -237,6 +237,7 @@ 1148 = Cannot drop %1$s %2$s being used by %3$s %4$s 1149 = Illegal function recursion 1150 = Illegal use of function %1$s +1151 = No authentication credentials provided, '%1$s' field is required for anonymous access # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 8e94263..7872cef 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -899,7 +899,7 @@ public class ExternalDataUtils { String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME); String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME); - // Constructor the connection string + // Construct the connection string // Connection string format: name1=value1;name2=value2;.... StringBuilder connectionStringBuilder = new StringBuilder(); BlobServiceClientBuilder builder = new BlobServiceClientBuilder(); @@ -926,17 +926,12 @@ public class ExternalDataUtils { if (authMethodFound) { throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED); } - authMethodFound = true; // account name + shared access token connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName) .append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=") .append(sharedAccessSignature).append(";"); } - if (!authMethodFound) { - throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED); - } - // Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed if (blobEndpoint != null) { connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint) @@ -954,6 +949,12 @@ public class ExternalDataUtils { } } + // No credentials or endpoint provided + if (connectionStringBuilder.length() == 0) { + throw new CompilationException(ErrorCode.NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS, + BLOB_ENDPOINT_FIELD_NAME); + } + try { return builder.connectionString(connectionStringBuilder.toString()).buildClient(); } catch (Exception ex) {
