This is an automated email from the ASF dual-hosted git repository. dlych pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2eb7cebaddcbdcadeb535e16c17a4f2cbd041d60 Author: Hussain Towaileb <[email protected]> AuthorDate: Wed Jan 13 20:57:26 2021 +0300 [ASTERIXDB-2816][EXT] Support Azure SAS authentication + connection string - user model changes: no - storage format changes: no - interface changes: no Details: - Add support to passing shared access signature and connection string as authentication methods. Change-Id: I18d578465dbb03d129b02d2c3076db3d3f1df4aa Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9204 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- asterixdb/NOTICE | 2 +- asterixdb/asterix-app/pom.xml | 4 ++ .../apache/asterix/test/common/TestConstants.java | 31 ++++++-- .../apache/asterix/test/common/TestExecutor.java | 28 ++++++-- .../AzureBlobStorageExternalDatasetTest.java | 18 +++++ .../invalid-auth-methods/test.000.ddl.sqlpp | 38 ++++++++++ .../invalid-auth-methods/test.099.ddl.sqlpp | 20 ++++++ .../invalid-no-auth/test.000.ddl.sqlpp | 35 +++++++++ .../invalid-no-auth/test.099.ddl.sqlpp | 20 ++++++ .../valid-auth-methods/test.000.ddl.sqlpp | 36 ++++++++++ .../valid-auth-methods/test.001.query.sqlpp | 21 ++++++ .../valid-auth-methods/test.099.ddl.sqlpp | 20 ++++++ .../auth-methods/valid-auth-methods/result.001.adm | 1 + ...stsuite_external_dataset_azure_blob_storage.xml | 56 +++++++++++++++ .../asterix/common/exceptions/ErrorCode.java | 2 + .../src/main/resources/asx_errormsg/en.properties | 2 + .../external/util/ExternalDataConstants.java | 6 +- .../asterix/external/util/ExternalDataUtils.java | 84 ++++++++++++++++++---- asterixdb/pom.xml | 47 ++++++++++++ hyracks-fullstack/NOTICE | 2 +- 20 files changed, 445 insertions(+), 28 deletions(-) diff --git a/asterixdb/NOTICE b/asterixdb/NOTICE index b4729a8..4aabe27 100644 --- a/asterixdb/NOTICE +++ b/asterixdb/NOTICE @@ -1,5 +1,5 @@ Apache AsterixDB -Copyright 2015-2020 The Apache Software Foundation +Copyright 2015-2021 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index 7f8085d..e309132 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -861,6 +861,10 @@ <artifactId>azure-storage-blob</artifactId> </dependency> <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-storage-common</artifactId> + </dependency> + <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-1.2-api</artifactId> </dependency> diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java index ef2ec6c..9188033 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java @@ -36,14 +36,37 @@ public class TestConstants { + "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_DEFAULT + "\")"; // Azure blob storage constants and place holders - public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%accountName%"; + // account name + public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%azureblob-accountname%"; public static final String AZURE_AZURITE_ACCOUNT_NAME_DEFAULT = "devstoreaccount1"; - public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%accountKey%"; + + // account key + public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%azureblob-accountkey%"; public static final String AZURE_AZURITE_ACCOUNT_KEY_DEFAULT = - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; - public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%blobEndpoint%"; + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + + // SAS token: this is generated and assigned at runtime at the start of the test + public static final String AZURE_SAS_TOKEN_PLACEHOLDER = "%azureblob-sas%"; + public static String sasToken = ""; + + // blob endpoint + public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%azureblob-blobendpoint%"; public static final String AZURE_BLOB_ENDPOINT_DEFAULT = "http://localhost:20000/"; + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT; + + // connection string with account name & account key + public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER = + "%azureblob-connectionstringaccountkey%"; + public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY = "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER + + ";AccountKey=" + AZURE_ACCOUNT_KEY_PLACEHOLDER + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER; + + // connection string with account name & sas token + public static final String AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER = "%azureblob-connectionstringsas%"; + public static final String AZURE_CONNECTION_STRING_SAS_TOKEN = + "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER + ";SharedAccessSignature=" + AZURE_SAS_TOKEN_PLACEHOLDER + + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER; + + // azure template and default template public static final String AZURE_TEMPLATE = "(\"accountName\"=\"" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT + "\"),\n" + "(\"accountKey\"=\"" + AZURE_AZURITE_ACCOUNT_KEY_DEFAULT + "\"),\n" + "(\"blobEndpoint\"=\"" + AZURE_BLOB_ENDPOINT_PLACEHOLDER + "\")"; diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 00d1034..e202ddf 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -2068,28 +2068,46 @@ public class TestExecutor { } protected String applyExternalDatasetSubstitution(String str, List<Placeholder> placeholders) { + // This replaces the full template of parameters depending on the adapter type for (Placeholder placeholder : placeholders) { + // For adapter placeholder, it means we have a template to replace if (placeholder.getName().equals("adapter")) { str = str.replace("%adapter%", placeholder.getValue()); // Early terminate if there are no template place holders to replace if (noTemplateRequired(str)) { - return str; + continue; } if (placeholder.getValue().equalsIgnoreCase("S3")) { - return applyS3Substitution(str, placeholders); + str = applyS3Substitution(str, placeholders); } else if (placeholder.getValue().equalsIgnoreCase("AzureBlob")) { - return applyAzureSubstitution(str, placeholders); - } else { - return str; + str = applyAzureSubstitution(str, placeholders); } + } else { + // Any other place holders, just replace with the value + str = str.replace("%" + placeholder.getName() + "%", placeholder.getValue()); } } + // This replaces specific external dataset placeholders + str = str.replace(TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER, + TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY); + str = str.replace(TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER, + TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN); + str = str.replace(TestConstants.AZURE_ACCOUNT_NAME_PLACEHOLDER, + TestConstants.AZURE_AZURITE_ACCOUNT_NAME_DEFAULT); + str = str.replace(TestConstants.AZURE_ACCOUNT_KEY_PLACEHOLDER, TestConstants.AZURE_AZURITE_ACCOUNT_KEY_DEFAULT); + str = str.replace(TestConstants.AZURE_SAS_TOKEN_PLACEHOLDER, TestConstants.sasToken); + str = replaceExternalEndpoint(str); + return str; } + protected String replaceExternalEndpoint(String str) { + return str.replace(TestConstants.AZURE_BLOB_ENDPOINT_PLACEHOLDER, TestConstants.AZURE_BLOB_ENDPOINT_DEFAULT); + } + protected boolean noTemplateRequired(String str) { return !str.contains("%template%"); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java index da9f912..27a46c1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java @@ -29,6 +29,8 @@ import java.net.InetSocketAddress; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.BitSet; import java.util.Collection; import java.util.HashMap; @@ -39,6 +41,7 @@ import java.util.Set; import java.util.zip.GZIPOutputStream; import org.apache.asterix.common.api.INcApplicationContext; +import org.apache.asterix.test.common.TestConstants; import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.runtime.ExecutionTestUtil; import org.apache.asterix.test.runtime.LangExecutionUtil; @@ -64,6 +67,10 @@ import org.junit.runners.Parameterized.Parameters; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.common.sas.AccountSasPermission; +import com.azure.storage.common.sas.AccountSasResourceType; +import com.azure.storage.common.sas.AccountSasService; +import com.azure.storage.common.sas.AccountSasSignatureValues; @Ignore @RunWith(Parameterized.class) @@ -164,12 +171,23 @@ public class AzureBlobStorageExternalDatasetTest { blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionString).buildClient(); LOGGER.info("Azurite Blob Service client created successfully"); + // Generate the SAS token for the SAS test cases + TestConstants.sasToken = generateSasToken(); + // Create the container and upload some json files PREPARE_PLAYGROUND_CONTAINER.run(); PREPARE_FIXED_DATA_CONTAINER.run(); PREPARE_MIXED_DATA_CONTAINER.run(); } + private static String generateSasToken() { + OffsetDateTime expiry = OffsetDateTime.now().plus(1, ChronoUnit.YEARS); + AccountSasService service = AccountSasService.parse("b"); + AccountSasPermission permission = AccountSasPermission.parse("acdlpruw"); + AccountSasResourceType type = AccountSasResourceType.parse("cos"); + return blobServiceClient.generateAccountSas(new AccountSasSignatureValues(expiry, permission, service, type)); + } + /** * Creates a container and fills it with some files for testing purpose. */ diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp new file mode 100644 index 0000000..c4ae026 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +// bad case: more than one authentication method is provided at once +drop dataset test if exists; +CREATE EXTERNAL DATASET test(test) USING AZUREBLOB ( +("accountName"="%azureblob-accountname%"), +("%azureblob-credentialsname-1%"="%azureblob-credentialsvalue-1%"), +("%azureblob-credentialsname-2%"="%azureblob-credentialsvalue-2%"), +("blobEndpoint"="%azureblob-blobendpoint%"), +("container"="playground"), +("definition"="json-data/reviews/single-line/json"), +("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp new file mode 100644 index 0000000..548e632 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp new file mode 100644 index 0000000..29fd14b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +// bad case: no auth method is provided +drop dataset test if exists; +CREATE EXTERNAL DATASET test(test) USING AZUREBLOB ( +("blobEndpoint"="%azureblob-blobendpoint%"), +("container"="playground"), +("definition"="json-data/reviews/single-line/json"), +("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp new file mode 100644 index 0000000..548e632 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp new file mode 100644 index 0000000..2a7eadd --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +drop type test if exists; +create type test as open { +}; + +drop dataset test if exists; +CREATE EXTERNAL DATASET test(test) USING AZUREBLOB ( +("accountName"="%azureblob-accountname%"), +("%azureblob-credentialsname%"="%azureblob-credentialsvalue%"), +("blobEndpoint"="%azureblob-blobendpoint%"), +("container"="playground"), +("definition"="json-data/reviews/single-line/json"), +("format"="json") +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp new file mode 100644 index 0000000..8ec9cc0 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; +select count(*) `count` from test; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp new file mode 100644 index 0000000..548e632 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm new file mode 100644 index 0000000..187a8cb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm @@ -0,0 +1 @@ +{ "count": 100 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index a715e7e..df60e60 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -18,6 +18,62 @@ ! under the License. !--> <test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp"> + <test-group name="authentication"> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="valid-auth-methods"> + <placeholder name="azureblob-credentialsname" value="accountKey" /> + <placeholder name="azureblob-credentialsvalue" value="%azureblob-accountkey%" /> + <output-dir compare="Text">valid-auth-methods</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="valid-auth-methods"> + <placeholder name="azureblob-credentialsname" value="sharedAccessSignature" /> + <placeholder name="azureblob-credentialsvalue" value="%azureblob-sas%" /> + <output-dir compare="Text">valid-auth-methods</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="valid-auth-methods"> + <placeholder name="azureblob-credentialsname" value="connectionString" /> + <placeholder name="azureblob-credentialsvalue" value="%azureblob-connectionstringaccountkey%" /> + <output-dir compare="Text">valid-auth-methods</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="valid-auth-methods"> + <placeholder name="azureblob-credentialsname" value="connectionString" /> + <placeholder name="azureblob-credentialsvalue" value="%azureblob-connectionstringsas%" /> + <output-dir compare="Text">valid-auth-methods</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="invalid-auth-methods"> + <placeholder name="azureblob-credentialsname-1" value="accountKey" /> + <placeholder name="azureblob-credentialsvalue-1" value="%azureblob-accountkey%" /> + <placeholder name="azureblob-credentialsname-2" value="connectionString" /> + <placeholder name="azureblob-credentialsvalue-2" value="%azureblob-connectionstringaccountkey%" /> + <output-dir compare="Text">invalid-auth-methods</output-dir> + <expected-error>ASX1133: Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="invalid-auth-methods"> + <placeholder name="azureblob-credentialsname-1" value="sharedAccessSignature" /> + <placeholder name="azureblob-credentialsvalue-1" value="%azureblob-sas%" /> + <placeholder name="azureblob-credentialsname-2" value="connectionString" /> + <placeholder name="azureblob-credentialsvalue-2" value="%azureblob-connectionstringaccountkey%" /> + <output-dir compare="Text">invalid-auth-methods</output-dir> + <expected-error>ASX1133: Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/azure_blob_storage/auth-methods"> + <compilation-unit name="invalid-no-auth"> + <output-dir compare="Text">invalid-no-auth</output-dir> + <expected-error>ASX1134: No authentication parameters provided</expected-error> + </compilation-unit> + </test-case> + </test-group> <test-group name="external-dataset"> <test-case FilePath="external-dataset"> <compilation-unit name="common/json/json"> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 6dbb46b..d662c8d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -222,6 +222,8 @@ public class ErrorCode { public static final int ILLEGAL_RIGHT_OUTER_JOIN = 1130; public static final int SYNONYM_EXISTS = 1131; public static final int INVALID_HINT = 1132; + public static final int ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED = 1133; + public static final int NO_AUTH_METHOD_PROVIDED = 1134; // Feed errors public static final int DATAFLOW_ILLEGAL_STATE = 3001; public static final int UTIL_DATAFLOW_UTILS_TUPLE_TOO_LARGE = 3002; diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index db8a70d..3eed1d8 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -219,6 +219,8 @@ 1130 = Illegal use of RIGHT OUTER JOIN 1131 = A synonym with this name %1$s already exists 1132 = Invalid specification for hint %1$s. %2$s +1133 = Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature +1134 = No authentication parameters provided # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index 326f53d..6be694b 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -304,15 +304,17 @@ public class ExternalDataConstants { throw new AssertionError("do not instantiate"); } + public static final String CONTAINER_NAME_FIELD_NAME = "container"; + public static final String DEFINITION_FIELD_NAME = "definition"; + public static final String CONNECTION_STRING_FIELD_NAME = "connectionString"; public static final String ACCOUNT_NAME_FIELD_NAME = "accountName"; public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey"; public static final String SHARED_ACCESS_SIGNATURE_FIELD_NAME = "sharedAccessSignature"; - public static final String CONTAINER_NAME_FIELD_NAME = "container"; - public static final String DEFINITION_FIELD_NAME = "definition"; public static final String BLOB_ENDPOINT_FIELD_NAME = "blobEndpoint"; public static final String ENDPOINT_SUFFIX_FIELD_NAME = "endpointSuffix"; // Connection string requires PascalCase (MyFieldFormat) + public static final String CONNECTION_STRING_CONNECTION_STRING = "ConnectionString"; public static final String CONNECTION_STRING_ACCOUNT_NAME = "AccountName"; public static final String CONNECTION_STRING_ACCOUNT_KEY = "AccountKey"; public static final String CONNECTION_STRING_SHARED_ACCESS_SIGNATURE = "SharedAccessSignature"; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java index 44ae25f..537933e 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java @@ -18,10 +18,17 @@ */ package org.apache.asterix.external.util; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_KEY; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_BLOB_ENDPOINT; import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ENDPOINT_SUFFIX; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_SHARED_ACCESS_SIGNATURE; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME; +import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE; import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE; @@ -819,25 +826,70 @@ public class ExternalDataUtils { public static BlobServiceClient buildAzureClient(Map<String, String> configuration) throws CompilationException { // TODO(Hussain): Need to ensure that all required parameters are present in a previous step - String accountName = configuration.get(ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME); - String accountKey = configuration.get(ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME); - String blobEndpoint = configuration.get(ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME); - String endpointSuffix = configuration.get(ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME); - - // format: name1=value1;name2=value2;.... - // TODO(Hussain): This will be different when SAS (Shared Access Signature) is introduced - StringBuilder connectionString = new StringBuilder(); - connectionString.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName).append(";"); - connectionString.append(CONNECTION_STRING_ACCOUNT_KEY).append("=").append(accountKey).append(";"); - connectionString.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint).append(";"); - - if (endpointSuffix != null) { - connectionString.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=").append(endpointSuffix) + String connectionString = configuration.get(CONNECTION_STRING_FIELD_NAME); + String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME); + String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME); + String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME); + String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME); + String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME); + + // Constructor the connection string + // Connection string format: name1=value1;name2=value2;.... + StringBuilder connectionStringBuilder = new StringBuilder(); + BlobServiceClientBuilder builder = new BlobServiceClientBuilder(); + + boolean authMethodFound = false; + + if (connectionString != null) { + // connection string + authMethodFound = true; + connectionStringBuilder.append(connectionString).append(";"); + } + + if (accountName != null && accountKey != null) { + if (authMethodFound) { + throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED); + } + authMethodFound = true; + // account name + account key + connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName) + .append(";").append(CONNECTION_STRING_ACCOUNT_KEY).append("=").append(accountKey).append(";"); + } + + if (accountName != null && sharedAccessSignature != null) { + if (authMethodFound) { + throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED); + } + authMethodFound = true; + // account name + shared access token + connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName) + .append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=") + .append(sharedAccessSignature).append(";"); + } + + if (!authMethodFound) { + throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED); + } + + // Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed + if (blobEndpoint != null) { + connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint) .append(";"); + if (endpointSuffix != null) { + String endpointSuffixUpdated; + if (blobEndpoint.endsWith("/")) { + endpointSuffixUpdated = + endpointSuffix.startsWith("/") ? endpointSuffix.substring(1) : endpointSuffix; + } else { + endpointSuffixUpdated = endpointSuffix.startsWith("/") ? endpointSuffix : "/" + endpointSuffix; + } + connectionStringBuilder.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=") + .append(endpointSuffixUpdated).append(";"); + } } try { - return new BlobServiceClientBuilder().connectionString(connectionString.toString()).buildClient(); + return builder.connectionString(connectionStringBuilder.toString()).buildClient(); } catch (Exception ex) { throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage()); } @@ -876,6 +928,8 @@ public class ExternalDataUtils { WarningUtil.forAsterix(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES); collector.warn(warning); } + } catch (CompilationException ex) { + throw ex; } catch (Exception ex) { throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage()); } diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index a6d4e7f..f8992d5 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -1542,6 +1542,53 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-storage-common</artifactId> + <version>${azurejavasdk.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-handler</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-handler-proxy</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec-http</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec-http2</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-buffer</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-epoll</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-unix-common</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + </exclusion> + </exclusions> + </dependency> <!-- Azure Blob Storage end --> <dependency> <groupId>org.mindrot</groupId> diff --git a/hyracks-fullstack/NOTICE b/hyracks-fullstack/NOTICE index 95fe98a..57c5843 100644 --- a/hyracks-fullstack/NOTICE +++ b/hyracks-fullstack/NOTICE @@ -1,5 +1,5 @@ Apache Hyracks and Algebricks -Copyright 2015-2020 The Apache Software Foundation +Copyright 2015-2021 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
