This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6abffc4 [ASTERIXDB-2803][EXT] Support MFA for S3
6abffc4 is described below
commit 6abffc45bbf599ae5eab341b514a8a8b93339db6
Author: Hussain Towaileb <[email protected]>
AuthorDate: Mon Nov 16 16:17:46 2020 +0300
[ASTERIXDB-2803][EXT] Support MFA for S3
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added MFA support for S3.
- Added a test case for the above item.
Change-Id: I0e9d37f83befb5da7a7028171a405462273f2056
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/8883
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
---
.../create-with-session-token/test.000.ddl.sqlpp | 38 ++++++++++++++++++++++
.../create-with-session-token/test.001.query.sqlpp | 21 ++++++++++++
.../create-with-session-token/test.099.ddl.sqlpp | 20 ++++++++++++
.../s3/create-with-session-token/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset_s3.xml | 5 +++
.../external/util/ExternalDataConstants.java | 1 +
.../asterix/external/util/ExternalDataUtils.java | 11 ++++++-
7 files changed, 96 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
new file mode 100644
index 0000000..3d62e15
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("sessionToken"="dummySessionToken"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001";),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.001.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.001.query.sqlpp
new file mode 100644
index 0000000..8ec9cc0
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.001.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.099.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/create-with-session-token/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/create-with-session-token/result.001.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/create-with-session-token/result.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/create-with-session-token/result.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index e05b859..f6c2a2f 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -25,6 +25,11 @@
<output-dir compare="Text">common/json/json</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="create-with-session-token">
+ <output-dir compare="Text">create-with-session-token</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="common/json/gz">
<placeholder name="adapter" value="S3" />
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index dcd58de..812cb96 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -293,6 +293,7 @@ public class ExternalDataConstants {
public static final String REGION_FIELD_NAME = "region";
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
public static final String SECRET_ACCESS_KEY_FIELD_NAME =
"secretAccessKey";
+ public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static final String DEFINITION_FIELD_NAME = "definition";
public static final String SERVICE_END_POINT_FIELD_NAME =
"serviceEndpoint";
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 617903d..036dbad 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -78,6 +78,8 @@ import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.ListBlobsOptions;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.regions.Region;
@@ -710,13 +712,20 @@ public class ExternalDataUtils {
// TODO(Hussain): Need to ensure that all required parameters are
present in a previous step
String accessKeyId =
configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
String secretAccessKey =
configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME);
+ String sessionToken =
configuration.get(ExternalDataConstants.AwsS3.SESSION_TOKEN_FIELD_NAME);
String regionId =
configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME);
String serviceEndpoint =
configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
S3ClientBuilder builder = S3Client.builder();
// Credentials
- AwsBasicCredentials credentials =
AwsBasicCredentials.create(accessKeyId, secretAccessKey);
+ AwsCredentials credentials;
+ if (sessionToken != null) {
+ credentials = AwsSessionCredentials.create(accessKeyId,
secretAccessKey, sessionToken);
+ } else {
+ credentials = AwsBasicCredentials.create(accessKeyId,
secretAccessKey);
+ }
+
builder.credentialsProvider(StaticCredentialsProvider.create(credentials));
// Validate the region
