>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19444 )
Change subject: [ASTERIXDB-3565][EXT]: Add impersonate service account auth for
GCS
......................................................................
[ASTERIXDB-3565][EXT]: Add impersonate service account auth for GCS
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add support to impersonating a service account as
an authentication method for GCS links. This works
by providing a target service account to impersonate
and provide source credentials (ours) to use to
impersonate the account. There is no need to store
any temporary credentials/tokens as the SDK automatically
picks up the token generated, and if expired, automatically
refresh it for subsequent requests.
Ext-ref: MB-65121
Change-Id: Ie1d69faa45a03550c8e0fe66eb18c2ae53a8454a
---
M
asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
M
asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
A
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSAuthUtils.java
M
asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
16 files changed, 297 insertions(+), 128 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/44/19444/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
index 4c382d0..10f536b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
@@ -78,14 +78,15 @@
String type =
configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE);
if (ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3.equals(type)) {
- credentials = generateAwsCredentials(configuration);
+ return generateAwsCredentials(configuration);
+ } else {
+ // this should never happen
+ throw new IllegalArgumentException("Unsupported external source
type: " + type);
}
-
- return credentials;
}
// TODO: this can probably be refactored out into something that is
AWS-specific
- private Object generateAwsCredentials(Map<String, String> configuration)
+ private AwsSessionCredentials generateAwsCredentials(Map<String, String>
configuration)
throws HyracksDataException, CompilationException {
String key = configuration.get(ExternalDataConstants.KEY_ENTITY_ID);
AwsSessionCredentials credentials;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 3ffb080..c6d7ba9 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -61,6 +61,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "gcp.impersonate.service.account.duration" : 900,
"library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "DEBUG",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index aeb3cea..562e195 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -61,6 +61,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "gcp.impersonate.service.account.duration" : 900,
"library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "WARN",
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index b475612..132fa5b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -61,6 +61,7 @@
"compiler\.textsearchmemory" : 163840,
"compiler\.windowmemory" : 196608,
"default\.dir" : "target/io/dir/asterixdb",
+ "gcp.impersonate.service.account.duration" : 900,
"library\.deploy\.timeout" : 1800,
"log\.dir" : "logs/",
"log\.level" : "WARN",
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
index 63a8366..1f1874f 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
@@ -27,7 +27,7 @@
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.google.gcs.GCSUtils;
+import org.apache.asterix.external.util.google.gcs.GCSAuthUtils;
import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration;
import org.apache.asterix.runtime.writer.IExternalFileWriter;
import org.apache.asterix.runtime.writer.IExternalFileWriterFactory;
@@ -64,7 +64,7 @@
@Override
ICloudClient createCloudClient(IApplicationContext appCtx) throws
CompilationException {
GCSClientConfig config = GCSClientConfig.of(configuration,
writeBufferSize);
- return new GCSCloudClient(config, GCSUtils.buildClient(configuration),
+ return new GCSCloudClient(config, GCSAuthUtils.buildClient(appCtx,
configuration),
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 62437b2..d233606 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -64,7 +64,12 @@
75,
"Percentage of duration passed before assume role credentials
need to be refreshed, the value ranges "
+ "from 25 to 90, default is 75. For example, if the
value is set to 65, this means the "
- + "credentials need to be refreshed if 65% of the
total expiration duration is already passed");
+ + "credentials need to be refreshed if 65% of the
total expiration duration is already passed"),
+ GCP_IMPERSONATE_SERVICE_ACCOUNT_DURATION(
+ getRangedIntegerType(60, 3600),
+ 900,
+ "GCS impersonating service account duration in seconds. "
+ + "Range from 60 seconds (1 min) to 3600 seconds (1
hour)");
private final IOptionType type;
private final Object defaultValue;
@@ -94,6 +99,7 @@
case AZURE_REQUEST_TIMEOUT:
case AWS_ASSUME_ROLE_DURATION:
case AWS_REFRESH_ASSUME_ROLE_THRESHOLD_PERCENTAGE:
+ case GCP_IMPERSONATE_SERVICE_ACCOUNT_DURATION:
return Section.COMMON;
case CC_JAVA_OPTS:
case NC_JAVA_OPTS:
@@ -182,4 +188,8 @@
public int getAwsRefreshAssumeRoleThresholdPercentage() {
return
accessor.getInt(Option.AWS_REFRESH_ASSUME_ROLE_THRESHOLD_PERCENTAGE);
}
+
+ public int getGcpImpersonateServiceAccountDuration() {
+ return
accessor.getInt(Option.GCP_IMPERSONATE_SERVICE_ACCOUNT_DURATION);
+ }
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index d06a9e5..1fa078b 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -320,6 +320,8 @@
FAILED_EXTERNAL_CROSS_ACCOUNT_AUTHENTICATION(1213),
LONG_LIVED_CREDENTIALS_NEEDED_TO_ASSUME_ROLE(1214),
TEMPORARY_CREDENTIALS_CANNOT_BE_USED_TO_ASSUME_ROLE(1215),
+ NO_VALID_AUTHENTICATION_PARAMS_PROVIDED(1216),
+
NO_VALID_AUTHENTICATION_PARAMS_PROVIDED_TO_IMPERSONATE_SERVICE_ACCOUNT(1217),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index c1ffd13..e1cc667 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -322,6 +322,8 @@
1213 = Failed to perform cross-account authentication. Encountered error :
'%1$s'
1214 = Long-lived credentials are required to assume a role
1215 = Temporary credentials cannot be used to assume a role
+1216 = No valid authentication parameters were provided
+1217 = No valid authentication parameters were provided to impersonate service
account
# Feed Errors
3001 = Illegal state.
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
index 89da065..1ef3fcd 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
@@ -27,13 +27,14 @@
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
+import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import
org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmbedder;
import
org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.google.gcs.GCSUtils;
+import org.apache.asterix.external.util.google.gcs.GCSAuthUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.CleanupUtils;
@@ -46,13 +47,15 @@
public class GCSInputStream extends AbstractExternalInputStream {
+ private final IApplicationContext ncAppCtx;
private final Storage client;
private final String container;
private static final int MAX_ATTEMPTS = 5; // We try a total of 5 times in
case of retryable errors
- public GCSInputStream(Map<String, String> configuration, List<String>
filePaths,
+ public GCSInputStream(IApplicationContext ncAppCtx, Map<String, String>
configuration, List<String> filePaths,
IExternalFilterValueEmbedder valueEmbedder) throws
HyracksDataException {
super(configuration, filePaths, valueEmbedder);
+ this.ncAppCtx = ncAppCtx;
this.client = buildClient(configuration);
this.container =
configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
}
@@ -136,7 +139,7 @@
private Storage buildClient(Map<String, String> configuration) throws
HyracksDataException {
try {
- return GCSUtils.buildClient(configuration);
+ return GCSAuthUtils.buildClient(ncAppCtx, configuration);
} catch (CompilationException ex) {
throw HyracksDataException.create(ex);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
index b6ad3cd..2de55a0 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStreamFactory.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.PriorityQueue;
+import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.external.IExternalFilterEvaluator;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.external.api.AsterixInputStream;
@@ -47,7 +48,9 @@
public AsterixInputStream createInputStream(IExternalDataRuntimeContext
context) throws HyracksDataException {
IExternalFilterValueEmbedder valueEmbedder =
context.getValueEmbedder();
int partition = context.getPartition();
- return new GCSInputStream(configuration,
partitionWorkLoadsBasedOnSize.get(partition).getFilePaths(),
+ IApplicationContext ncAppCtx = (IApplicationContext)
context.getTaskContext().getJobletContext()
+ .getServiceContext().getApplicationContext();
+ return new GCSInputStream(ncAppCtx, configuration,
partitionWorkLoadsBasedOnSize.get(partition).getFilePaths(),
valueEmbedder);
}
@@ -65,7 +68,8 @@
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME,
externalDataPrefix.getRoot());
// get the items
- List<Blob> filesOnly = GCSUtils.listItems(configuration,
includeExcludeMatcher, warningCollector,
+ IApplicationContext appCtx = (IApplicationContext)
ctx.getApplicationContext();
+ List<Blob> filesOnly = GCSUtils.listItems(appCtx, configuration,
includeExcludeMatcher, warningCollector,
externalDataPrefix, evaluator);
// Distribute work load amongst the partitions
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
index 17cad3e..67b72f3 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/parquet/GCSParquetReaderFactory.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Set;
+import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.external.IExternalFilterEvaluator;
import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
import org.apache.asterix.external.input.HDFSDataSourceFactory;
@@ -59,7 +60,8 @@
configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME,
externalDataPrefix.getRoot());
String container =
configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- List<Blob> filesOnly = GCSUtils.listItems(configuration,
includeExcludeMatcher, warningCollector,
+ IApplicationContext appCtx = (IApplicationContext)
serviceCtx.getApplicationContext();
+ List<Blob> filesOnly = GCSUtils.listItems(appCtx, configuration,
includeExcludeMatcher, warningCollector,
externalDataPrefix, evaluator);
// get path
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 82b5dad..0298f4c 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -725,7 +725,7 @@
validateAzureDataLakeProperties(configuration, srcLoc,
collector, appCtx);
break;
case ExternalDataConstants.KEY_ADAPTER_NAME_GCS:
- validateProperties(configuration, srcLoc, collector);
+ validateProperties(appCtx, configuration, srcLoc, collector);
break;
case ExternalDataConstants.KEY_ADAPTER_NAME_HDFS:
HDFSUtils.validateProperties(configuration, srcLoc, collector);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 4e0a435..b39341d 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@ -104,7 +104,7 @@
public class S3AuthUtils {
enum AuthenticationType {
ANONYMOUS,
- ARN,
+ ARN_ASSUME_ROLE,
INSTANCE_PROFILE,
ACCESS_KEYS,
BAD_AUTHENTICATION
@@ -119,7 +119,8 @@
}
public static boolean isArnAssumedRoleExpiredToken(Map<String, String>
configuration, String errorCode) {
- return ERROR_EXPIRED_TOKEN.equals(errorCode) &&
getAuthenticationType(configuration) == AuthenticationType.ARN;
+ return ERROR_EXPIRED_TOKEN.equals(errorCode)
+ && getAuthenticationType(configuration) ==
AuthenticationType.ARN_ASSUME_ROLE;
}
/**
@@ -167,7 +168,7 @@
switch (authenticationType) {
case ANONYMOUS:
return AnonymousCredentialsProvider.create();
- case ARN:
+ case ARN_ASSUME_ROLE:
return getTrustAccountCredentials(appCtx, configuration);
case INSTANCE_PROFILE:
return getInstanceProfileCredentials(configuration);
@@ -205,7 +206,7 @@
if (noAuth(configuration)) {
return AuthenticationType.ANONYMOUS;
} else if (roleArn != null) {
- return AuthenticationType.ARN;
+ return AuthenticationType.ARN_ASSUME_ROLE;
} else if (instanceProfile != null) {
return AuthenticationType.INSTANCE_PROFILE;
} else if (accessKeyId != null || secretAccessKey != null) {
@@ -231,11 +232,11 @@
}
/**
- * Returns the cached credentials if valid, otherwise, generates new
credentials by assume a role
+ * Returns the cached credentials if valid, otherwise, generates new
credentials
*
* @param appCtx application context
* @param configuration configuration
- * @return returns the cached credentials if valid, otherwise, generates
new credentials by assume a role
+ * @return returns the cached credentials if valid, otherwise, generates
new credentials
* @throws CompilationException CompilationException
*/
public static AwsCredentialsProvider
getTrustAccountCredentials(IApplicationContext appCtx,
@@ -277,20 +278,8 @@
if (externalId != null) {
builder.externalId(externalId);
}
-
- // credentials to be used to assume the role
- AwsCredentialsProvider credentialsProvider;
AssumeRoleRequest request = builder.build();
- String instanceProfile =
configuration.get(INSTANCE_PROFILE_FIELD_NAME);
- String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey =
configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
- if ("true".equalsIgnoreCase(instanceProfile)) {
- credentialsProvider = getInstanceProfileCredentials(configuration,
true);
- } else if (accessKeyId != null && secretAccessKey != null) {
- credentialsProvider = getAccessKeyCredentials(configuration, true);
- } else {
- throw new
CompilationException(ErrorCode.NO_AWS_VALID_PARAMS_FOUND_FOR_CROSS_ACCOUNT_TRUST_AUTHENTICATION);
- }
+ AwsCredentialsProvider credentialsProvider =
getCredentialsToAssumeRole(configuration);
// assume the role from the provided arn
try (StsClient stsClient =
@@ -304,13 +293,22 @@
}
}
- private static AwsCredentialsProvider
getInstanceProfileCredentials(Map<String, String> configuration)
+ private static AwsCredentialsProvider
getCredentialsToAssumeRole(Map<String, String> configuration)
throws CompilationException {
- return getInstanceProfileCredentials(configuration, false);
+ String instanceProfile =
configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey =
configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ if (instanceProfile != null) {
+ return getInstanceProfileCredentials(configuration);
+ } else if (accessKeyId != null || secretAccessKey != null) {
+ return getAccessKeyCredentials(configuration);
+ } else {
+ throw new
CompilationException(ErrorCode.NO_AWS_VALID_PARAMS_FOUND_FOR_CROSS_ACCOUNT_TRUST_AUTHENTICATION);
+ }
}
- private static AwsCredentialsProvider
getInstanceProfileCredentials(Map<String, String> configuration,
- boolean assumeRoleAuthentication) throws CompilationException {
+ private static AwsCredentialsProvider
getInstanceProfileCredentials(Map<String, String> configuration)
+ throws CompilationException {
String instanceProfile =
configuration.get(INSTANCE_PROFILE_FIELD_NAME);
// only "true" value is allowed
@@ -318,24 +316,17 @@
throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE,
INSTANCE_PROFILE_FIELD_NAME, "true");
}
- if (!assumeRoleAuthentication) {
- String notAllowed = getNonNull(configuration,
ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
- SESSION_TOKEN_FIELD_NAME);
- if (notAllowed != null) {
- throw new
CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, notAllowed,
- INSTANCE_PROFILE_FIELD_NAME);
- }
+ String notAllowed = getNonNull(configuration,
ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
+ SESSION_TOKEN_FIELD_NAME);
+ if (notAllowed != null) {
+ throw new
CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, notAllowed,
+ INSTANCE_PROFILE_FIELD_NAME);
}
return InstanceProfileCredentialsProvider.create();
}
private static AwsCredentialsProvider getAccessKeyCredentials(Map<String,
String> configuration)
throws CompilationException {
- return getAccessKeyCredentials(configuration, false);
- }
-
- private static AwsCredentialsProvider getAccessKeyCredentials(Map<String,
String> configuration,
- boolean assumeRoleAuthentication) throws CompilationException {
String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
String secretAccessKey =
configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
@@ -349,14 +340,6 @@
ACCESS_KEY_ID_FIELD_NAME);
}
- if (!assumeRoleAuthentication) {
- String notAllowed = getNonNull(configuration,
INSTANCE_PROFILE_FIELD_NAME, EXTERNAL_ID_FIELD_NAME);
- if (notAllowed != null) {
- throw new
CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, notAllowed,
- INSTANCE_PROFILE_FIELD_NAME);
- }
- }
-
// use session token if provided
if (sessionToken != null) {
return StaticCredentialsProvider
@@ -422,13 +405,13 @@
* @param configuration external details configuration
*/
private static void setHadoopCredentials(IApplicationContext appCtx,
JobConf jobConf,
- Map<String, String> configuration) {
+ Map<String, String> configuration) throws CompilationException {
AuthenticationType authenticationType =
getAuthenticationType(configuration);
switch (authenticationType) {
case ANONYMOUS:
jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ANONYMOUS);
break;
- case ARN:
+ case ARN_ASSUME_ROLE:
jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY,
HADOOP_ASSUMED_ROLE);
jobConf.set(HADOOP_ASSUME_ROLE_ARN,
configuration.get(ROLE_ARN_FIELD_NAME));
jobConf.set(HADOOP_ASSUME_ROLE_EXTERNAL_ID,
configuration.get(EXTERNAL_ID_FIELD_NAME));
@@ -457,6 +440,7 @@
}
break;
case BAD_AUTHENTICATION:
+ throw new
CompilationException(ErrorCode.NO_VALID_AUTHENTICATION_PARAMS_PROVIDED);
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSAuthUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSAuthUtils.java
new file mode 100644
index 0000000..cf3e64e
--- /dev/null
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSAuthUtils.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.google.gcs;
+
+import static
org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
+import static
org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.ENDPOINT_FIELD_NAME;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.JSON_CREDENTIALS_FIELD_NAME;
+import static
org.apache.asterix.external.util.google.gcs.GCSConstants.TARGET_SERVICE_ACCOUNT_FIELD_NAME;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+
+import com.google.auth.Credentials;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.oauth2.ImpersonatedCredentials;
+import com.google.cloud.NoCredentials;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+
+public class GCSAuthUtils {
+ enum AuthenticationType {
+ ANONYMOUS,
+ IMPERSONATE_SERVICE_ACCOUNT,
+ APPLICATION_DEFAULT_CREDENTIALS,
+ SERVICE_ACCOUNT_KEY_JSON_CREDENTIALS,
+ BAD_AUTHENTICATION
+ }
+
+ private static final List<String> READ_WRITE_SCOPE_PERMISSION =
+
Collections.singletonList("https://www.googleapis.com/auth/devstorage.read_write";);
+
+ private GCSAuthUtils() {
+ throw new AssertionError("do not instantiate");
+ }
+
+ /**
+ * Builds the client using the provided configuration
+ *
+ * @param appCtx application context
+ * @param configuration properties
+ * @return Storage client
+ * @throws CompilationException CompilationException
+ */
+ public static Storage buildClient(IApplicationContext appCtx, Map<String,
String> configuration)
+ throws CompilationException {
+ String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+
+ Credentials credentials = buildCredentials(appCtx, configuration);
+ StorageOptions.Builder builder = StorageOptions.newBuilder();
+ builder.setCredentials(credentials);
+
+ if (endpoint != null) {
+ builder.setHost(endpoint);
+ }
+
+ return builder.build().getService();
+ }
+
+ public static Credentials buildCredentials(IApplicationContext appCtx,
Map<String, String> configuration) throws CompilationException {
+ AuthenticationType authenticationType =
getAuthenticationType(configuration);
+ return switch (authenticationType) {
+ case ANONYMOUS -> NoCredentials.getInstance();
+ case IMPERSONATE_SERVICE_ACCOUNT ->
getImpersonatedServiceAccountCredentials(appCtx, configuration);
+ case APPLICATION_DEFAULT_CREDENTIALS ->
getApplicationDefaultCredentials(configuration);
+ case SERVICE_ACCOUNT_KEY_JSON_CREDENTIALS ->
getServiceAccountKeyCredentials(configuration);
+ case BAD_AUTHENTICATION -> throw new
CompilationException(ErrorCode.NO_VALID_AUTHENTICATION_PARAMS_PROVIDED);
+ };
+ }
+
+ private static AuthenticationType getAuthenticationType(Map<String,
String> configuration) {
+ String impersonateServiceAccount =
configuration.get(TARGET_SERVICE_ACCOUNT_FIELD_NAME);
+ String applicationDefaultCredentials =
configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
+ String jsonCredentials =
configuration.get(JSON_CREDENTIALS_FIELD_NAME);
+
+ if (noAuth(configuration)) {
+ return AuthenticationType.ANONYMOUS;
+ } else if (impersonateServiceAccount != null) {
+ return AuthenticationType.IMPERSONATE_SERVICE_ACCOUNT;
+ } else if (applicationDefaultCredentials != null) {
+ return AuthenticationType.APPLICATION_DEFAULT_CREDENTIALS;
+ } else if (jsonCredentials != null) {
+ return AuthenticationType.SERVICE_ACCOUNT_KEY_JSON_CREDENTIALS;
+ } else {
+ return AuthenticationType.BAD_AUTHENTICATION;
+ }
+ }
+
+ private static boolean noAuth(Map<String, String> configuration) {
+ return getNonNull(configuration,
APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME, JSON_CREDENTIALS_FIELD_NAME,
+ TARGET_SERVICE_ACCOUNT_FIELD_NAME) == null;
+ }
+
+ /**
+ * Returns the cached credentials if valid, otherwise, generates new
credentials
+ *
+ * @param appCtx application context
+ * @param configuration configuration
+ * @return returns the cached credentials if valid, otherwise, generates
new credentials
+ * @throws CompilationException CompilationException
+ */
+ public static GoogleCredentials
getImpersonatedServiceAccountCredentials(IApplicationContext appCtx,
+ Map<String, String> configuration) throws CompilationException {
+ GoogleCredentials sourceCredentials =
getCredentialsToImpersonateServiceAccount(configuration);
+ String impersonateServiceAccount =
configuration.get(TARGET_SERVICE_ACCOUNT_FIELD_NAME);
+ int duration =
appCtx.getExternalProperties().getGcpImpersonateServiceAccountDuration();
+
+ // Create impersonated credentials
+ return ImpersonatedCredentials.create(sourceCredentials,
impersonateServiceAccount, null,
+ READ_WRITE_SCOPE_PERMISSION, duration);
+ }
+
+ private static GoogleCredentials
getCredentialsToImpersonateServiceAccount(Map<String, String> configuration)
+ throws CompilationException {
+ String applicationDefaultCredentials =
configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
+ String jsonCredentials =
configuration.get(JSON_CREDENTIALS_FIELD_NAME);
+
+ if (applicationDefaultCredentials != null) {
+ return getApplicationDefaultCredentials(configuration);
+ } else if (jsonCredentials != null) {
+ return getServiceAccountKeyCredentials(configuration);
+ } else {
+ throw new CompilationException(
+
ErrorCode.NO_VALID_AUTHENTICATION_PARAMS_PROVIDED_TO_IMPERSONATE_SERVICE_ACCOUNT);
+ }
+ }
+
+ private static GoogleCredentials
getApplicationDefaultCredentials(Map<String, String> configuration)
+ throws CompilationException {
+ try {
+ String notAllowed = getNonNull(configuration,
JSON_CREDENTIALS_FIELD_NAME);
+ if (notAllowed != null) {
+ throw new
CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, notAllowed,
+ APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
+ }
+ return GoogleCredentials.getApplicationDefault();
+ } catch (Exception ex) {
+ throw CompilationException.create(EXTERNAL_SOURCE_ERROR, ex,
getMessageOrToString(ex));
+ }
+ }
+
+ private static GoogleCredentials
getServiceAccountKeyCredentials(Map<String, String> configuration)
+ throws CompilationException {
+ String jsonCredentials =
configuration.get(JSON_CREDENTIALS_FIELD_NAME);
+ try (InputStream credentialsStream = new
ByteArrayInputStream(jsonCredentials.getBytes())) {
+ return GoogleCredentials.fromStream(credentialsStream);
+ } catch (IOException ex) {
+ throw CompilationException.create(EXTERNAL_SOURCE_ERROR, ex,
getMessageOrToString(ex));
+ } catch (Exception ex) {
+ throw new CompilationException(EXTERNAL_SOURCE_ERROR,
+ "Encountered an issue while processing the JSON
credentials. Please ensure the provided credentials are valid.");
+ }
+ }
+
+ private static String getNonNull(Map<String, String> configuration,
String... fieldNames) {
+ for (String fieldName : fieldNames) {
+ if (configuration.get(fieldName) != null) {
+ return fieldName;
+ }
+ }
+ return null;
+ }
+}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
index 2613f34..f9f4c75 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
@@ -24,6 +24,7 @@
}
public static final String APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME =
"applicationDefaultCredentials";
+ public static final String TARGET_SERVICE_ACCOUNT_FIELD_NAME =
"targetServiceAccount";
public static final String JSON_CREDENTIALS_FIELD_NAME = "jsonCredentials";
public static final String ENDPOINT_FIELD_NAME = "endpoint";
public static final String STORAGE_PREFIX = "prefix";
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
index 481b7ff..913a913 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -19,13 +19,11 @@
package org.apache.asterix.external.util.google.gcs;
import static
org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
-import static
org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
-import static
org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable;
import static
org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
import static
org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
-import static
org.apache.asterix.external.util.google.gcs.GCSConstants.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME;
+import static
org.apache.asterix.external.util.google.gcs.GCSAuthUtils.buildClient;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.ENDPOINT_FIELD_NAME;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_TYPE;
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_UNAUTHENTICATED;
@@ -34,15 +32,13 @@
import static
org.apache.asterix.external.util.google.gcs.GCSConstants.JSON_CREDENTIALS_FIELD_NAME;
import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.BiPredicate;
import java.util.regex.Matcher;
+import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.external.IExternalFilterEvaluator;
@@ -63,12 +59,9 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.gax.paging.Page;
-import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.BaseServiceException;
-import com.google.cloud.NoCredentials;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
public class GCSUtils {
private GCSUtils() {
@@ -82,66 +75,16 @@
}
/**
- * Builds the client using the provided configuration
- *
- * @param configuration properties
- * @return
clientasterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
- * @throws CompilationException CompilationException
- */
- public static Storage buildClient(Map<String, String> configuration)
throws CompilationException {
- String applicationDefaultCredentials =
configuration.get(APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
- String jsonCredentials =
configuration.get(JSON_CREDENTIALS_FIELD_NAME);
- String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
-
- StorageOptions.Builder builder = StorageOptions.newBuilder();
-
- // default credentials provider
- if (applicationDefaultCredentials != null) {
- // only "true" value is allowed
- if (!applicationDefaultCredentials.equalsIgnoreCase("true")) {
- throw new
CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE,
- APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME, "true");
- }
-
- // no other authentication parameters are allowed
- if (jsonCredentials != null) {
- throw new
CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
JSON_CREDENTIALS_FIELD_NAME,
- APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME);
- }
-
- try {
-
builder.setCredentials(GoogleCredentials.getApplicationDefault());
- } catch (Exception ex) {
- throw CompilationException.create(EXTERNAL_SOURCE_ERROR, ex,
getMessageOrToString(ex));
- }
- } else if (jsonCredentials != null) {
- try (InputStream credentialsStream = new
ByteArrayInputStream(jsonCredentials.getBytes())) {
-
builder.setCredentials(GoogleCredentials.fromStream(credentialsStream));
- } catch (IOException ex) {
- throw CompilationException.create(EXTERNAL_SOURCE_ERROR, ex,
getMessageOrToString(ex));
- } catch (Exception ex) {
- throw new CompilationException(EXTERNAL_SOURCE_ERROR,
- "Encountered an issue while processing the JSON
credentials. Please ensure the provided credentials are valid.");
- }
- } else {
- builder.setCredentials(NoCredentials.getInstance());
- }
-
- if (endpoint != null) {
- builder.setHost(endpoint);
- }
-
- return builder.build().getService();
- }
-
- /**
* Validate external dataset properties
*
+ * @param appCtx application context
* @param configuration properties
+ * @param srcLoc source location
+ * @param collector warning collector
* @throws CompilationException Compilation exception
*/
- public static void validateProperties(Map<String, String> configuration,
SourceLocation srcLoc,
- IWarningCollector collector) throws CompilationException {
+ public static void validateProperties(IApplicationContext appCtx,
Map<String, String> configuration,
+ SourceLocation srcLoc, IWarningCollector collector) throws
CompilationException {
if (isDeltaTable(configuration)) {
validateDeltaTableProperties(configuration);
}
@@ -163,7 +106,7 @@
try {
Storage.BlobListOption limitOption =
Storage.BlobListOption.pageSize(1);
Storage.BlobListOption prefixOption =
Storage.BlobListOption.prefix(getPrefix(configuration));
- Storage storage = buildClient(configuration);
+ Storage storage = buildClient(appCtx, configuration);
Page<Blob> items = storage.list(container, limitOption,
prefixOption);
if (!items.iterateAll().iterator().hasNext() &&
collector.shouldWarn()) {
@@ -177,13 +120,14 @@
}
}
- public static List<Blob> listItems(Map<String, String> configuration,
IncludeExcludeMatcher includeExcludeMatcher,
- IWarningCollector warningCollector, ExternalDataPrefix
externalDataPrefix,
- IExternalFilterEvaluator evaluator) throws CompilationException,
HyracksDataException {
+ public static List<Blob> listItems(IApplicationContext appCtx, Map<String,
String> configuration,
+ IncludeExcludeMatcher includeExcludeMatcher, IWarningCollector
warningCollector,
+ ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator
evaluator)
+ throws CompilationException, HyracksDataException {
// Prepare to retrieve the objects
List<Blob> filesOnly = new ArrayList<>();
String container =
configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- Storage gcs = buildClient(configuration);
+ Storage gcs = buildClient(appCtx, configuration);
Storage.BlobListOption options =
Storage.BlobListOption.prefix(ExternalDataUtils.getPrefix(configuration));
Page<Blob> items;
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19444
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ie1d69faa45a03550c8e0fe66eb18c2ae53a8454a
Gerrit-Change-Number: 19444
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
