>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19246 )
Change subject: [ASTERIXDB-3514][EXT]: Support trust auth for parquet + delete
assumed creds on collection drop
......................................................................
[ASTERIXDB-3514][EXT]: Support trust auth for parquet + delete assumed creds on
collection drop
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Support using trusted credentials for reading S3 parquet
files since it uses different code path to build the client.
- Delete the temporarily generated credentials when the
collection is dropped.
Ext-ref: MB-63505
Change-Id: I77998a5dfcc304692e12280b7b4018f3593085b9
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalCredentialsCache.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCache.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/delta/GCSDeltaReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
12 files changed, 155 insertions(+), 47 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/46/19246/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCache.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCache.java
index 0ddca4e..189865f 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCache.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCache.java
@@ -60,6 +60,11 @@
}
@Override
+ public void deleteCredentials(String name) {
+ cache.remove(name);
+ }
+
+ @Override
public String getName(Map<String, String> configuration) {
String database =
configuration.get(ExternalDataConstants.KEY_DATASET_DATABASE);
if (database == null) {
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ed93838..4ee674e 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2448,6 +2448,8 @@
sourceLoc, EnumSet.of(DropOption.IF_EXISTS),
requestParameters.isForceDropDataset());
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
+ appCtx.getExternalCredentialsCache()
+ .deleteCredentials(String.join(".", databaseName,
dataverseName.getCanonicalForm(), datasetName));
return true;
} catch (Exception e) {
LOGGER.error("failed to drop dataset; executing compensating
operations", e);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 8aa4532..ebc1cee 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.control.common.config.OptionTypes.DOUBLE;
import static org.apache.hyracks.control.common.config.OptionTypes.LEVEL;
import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
@@ -52,7 +53,16 @@
"The maximum accepted web request size in bytes"),
REQUESTS_ARCHIVE_SIZE(NONNEGATIVE_INTEGER, 1000, "The maximum number
of archived requests to maintain"),
LIBRARY_DEPLOY_TIMEOUT(POSITIVE_INTEGER, 1800, "Timeout to upload a
UDF in seconds"),
- AZURE_REQUEST_TIMEOUT(POSITIVE_INTEGER, 120, "Timeout for Azure client
requests in seconds");
+ AZURE_REQUEST_TIMEOUT(POSITIVE_INTEGER, 120, "Timeout for Azure client
requests in seconds"),
+ AWS_ASSUME_ROLE_DURATION(
+ POSITIVE_INTEGER,
+ 900,
+ "AWS assuming role duration in seconds. "
+ + "Range from 900 seconds (15 mins) to 43200 seconds
(12 hours)"),
+ AWS_REFRESH_ASSUME_ROLE_DURATION(
+ DOUBLE,
+ .5,
+ "Percentage of left duration before assume role credentials "
+ "needs to be refreshed");
private final IOptionType type;
private final Object defaultValue;
@@ -80,6 +90,8 @@
case MAX_WEB_REQUEST_SIZE:
case LIBRARY_DEPLOY_TIMEOUT:
case AZURE_REQUEST_TIMEOUT:
+ case AWS_ASSUME_ROLE_DURATION:
+ case AWS_REFRESH_ASSUME_ROLE_DURATION:
return Section.COMMON;
case CC_JAVA_OPTS:
case NC_JAVA_OPTS:
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalCredentialsCache.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalCredentialsCache.java
index 245b350..c603893 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalCredentialsCache.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/external/IExternalCredentialsCache.java
@@ -39,6 +39,13 @@
void updateCache(Map<String, String> configuration, Map<String, String>
credentials);
/**
+ * Deletes the cache for the provided entity name
+ *
+ * @param name name of the entity for which the credentials are to be
deleted
+ */
+ void deleteCredentials(String name);
+
+ /**
* Returns the name of the entity which the cached credentials belong to
*
* @param configuration configuration containing external collection
details
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
index ba0d0f4..7d678a9 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
@@ -24,6 +24,8 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.aws.s3.S3Utils;
import org.apache.hadoop.mapred.JobConf;
@@ -34,8 +36,9 @@
Collections.singletonList(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3);
@Override
- protected void configureJobConf(JobConf conf, Map<String, String>
configuration) {
- configureAwsS3HdfsJobConf(conf, configuration);
+ protected void configureJobConf(IApplicationContext appCtx, JobConf conf,
Map<String, String> configuration)
+ throws CompilationException {
+ configureAwsS3HdfsJobConf(appCtx, conf, configuration);
}
@Override
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaReaderFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaReaderFactory.java
index dc4c310..6b98225 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaReaderFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaReaderFactory.java
@@ -30,6 +30,7 @@
import java.util.PriorityQueue;
import java.util.Set;
+import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -82,8 +83,8 @@
return locationConstraints;
}
- protected abstract void configureJobConf(JobConf conf, Map<String, String>
configuration)
- throws AlgebricksException;
+ protected abstract void configureJobConf(IApplicationContext appCtx,
JobConf conf,
+ Map<String, String> configuration) throws AlgebricksException;
protected abstract String getTablePath(Map<String, String> configuration)
throws AlgebricksException;
@@ -93,7 +94,7 @@
throws AlgebricksException, HyracksDataException {
JobConf conf = new JobConf();
ICcApplicationContext appCtx = (ICcApplicationContext)
serviceCtx.getApplicationContext();
- configureJobConf(conf, configuration);
+ configureJobConf(appCtx, conf, configuration);
confFactory = new ConfFactory(conf);
String tableMetadataPath = getTablePath(configuration);
Engine engine = DefaultEngine.create(conf);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
index 7ddbab91..2d92e10 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
@@ -87,7 +87,9 @@
try {
JobConf conf = prepareHDFSConf(serviceCtx, configuration,
filterEvaluatorFactory);
int numberOfPartitions =
getPartitionConstraint().getLocations().length;
- configureAwsS3HdfsJobConf(conf, configuration, numberOfPartitions);
+
+ IApplicationContext appCtx = (IApplicationContext)
serviceCtx.getApplicationContext();
+ configureAwsS3HdfsJobConf(appCtx, conf, configuration,
numberOfPartitions);
configureHdfsConf(conf, configuration);
} catch (SdkException | SdkBaseException ex) {
throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR,
ex, getMessageOrToString(ex));
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/delta/GCSDeltaReaderFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/delta/GCSDeltaReaderFactory.java
index ee88569..db7673c 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/delta/GCSDeltaReaderFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/delta/GCSDeltaReaderFactory.java
@@ -22,6 +22,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.api.IApplicationContext;
import
org.apache.asterix.external.input.record.reader.aws.delta.DeltaReaderFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.google.gcs.GCSUtils;
@@ -34,7 +35,8 @@
Collections.singletonList(ExternalDataConstants.KEY_ADAPTER_NAME_GCS);
@Override
- protected void configureJobConf(JobConf conf, Map<String, String>
configuration) throws AlgebricksException {
+ protected void configureJobConf(IApplicationContext appCtx, JobConf conf,
Map<String, String> configuration)
+ throws AlgebricksException {
GCSUtils.configureHdfsJobConf(conf, configuration);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 6767f93..82b5dad 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -519,12 +519,13 @@
}
}
- public static void validateDeltaTableExists(Map<String, String>
configuration) throws AlgebricksException {
+ public static void validateDeltaTableExists(IApplicationContext appCtx,
Map<String, String> configuration)
+ throws AlgebricksException {
String tableMetadataPath = null;
JobConf conf = new JobConf();
if (configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE)
.equals(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3)) {
- configureAwsS3HdfsJobConf(conf, configuration);
+ configureAwsS3HdfsJobConf(appCtx, conf, configuration);
tableMetadataPath = S3Utils.getPath(configuration);
} else if
(configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE)
.equals(ExternalDataConstants.KEY_ADAPTER_NAME_GCS)) {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 45988e8..45a5d43 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@ -28,6 +28,7 @@
import static
org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
import static
org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
+import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_EXPIRED_TOKEN;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
@@ -94,6 +95,14 @@
import software.amazon.awssdk.services.sts.model.Credentials;
public class S3AuthUtils {
+ enum AuthenticationType {
+ ANONYMOUS,
+ ARN,
+ INSTANCE_PROFILE,
+ ACCESS_KEYS,
+ BAD_AUTHENTICATION
+ }
+
private S3AuthUtils() {
throw new AssertionError("do not instantiate");
}
@@ -102,6 +111,10 @@
return errorCode.equals(ERROR_INTERNAL_ERROR) ||
errorCode.equals(ERROR_SLOW_DOWN);
}
+ public static boolean isArnAssumedRoleExpiredToken(Map<String, String>
configuration, String errorCode) {
+ return ERROR_EXPIRED_TOKEN.equals(errorCode) &&
getAuthenticationType(configuration) == AuthenticationType.ARN;
+ }
+
/**
* Builds the S3 client using the provided configuration
*
@@ -142,28 +155,26 @@
public static AwsCredentialsProvider
buildCredentialsProvider(IApplicationContext appCtx,
Map<String, String> configuration) throws CompilationException {
- String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
- String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
- String instanceProfile =
configuration.get(INSTANCE_PROFILE_FIELD_NAME);
- String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey =
configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
-
- if (noAuth(configuration)) {
- return AnonymousCredentialsProvider.create();
- } else if (arnRole != null) {
- return getTrustAccountCredentials(appCtx, configuration);
- } else if (instanceProfile != null) {
- return getInstanceProfileCredentials(configuration);
- } else if (accessKeyId != null || secretAccessKey != null) {
- return getAccessKeyCredentials(configuration);
- } else {
- if (externalId != null) {
- throw new
CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ROLE_ARN_FIELD_NAME,
- EXTERNAL_ID_FIELD_NAME);
- } else {
- throw new
CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT,
ACCESS_KEY_ID_FIELD_NAME,
- SESSION_TOKEN_FIELD_NAME);
- }
+ AuthenticationType authenticationType =
getAuthenticationType(configuration);
+ switch (authenticationType) {
+ case ANONYMOUS:
+ return AnonymousCredentialsProvider.create();
+ case ARN:
+ return getTrustAccountCredentials(appCtx, configuration);
+ case INSTANCE_PROFILE:
+ return getInstanceProfileCredentials(configuration);
+ case ACCESS_KEYS:
+ return getAccessKeyCredentials(configuration);
+ default:
+ // missing required creds, report correct error message
+ String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
+ if (externalId != null) {
+ throw new
CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ROLE_ARN_FIELD_NAME,
+ EXTERNAL_ID_FIELD_NAME);
+ } else {
+ throw new
CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT,
ACCESS_KEY_ID_FIELD_NAME,
+ SESSION_TOKEN_FIELD_NAME);
+ }
}
}
@@ -177,6 +188,25 @@
return selectedRegion.get();
}
+ private static AuthenticationType getAuthenticationType(Map<String,
String> configuration) {
+ String roleArn = configuration.get(ROLE_ARN_FIELD_NAME);
+ String instanceProfile =
configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey =
configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+
+ if (noAuth(configuration)) {
+ return AuthenticationType.ANONYMOUS;
+ } else if (roleArn != null) {
+ return AuthenticationType.ARN;
+ } else if (instanceProfile != null) {
+ return AuthenticationType.INSTANCE_PROFILE;
+ } else if (accessKeyId != null || secretAccessKey != null) {
+ return AuthenticationType.ACCESS_KEYS;
+ } else {
+ return AuthenticationType.BAD_AUTHENTICATION;
+ }
+ }
+
private static boolean noAuth(Map<String, String> configuration) {
return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME,
ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
SESSION_TOKEN_FIELD_NAME) == null;
@@ -327,23 +357,38 @@
return null;
}
+ public static void configureAwsS3HdfsJobConf(IApplicationContext appCtx,
JobConf conf,
+ Map<String, String> configuration) throws CompilationException {
+ configureAwsS3HdfsJobConf(appCtx, conf, configuration, 0);
+ }
+
/**
* Builds the S3 client using the provided configuration
*
+ * @param appCtx application context
* @param configuration properties
* @param numberOfPartitions number of partitions in the cluster
*/
- public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String,
String> configuration) {
- configureAwsS3HdfsJobConf(conf, configuration, 0);
- }
-
- public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String,
String> configuration,
- int numberOfPartitions) {
- String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey =
configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
- String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
+ public static void configureAwsS3HdfsJobConf(IApplicationContext appCtx,
JobConf conf,
+ Map<String, String> configuration, int numberOfPartitions) throws
CompilationException {
+ String accessKeyId;
+ String secretAccessKey;
+ String sessionToken;
String serviceEndpoint =
configuration.get(SERVICE_END_POINT_FIELD_NAME);
+ if (getAuthenticationType(configuration) == AuthenticationType.ARN) {
+ AwsCredentialsProvider credentialsProvider =
getTrustAccountCredentials(appCtx, configuration);
+ AwsSessionCredentials sessionCredentials = (AwsSessionCredentials)
credentialsProvider.resolveCredentials();
+ accessKeyId = sessionCredentials.accessKeyId();
+ secretAccessKey = sessionCredentials.secretAccessKey();
+ sessionToken = sessionCredentials.sessionToken();
+ } else {
+ accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
+ serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+ }
+
//Disable caching S3 FileSystem
HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_S3_PROTOCOL);
@@ -477,7 +522,7 @@
}
if (isDeltaTable(configuration)) {
try {
- validateDeltaTableExists(configuration);
+ validateDeltaTableExists(appCtx, configuration);
} catch (AlgebricksException e) {
throw new
CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, e);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index 126c868..efebfb2 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -23,6 +23,7 @@
throw new AssertionError("do not instantiate");
}
+ // Authentication specific parameters
public static final String REGION_FIELD_NAME = "region";
public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
@@ -32,10 +33,17 @@
public static final String EXTERNAL_ID_FIELD_NAME = "externalId";
public static final String SERVICE_END_POINT_FIELD_NAME =
"serviceEndpoint";
+ // AWS assuming role authentication properties
+ public static final String ASSUME_ROLE_DURATION_FIELD_NAME =
"assumeRoleDuration";
+ public static final String ASSUME_ROLE_REFRESH_PERCENTAGE_FIELD_NAME =
"assumeRoleRefreshPercentage";
+ public static final int ASSUME_ROLE_DURATION_DEFAULT = 900;
+ public static final double ASSUME_ROLE_REFRESH_PERCENTAGE = .5d;
+
// AWS S3 specific error codes
public static final String ERROR_INTERNAL_ERROR = "InternalError";
public static final String ERROR_SLOW_DOWN = "SlowDown";
public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
+ public static final String ERROR_EXPIRED_TOKEN = "ExpiredToken";
/*
* Hadoop-AWS
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
index bfd35fc..481b7ff 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -222,17 +222,17 @@
}
}
+ public static void configureHdfsJobConf(JobConf conf, Map<String, String>
configuration)
+ throws AlgebricksException {
+ configureHdfsJobConf(conf, configuration, 0);
+ }
+
/**
* Builds the client using the provided configuration
*
* @param configuration properties
* @param numberOfPartitions number of partitions in the cluster
*/
- public static void configureHdfsJobConf(JobConf conf, Map<String, String>
configuration)
- throws AlgebricksException {
- configureHdfsJobConf(conf, configuration, 0);
- }
-
public static void configureHdfsJobConf(JobConf conf, Map<String, String>
configuration, int numberOfPartitions)
throws AlgebricksException {
String jsonCredentials =
configuration.get(JSON_CREDENTIALS_FIELD_NAME);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19246
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I77998a5dfcc304692e12280b7b4018f3593085b9
Gerrit-Change-Number: 19246
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
