>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19320 )
Change subject: [ASTERIXDB-3549][EXT]: Add support to enable cross-region for
AWS S3 external collections
......................................................................
[ASTERIXDB-3549][EXT]: Add support to enable cross-region for AWS S3 external
collections
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
cross-region support allows the AWS SDK to automatically
detect the correct region and cache it if an incorrect
region is specified.
Ext-ref: MB-64169
Change-Id: I3d552457cfa77aa2abe4a2ecb7d77d2d36cb70d3
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
2 files changed, 38 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/20/19320/1
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 45988e8..794e706 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@ -28,6 +28,7 @@
import static
org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
import static
org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
+import static
org.apache.asterix.external.util.aws.s3.S3Constants.CROSS_REGION_FIELD_NAME;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
import static
org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
@@ -115,11 +116,12 @@
String serviceEndpoint =
configuration.get(SERVICE_END_POINT_FIELD_NAME);
Region region = validateAndGetRegion(regionId);
+ boolean crossRegion =
validateAndGetCrossRegion(configuration.get(CROSS_REGION_FIELD_NAME));
AwsCredentialsProvider credentialsProvider =
buildCredentialsProvider(appCtx, configuration);
S3ClientBuilder builder = S3Client.builder();
builder.region(region);
- builder.crossRegionAccessEnabled(true);
+ builder.crossRegionAccessEnabled(crossRegion);
builder.credentialsProvider(credentialsProvider);
// Validate the service endpoint if present
@@ -177,6 +179,17 @@
return selectedRegion.get();
}
+ public static boolean validateAndGetCrossRegion(String crossRegion) throws
CompilationException {
+ if (crossRegion == null) {
+ return false;
+ }
+
+ if (!"true".equalsIgnoreCase(crossRegion) &&
!"false".equalsIgnoreCase(crossRegion)) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE,
CROSS_REGION_FIELD_NAME, "true, false");
+ }
+ return Boolean.parseBoolean(crossRegion);
+ }
+
private static boolean noAuth(Map<String, String> configuration) {
return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME,
ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
SESSION_TOKEN_FIELD_NAME) == null;
@@ -327,16 +340,16 @@
return null;
}
+ public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String,
String> configuration) {
+ configureAwsS3HdfsJobConf(conf, configuration, 0);
+ }
+
/**
* Builds the S3 client using the provided configuration
*
* @param configuration properties
* @param numberOfPartitions number of partitions in the cluster
*/
- public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String,
String> configuration) {
- configureAwsS3HdfsJobConf(conf, configuration, 0);
- }
-
public static void configureAwsS3HdfsJobConf(JobConf conf, Map<String,
String> configuration,
int numberOfPartitions) {
String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index 126c868..13c67d3 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -24,6 +24,7 @@
}
public static final String REGION_FIELD_NAME = "region";
+ public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
public static final String SECRET_ACCESS_KEY_FIELD_NAME =
"secretAccessKey";
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19320
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I3d552457cfa77aa2abe4a2ecb7d77d2d36cb70d3
Gerrit-Change-Number: 19320
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
