This is an automated email from the ASF dual-hosted git repository. vbalaji pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new abd3afc44a2 [HUDI-6695] Use the AWS provider chain in Glue sync and add a new provider for STS assume role (#9260) abd3afc44a2 is described below commit abd3afc44a2003ff28ed2d9da208dc68ddfce149 Author: Hussein Awala <huss...@awala.fr> AuthorDate: Wed Nov 15 06:55:47 2023 +0200 [HUDI-6695] Use the AWS provider chain in Glue sync and add a new provider for STS assume role (#9260) --- hudi-aws/pom.xml | 6 ++ .../HoodieAWSCredentialsProviderFactory.java | 3 + ...dieConfigAWSAssumedRoleCredentialsProvider.java | 65 ++++++++++++++++++++++ .../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 5 +- .../org/apache/hudi/config/HoodieAWSConfig.java | 18 +++++- .../TestHoodieAWSCredentialsProviderFactory.java | 16 ++++++ 6 files changed, 111 insertions(+), 2 deletions(-) diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml index 30f2d892d2f..682b73ecacd 100644 --- a/hudi-aws/pom.xml +++ b/hudi-aws/pom.xml @@ -186,6 +186,12 @@ <version>${aws.sdk.httpcore.version}</version> </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>sts</artifactId> + <version>${aws.sdk.version}</version> + </dependency> + <!-- Test --> <dependency> <groupId>org.apache.hudi</groupId> diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java index 4342a529d29..97df83929e9 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java @@ -36,6 +36,9 @@ public class HoodieAWSCredentialsProviderFactory { private static AwsCredentialsProvider getAwsCredentialsProviderChain(Properties props) { List<AwsCredentialsProvider> providers = new ArrayList<>(); + if (HoodieConfigAWSAssumedRoleCredentialsProvider.validConf(props)) { + providers.add(new HoodieConfigAWSAssumedRoleCredentialsProvider(props)); + } HoodieConfigAWSCredentialsProvider hoodieConfigAWSCredentialsProvider = new HoodieConfigAWSCredentialsProvider(props); if (hoodieConfigAWSCredentialsProvider.resolveCredentials() != null) { providers.add(hoodieConfigAWSCredentialsProvider); diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java new file mode 100644 index 00000000000..89c31b8a08b --- /dev/null +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.aws.credentials; + +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.config.HoodieAWSConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + +import java.util.Properties; + +/** + * Credentials provider which assumes AWS role from Hoodie config and fetches its credentials. + */ +public class HoodieConfigAWSAssumedRoleCredentialsProvider implements AwsCredentialsProvider { + + private static final Logger LOG = LoggerFactory.getLogger(HoodieConfigAWSAssumedRoleCredentialsProvider.class); + + private final StsAssumeRoleCredentialsProvider credentialsProvider; + + public HoodieConfigAWSAssumedRoleCredentialsProvider(Properties props) { + String roleArn = props.getProperty(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN.key()); + AssumeRoleRequest req = AssumeRoleRequest.builder() + .roleArn(roleArn) + .roleSessionName("hoodie") + .build(); + StsClient stsClient = StsClient.builder().build(); + + this.credentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .stsClient(stsClient) + .refreshRequest(req) + .build(); + } + + public static boolean validConf(Properties props) { + String roleArn = props.getProperty(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN.key()); + return !StringUtils.isNullOrEmpty(roleArn); + } + + @Override + public AwsCredentials resolveCredentials() { + return credentialsProvider.resolveCredentials(); + } +} diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index 0e7609aba5c..4a846e4970f 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -81,6 +81,7 @@ import static org.apache.hudi.hive.util.HiveSchemaUtil.parquetSchemaToMapSchema; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; import static org.apache.hudi.sync.common.util.TableUtils.tableId; +import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory; /** * This class implements all the AWS APIs to enable syncing of a Hudi Table with the @@ -107,7 +108,9 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient { public AWSGlueCatalogSyncClient(HiveSyncConfig config) { super(config); - this.awsGlue = GlueAsyncClient.builder().build(); + this.awsGlue = GlueAsyncClient.builder() + .credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(config.getProps())) + .build(); this.databaseName = config.getStringOrDefault(META_SYNC_DATABASE_NAME); this.skipTableArchive = config.getBooleanOrDefault(GlueCatalogSyncClientConfig.GLUE_SKIP_TABLE_ARCHIVE); this.enableMetadataTable = Boolean.toString(config.getBoolean(GLUE_METADATA_FILE_LISTING)).toUpperCase(); diff --git a/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java b/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java index 45d6878fa3d..a205dc94b30 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java +++ b/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java @@ -46,7 +46,7 @@ import static org.apache.hudi.config.GlueCatalogSyncClientConfig.GLUE_SKIP_TABLE @ConfigClassProperty(name = "Amazon Web Services Configs", groupName = ConfigGroups.Names.AWS, description = "Amazon Web Services configurations to access resources like Amazon DynamoDB (for locks)," - + " Amazon CloudWatch (metrics).") + + " Amazon CloudWatch (metrics) and Amazon Glue (metadata).") public class HoodieAWSConfig extends HoodieConfig { public static final ConfigProperty<String> AWS_ACCESS_KEY = ConfigProperty .key("hoodie.aws.access.key") @@ -69,6 +69,13 @@ public class HoodieAWSConfig extends HoodieConfig { .sinceVersion("0.10.0") .withDocumentation("AWS session token"); + public static final ConfigProperty<String> AWS_ASSUME_ROLE_ARN = ConfigProperty + .key("hoodie.aws.role.arn") + .noDefaultValue() + .markAdvanced() + .sinceVersion("0.13.2") + .withDocumentation("AWS Role ARN to assume"); + private HoodieAWSConfig() { super(); } @@ -89,6 +96,10 @@ public class HoodieAWSConfig extends HoodieConfig { return getString(AWS_SESSION_TOKEN); } + public String getAWSAssumeRoleARN() { + return getString(AWS_ASSUME_ROLE_ARN); + } + public static class Builder { private final HoodieAWSConfig awsConfig = new HoodieAWSConfig(); @@ -120,6 +131,11 @@ public class HoodieAWSConfig extends HoodieConfig { return this; } + public HoodieAWSConfig.Builder withAssumeRoleARN(String assumeRoleARN) { + awsConfig.setValue(AWS_ASSUME_ROLE_ARN, assumeRoleARN); + return this; + } + public Builder withDynamoDBTable(String dynamoDbTableName) { awsConfig.setValue(DYNAMODB_LOCK_TABLE_NAME, dynamoDbTableName); return this; diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java b/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java index 7a5e776db8d..d65f32109c1 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java @@ -39,4 +39,20 @@ public class TestHoodieAWSCredentialsProviderFactory { assertEquals("random-secret-key", credentials.secretAccessKey()); assertEquals("random-session-token", credentials.sessionToken()); } + + @Test + public void testGetAWSCredentialsWithInvalidAssumeRole() { + // This test is to ensure that the AWS credentials provider factory fallbacks to default credentials + // when the assume role ARN is invalid. + HoodieConfig cfg = new HoodieConfig(); + cfg.setValue(HoodieAWSConfig.AWS_ACCESS_KEY, "random-access-key"); + cfg.setValue(HoodieAWSConfig.AWS_SECRET_KEY, "random-secret-key"); + cfg.setValue(HoodieAWSConfig.AWS_SESSION_TOKEN, "random-session-token"); + cfg.setValue(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN, "invalid-role-arn"); + AwsSessionCredentials credentials = (AwsSessionCredentials) org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(cfg.getProps()).resolveCredentials(); + assertEquals("random-access-key", credentials.accessKeyId()); + assertEquals("random-secret-key", credentials.secretAccessKey()); + assertEquals("random-session-token", credentials.sessionToken()); + } + }