This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new abd3afc44a2 [HUDI-6695] Use the AWS provider chain in Glue sync and
add a new provider for STS assume role (#9260)
abd3afc44a2 is described below
commit abd3afc44a2003ff28ed2d9da208dc68ddfce149
Author: Hussein Awala <[email protected]>
AuthorDate: Wed Nov 15 06:55:47 2023 +0200
[HUDI-6695] Use the AWS provider chain in Glue sync and add a new provider
for STS assume role (#9260)
---
hudi-aws/pom.xml | 6 ++
.../HoodieAWSCredentialsProviderFactory.java | 3 +
...dieConfigAWSAssumedRoleCredentialsProvider.java | 65 ++++++++++++++++++++++
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 5 +-
.../org/apache/hudi/config/HoodieAWSConfig.java | 18 +++++-
.../TestHoodieAWSCredentialsProviderFactory.java | 16 ++++++
6 files changed, 111 insertions(+), 2 deletions(-)
diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml
index 30f2d892d2f..682b73ecacd 100644
--- a/hudi-aws/pom.xml
+++ b/hudi-aws/pom.xml
@@ -186,6 +186,12 @@
<version>${aws.sdk.httpcore.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sts</artifactId>
+ <version>${aws.sdk.version}</version>
+ </dependency>
+
<!-- Test -->
<dependency>
<groupId>org.apache.hudi</groupId>
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java
index 4342a529d29..97df83929e9 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java
@@ -36,6 +36,9 @@ public class HoodieAWSCredentialsProviderFactory {
private static AwsCredentialsProvider
getAwsCredentialsProviderChain(Properties props) {
List<AwsCredentialsProvider> providers = new ArrayList<>();
+ if (HoodieConfigAWSAssumedRoleCredentialsProvider.validConf(props)) {
+ providers.add(new HoodieConfigAWSAssumedRoleCredentialsProvider(props));
+ }
HoodieConfigAWSCredentialsProvider hoodieConfigAWSCredentialsProvider =
new HoodieConfigAWSCredentialsProvider(props);
if (hoodieConfigAWSCredentialsProvider.resolveCredentials() != null) {
providers.add(hoodieConfigAWSCredentialsProvider);
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java
new file mode 100644
index 00000000000..89c31b8a08b
--- /dev/null
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.aws.credentials;
+
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.config.HoodieAWSConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.services.sts.StsClient;
+import
software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+
+import java.util.Properties;
+
+/**
+ * Credentials provider which assumes AWS role from Hoodie config and fetches
its credentials.
+ */
+public class HoodieConfigAWSAssumedRoleCredentialsProvider implements
AwsCredentialsProvider {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HoodieConfigAWSAssumedRoleCredentialsProvider.class);
+
+ private final StsAssumeRoleCredentialsProvider credentialsProvider;
+
+ public HoodieConfigAWSAssumedRoleCredentialsProvider(Properties props) {
+ String roleArn =
props.getProperty(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN.key());
+ AssumeRoleRequest req = AssumeRoleRequest.builder()
+ .roleArn(roleArn)
+ .roleSessionName("hoodie")
+ .build();
+ StsClient stsClient = StsClient.builder().build();
+
+ this.credentialsProvider = StsAssumeRoleCredentialsProvider.builder()
+ .stsClient(stsClient)
+ .refreshRequest(req)
+ .build();
+ }
+
+ public static boolean validConf(Properties props) {
+ String roleArn =
props.getProperty(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN.key());
+ return !StringUtils.isNullOrEmpty(roleArn);
+ }
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return credentialsProvider.resolveCredentials();
+ }
+}
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index 0e7609aba5c..4a846e4970f 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -81,6 +81,7 @@ import static
org.apache.hudi.hive.util.HiveSchemaUtil.parquetSchemaToMapSchema;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
import static org.apache.hudi.sync.common.util.TableUtils.tableId;
+import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory;
/**
* This class implements all the AWS APIs to enable syncing of a Hudi Table
with the
@@ -107,7 +108,9 @@ public class AWSGlueCatalogSyncClient extends
HoodieSyncClient {
public AWSGlueCatalogSyncClient(HiveSyncConfig config) {
super(config);
- this.awsGlue = GlueAsyncClient.builder().build();
+ this.awsGlue = GlueAsyncClient.builder()
+
.credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(config.getProps()))
+ .build();
this.databaseName = config.getStringOrDefault(META_SYNC_DATABASE_NAME);
this.skipTableArchive =
config.getBooleanOrDefault(GlueCatalogSyncClientConfig.GLUE_SKIP_TABLE_ARCHIVE);
this.enableMetadataTable =
Boolean.toString(config.getBoolean(GLUE_METADATA_FILE_LISTING)).toUpperCase();
diff --git a/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java
b/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java
index 45d6878fa3d..a205dc94b30 100644
--- a/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java
+++ b/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java
@@ -46,7 +46,7 @@ import static
org.apache.hudi.config.GlueCatalogSyncClientConfig.GLUE_SKIP_TABLE
@ConfigClassProperty(name = "Amazon Web Services Configs",
groupName = ConfigGroups.Names.AWS,
description = "Amazon Web Services configurations to access resources
like Amazon DynamoDB (for locks),"
- + " Amazon CloudWatch (metrics).")
+ + " Amazon CloudWatch (metrics) and Amazon Glue (metadata).")
public class HoodieAWSConfig extends HoodieConfig {
public static final ConfigProperty<String> AWS_ACCESS_KEY = ConfigProperty
.key("hoodie.aws.access.key")
@@ -69,6 +69,13 @@ public class HoodieAWSConfig extends HoodieConfig {
.sinceVersion("0.10.0")
.withDocumentation("AWS session token");
+ public static final ConfigProperty<String> AWS_ASSUME_ROLE_ARN =
ConfigProperty
+ .key("hoodie.aws.role.arn")
+ .noDefaultValue()
+ .markAdvanced()
+ .sinceVersion("0.13.2")
+ .withDocumentation("AWS Role ARN to assume");
+
private HoodieAWSConfig() {
super();
}
@@ -89,6 +96,10 @@ public class HoodieAWSConfig extends HoodieConfig {
return getString(AWS_SESSION_TOKEN);
}
+ public String getAWSAssumeRoleARN() {
+ return getString(AWS_ASSUME_ROLE_ARN);
+ }
+
public static class Builder {
private final HoodieAWSConfig awsConfig = new HoodieAWSConfig();
@@ -120,6 +131,11 @@ public class HoodieAWSConfig extends HoodieConfig {
return this;
}
+ public HoodieAWSConfig.Builder withAssumeRoleARN(String assumeRoleARN) {
+ awsConfig.setValue(AWS_ASSUME_ROLE_ARN, assumeRoleARN);
+ return this;
+ }
+
public Builder withDynamoDBTable(String dynamoDbTableName) {
awsConfig.setValue(DYNAMODB_LOCK_TABLE_NAME, dynamoDbTableName);
return this;
diff --git
a/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java
b/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java
index 7a5e776db8d..d65f32109c1 100644
---
a/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java
+++
b/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java
@@ -39,4 +39,20 @@ public class TestHoodieAWSCredentialsProviderFactory {
assertEquals("random-secret-key", credentials.secretAccessKey());
assertEquals("random-session-token", credentials.sessionToken());
}
+
+ @Test
+ public void testGetAWSCredentialsWithInvalidAssumeRole() {
+ // This test is to ensure that the AWS credentials provider factory
fallbacks to default credentials
+ // when the assume role ARN is invalid.
+ HoodieConfig cfg = new HoodieConfig();
+ cfg.setValue(HoodieAWSConfig.AWS_ACCESS_KEY, "random-access-key");
+ cfg.setValue(HoodieAWSConfig.AWS_SECRET_KEY, "random-secret-key");
+ cfg.setValue(HoodieAWSConfig.AWS_SESSION_TOKEN, "random-session-token");
+ cfg.setValue(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN, "invalid-role-arn");
+ AwsSessionCredentials credentials = (AwsSessionCredentials)
org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(cfg.getProps()).resolveCredentials();
+ assertEquals("random-access-key", credentials.accessKeyId());
+ assertEquals("random-secret-key", credentials.secretAccessKey());
+ assertEquals("random-session-token", credentials.sessionToken());
+ }
+
}
