This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new ba1b0e39e32 branch-4.0: [feat](catalog)Support for Loading Catalog
Credentials via AwsCredentialsProviderChain #58740 (#59055)
ba1b0e39e32 is described below
commit ba1b0e39e322dc551bbe7a9886ebb4cd1a2e9022
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Dec 22 14:55:50 2025 +0800
branch-4.0: [feat](catalog)Support for Loading Catalog Credentials via
AwsCredentialsProviderChain #58740 (#59055)
Cherry-picked from #58740
Co-authored-by: Calvin Kirs <[email protected]>
---
.../ConfigurationAWSCredentialsProvider.java | 13 +-
.../amazonaws/glue/catalog/util/AWSGlueConfig.java | 1 +
.../common/AwsCredentialsProviderFactory.java | 158 +++++++++++++++++++++
.../common/AwsCredentialsProviderMode.java | 74 ++++++++++
.../metastore/AWSGlueMetaStoreBaseProperties.java | 18 ++-
.../metastore/HiveGlueMetaStoreProperties.java | 11 ++
.../metastore/IcebergGlueMetaStoreProperties.java | 1 +
.../datasource/property/storage/S3Properties.java | 74 +++++-----
.../property/storage/S3PropertiesTest.java | 28 +++-
fe/pom.xml | 2 +-
.../test_catalog_instance_profile.groovy | 125 ++++++++++++++++
11 files changed, 462 insertions(+), 43 deletions(-)
diff --git
a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java
b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java
index 49bea54c38e..23ba972fdfe 100644
---
a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java
+++
b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java
@@ -22,10 +22,12 @@ import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.glue.catalog.util.AWSGlueConfig;
import com.amazonaws.util.StringUtils;
+import org.apache.doris.common.Config;
+import
org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
import org.apache.hadoop.conf.Configuration;
public class ConfigurationAWSCredentialsProvider implements
AWSCredentialsProvider {
@@ -47,9 +49,9 @@ public class ConfigurationAWSCredentialsProvider implements
AWSCredentialsProvid
return (StringUtils.isNullOrEmpty(sessionToken) ? new
BasicAWSCredentials(accessKey,
secretKey) : new BasicSessionCredentials(accessKey,
secretKey, sessionToken));
}
-
- AWSCredentialsProvider longLivedProvider = new
DefaultAWSCredentialsProviderChain();
-
+ String credentialsProviderModeString =
StringUtils.lowerCase(conf.get(AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE));
+ AwsCredentialsProviderMode
credentialsProviderMode=AwsCredentialsProviderMode.fromString(credentialsProviderModeString);
+ AWSCredentialsProvider longLivedProvider =
AwsCredentialsProviderFactory.createV1(credentialsProviderMode);
if (!StringUtils.isNullOrEmpty(roleArn)) {
STSAssumeRoleSessionCredentialsProvider.Builder builder =
new
STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, "local-session")
@@ -61,6 +63,9 @@ public class ConfigurationAWSCredentialsProvider implements
AWSCredentialsProvid
STSAssumeRoleSessionCredentialsProvider provider = builder.build();
return provider.getCredentials();
}
+ if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
+ return longLivedProvider.getCredentials();
+ }
throw new SdkClientException("Unable to load AWS credentials from any
provider in the chain");
}
diff --git
a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java
b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java
index 4296eee0e94..7b0bd3ef979 100644
---
a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java
+++
b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java
@@ -63,4 +63,5 @@ public final class AWSGlueConfig {
public static final String AWS_GLUE_SESSION_TOKEN =
"aws.glue.session-token";
public static final String AWS_GLUE_ROLE_ARN = "aws.glue.role-arn";
public static final String AWS_GLUE_EXTERNAL_ID = "aws.glue.external-id";
+ public static final String AWS_CREDENTIALS_PROVIDER_MODE =
"aws.credentials.provider.mode";
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
new file mode 100644
index 00000000000..f56aed0533e
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Copied from
+//
https://github.com/awslabs/aws-glue-data-catalog-client-for-apache-hive-metastore/blob/branch-3.4.0/
+//
+
+package org.apache.doris.datasource.property.common;
+
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
+import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
+import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class AwsCredentialsProviderFactory {
+
+ private AwsCredentialsProviderFactory() {
+ }
+
+ /* =========================
+ * AWS SDK V1
+ * ========================= */
+
+ public static com.amazonaws.auth.AWSCredentialsProvider createV1(
+ AwsCredentialsProviderMode mode) {
+
+ switch (mode) {
+ case ENV:
+ return new
com.amazonaws.auth.EnvironmentVariableCredentialsProvider();
+ case SYSTEM_PROPERTIES:
+ return new
com.amazonaws.auth.SystemPropertiesCredentialsProvider();
+ case WEB_IDENTITY:
+ return
com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create();
+ case CONTAINER:
+ return new
com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper();
+ case ANONYMOUS:
+ throw new UnsupportedOperationException(
+ "AWS SDK V1 does not support anonymous credentials
provider.");
+ case INSTANCE_PROFILE:
+ return new
com.amazonaws.auth.InstanceProfileCredentialsProvider();
+ case DEFAULT:
+ return createDefaultV1();
+ default:
+ throw new UnsupportedOperationException(
+ "AWS SDK V1 does not support credentials provider
mode: " + mode);
+ }
+ }
+
+ private static com.amazonaws.auth.AWSCredentialsProvider createDefaultV1()
{
+ List<com.amazonaws.auth.AWSCredentialsProvider> providers = new
ArrayList<>();
+ providers.add(new
com.amazonaws.auth.InstanceProfileCredentialsProvider());
+ //lazy + env
+
providers.add(com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create());
+ providers.add(new
com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper());
+ providers.add(new
com.amazonaws.auth.EnvironmentVariableCredentialsProvider());
+ providers.add(new
com.amazonaws.auth.SystemPropertiesCredentialsProvider());
+ return new com.amazonaws.auth.AWSCredentialsProviderChain(
+ providers.toArray(new
com.amazonaws.auth.AWSCredentialsProvider[0]));
+ }
+
+ /* =========================
+ * AWS SDK V2
+ * ========================= */
+
+ public static AwsCredentialsProvider createV2(
+ AwsCredentialsProviderMode mode,
+ boolean includeAnonymousInDefault) {
+ switch (mode) {
+ case ENV:
+ return EnvironmentVariableCredentialsProvider.create();
+ case SYSTEM_PROPERTIES:
+ return SystemPropertyCredentialsProvider.create();
+ case WEB_IDENTITY:
+ return WebIdentityTokenFileCredentialsProvider.create();
+ case CONTAINER:
+ return ContainerCredentialsProvider.create();
+ case INSTANCE_PROFILE:
+ return InstanceProfileCredentialsProvider.create();
+ case ANONYMOUS:
+ return AnonymousCredentialsProvider.create();
+ case DEFAULT:
+ return createDefaultV2(includeAnonymousInDefault);
+ default:
+ throw new UnsupportedOperationException(
+ "AWS SDK V2 does not support credentials provider
mode: " + mode);
+ }
+ }
+
+ private static AwsCredentialsProvider createDefaultV2(
+ boolean includeAnonymous) {
+
+ List<AwsCredentialsProvider> providers = new ArrayList<>();
+ providers.add(InstanceProfileCredentialsProvider.create());
+ providers.add(WebIdentityTokenFileCredentialsProvider.create());
+ providers.add(ContainerCredentialsProvider.create());
+ providers.add(EnvironmentVariableCredentialsProvider.create());
+ providers.add(SystemPropertyCredentialsProvider.create());
+ if (includeAnonymous) {
+ providers.add(AnonymousCredentialsProvider.create());
+ }
+ return AwsCredentialsProviderChain.builder()
+ .credentialsProviders(providers)
+ .build();
+ }
+
+ public static String getV2ClassName(AwsCredentialsProviderMode mode,
boolean includeAnonymousInDefault) {
+ switch (mode) {
+ case ENV:
+ return EnvironmentVariableCredentialsProvider.class.getName();
+ case SYSTEM_PROPERTIES:
+ return SystemPropertyCredentialsProvider.class.getName();
+ case WEB_IDENTITY:
+ return WebIdentityTokenFileCredentialsProvider.class.getName();
+ case CONTAINER:
+ return ContainerCredentialsProvider.class.getName();
+ case INSTANCE_PROFILE:
+ return InstanceProfileCredentialsProvider.class.getName();
+ case ANONYMOUS:
+ return AnonymousCredentialsProvider.class.getName();
+ case DEFAULT:
+ List<String> providers = new ArrayList<>();
+
providers.add(EnvironmentVariableCredentialsProvider.class.getName());
+
providers.add(SystemPropertyCredentialsProvider.class.getName());
+
providers.add(WebIdentityTokenFileCredentialsProvider.class.getName());
+ providers.add(ContainerCredentialsProvider.class.getName());
+
providers.add(InstanceProfileCredentialsProvider.class.getName());
+ if (includeAnonymousInDefault) {
+
providers.add(AnonymousCredentialsProvider.class.getName());
+ }
+ return String.join("+", providers);
+ default:
+ throw new UnsupportedOperationException(
+ "AWS SDK V2 does not support credentials provider
mode: " + mode);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java
new file mode 100644
index 00000000000..63481ca5a7a
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.common;
+
+public enum AwsCredentialsProviderMode {
+
+ DEFAULT("DEFAULT"),
+
+ ENV("ENV"),
+
+ SYSTEM_PROPERTIES("SYSTEM_PROPERTIES"),
+
+ WEB_IDENTITY("WEB_IDENTITY"),
+
+ CONTAINER("CONTAINER"),
+
+ INSTANCE_PROFILE("INSTANCE_PROFILE"),
+
+ ANONYMOUS("ANONYMOUS");
+
+ private final String mode;
+
+ AwsCredentialsProviderMode(String mode) {
+ this.mode = mode;
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+
+ public static AwsCredentialsProviderMode fromString(String value) {
+ if (value == null || value.isEmpty()) {
+ return DEFAULT;
+ }
+
+ String normalized = value.trim().toUpperCase().replace('-', '_');
+
+ switch (normalized) {
+ case "ENV":
+ return ENV;
+ case "SYSTEM_PROPERTIES":
+ return SYSTEM_PROPERTIES;
+ case "WEB_IDENTITY":
+ return WEB_IDENTITY;
+ case "CONTAINER":
+ return CONTAINER;
+ case "INSTANCE_PROFILE":
+ return INSTANCE_PROFILE;
+ case "ANONYMOUS":
+ return ANONYMOUS;
+ case "DEFAULT":
+ return DEFAULT;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported AWS credentials provider mode: " + value);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
index 445b8311f85..b17504fbca7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
@@ -114,8 +114,6 @@ public class AWSGlueMetaStoreBaseProperties {
return new ParamRules().requireTogether(new String[]{glueAccessKey,
glueSecretKey},
"glue.access_key and glue.secret_key must be set
together")
- .requireAtLeastOne(new String[]{glueAccessKey, glueIAMRole},
- "At least one of glue.access_key or glue.role_arn must
be set")
.require(glueEndpoint, "glue.endpoint must be set")
.check(() -> StringUtils.isNotBlank(glueEndpoint) &&
!glueEndpoint.startsWith("https://";),
"glue.endpoint must use https protocol,please set
glue.endpoint to https://...";);
@@ -137,6 +135,22 @@ public class AWSGlueMetaStoreBaseProperties {
}
}
+ /**
+ * Validate that at least one Glue credential (an access key or an IAM
role) is explicitly provided.
+ *
+ * Purpose: Some catalog implementations (for example, Iceberg) do not
support obtaining credentials
+ * from the default credential chain (instance metadata, environment
variables, etc.). In addition,
+ * the configuration or UI may only expose two options: {@code
glue.access_key} and {@code glue.role_arn}.
+ * In such cases, at least one of these must be explicitly set. If neither
is provided, an
+ * {@link IllegalArgumentException} is thrown to prompt the user to
complete the configuration.
+ */
+ protected void requireExplicitGlueCredentials() {
+ if (StringUtils.isNotBlank(glueAccessKey) ||
StringUtils.isNotBlank(glueIAMRole)) {
+ return;
+ }
+ throw new IllegalArgumentException("At least one of glue.access_key or
glue.role_arn must be set");
+ }
+
private String extractRegionFromEndpoint(Matcher matcher) {
for (int i = 1; i <= matcher.groupCount(); i++) {
String group = matcher.group(i);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java
index 5a90a3af7dc..06cd55b431b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.metastore;
import org.apache.doris.datasource.property.ConnectorProperty;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.glue.catalog.util.AWSGlueConfig;
@@ -75,6 +76,14 @@ public class HiveGlueMetaStoreProperties extends
AbstractHiveProperties {
description = "Catalog separator character for AWS Glue.")
protected String awsGlueCatalogSeparator = "";
+ @ConnectorProperty(names = {"glue.credentials_provider_type"},
+ required = false,
+ description = "The credentials provider type of S3. "
+ + "Options are: DEFAULT, ASSUME_ROLE, ANONYMOUS,
ENVIRONMENT, SYSTEM_PROPERTIES, "
+ + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. "
+ + "If not set, it will use the default provider chain of
AWS SDK.")
+ protected String awsCredentialsProviderType =
AwsCredentialsProviderMode.DEFAULT.name();
+
// ========== Constructor ==========
/**
@@ -115,6 +124,8 @@ public class HiveGlueMetaStoreProperties extends
AbstractHiveProperties {
setHiveConfPropertiesIfNotNull(hiveConf,
AWSGlueConfig.AWS_GLUE_SESSION_TOKEN, baseProperties.glueSessionToken);
setHiveConfPropertiesIfNotNull(hiveConf,
AWSGlueConfig.AWS_GLUE_ROLE_ARN, baseProperties.glueIAMRole);
setHiveConfPropertiesIfNotNull(hiveConf,
AWSGlueConfig.AWS_GLUE_EXTERNAL_ID, baseProperties.glueExternalId);
+ setHiveConfPropertiesIfNotNull(hiveConf,
+ AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE,
awsCredentialsProviderType);
}
private static void setHiveConfPropertiesIfNotNull(HiveConf hiveConf,
String key, String value) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
index ba40940005b..3039a96ea9d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
@@ -55,6 +55,7 @@ public class IcebergGlueMetaStoreProperties extends
AbstractIcebergProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
+ glueProperties.requireExplicitGlueCredentials();
s3Properties = S3Properties.of(origProps);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index ffa6e6c7450..3452d759395 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -24,6 +24,8 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
import org.apache.doris.datasource.property.ConnectorProperty;
+import
org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
import org.apache.doris.thrift.TCredProviderType;
import org.apache.doris.thrift.TS3StorageParam;
@@ -33,15 +35,11 @@ import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
-import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.sts.StsClient;
import
software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
@@ -57,6 +55,8 @@ import java.util.stream.Stream;
public class S3Properties extends AbstractS3CompatibleProperties {
+ private static final Logger LOG = LogManager.getLogger(S3Properties.class);
+
public static final String USE_PATH_STYLE = "use_path_style";
private static final String[] ENDPOINT_NAMES_FOR_GUESSING = {
@@ -170,6 +170,16 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
description = "The external id of S3.")
protected String s3ExternalId = "";
+ @ConnectorProperty(names = {"s3.credentials_provider_type",
"glue.credentials_provider_type"},
+ required = false,
+ description = "The credentials provider type of S3. "
+ + "Options are: DEFAULT, ASSUME_ROLE, ENVIRONMENT,
SYSTEM_PROPERTIES, "
+ + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. "
+ + "If not set, it will use the default provider chain of
AWS SDK.")
+ protected String awsCredentialsProviderType =
AwsCredentialsProviderMode.DEFAULT.name();
+
+ private AwsCredentialsProviderMode awsCredentialsProviderMode;
+
public static S3Properties of(Map<String, String> properties) {
S3Properties propertiesObj = new S3Properties(properties);
ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj,
properties);
@@ -215,6 +225,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
throw new IllegalArgumentException("s3.external_id must be used
with s3.role_arn");
}
convertGlueToS3EndpointIfNeeded();
+ awsCredentialsProviderMode =
AwsCredentialsProviderMode.fromString(awsCredentialsProviderType);
}
/**
@@ -310,15 +321,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
}).build();
}
// For anonymous access (no credentials required)
- //fixme: should return AwsCredentialsProviderChain
- if (StringUtils.isBlank(accessKey) && StringUtils.isBlank(secretKey)) {
- return AnonymousCredentialsProvider.create();
- }
- return
AwsCredentialsProviderChain.of(SystemPropertyCredentialsProvider.create(),
- EnvironmentVariableCredentialsProvider.create(),
- WebIdentityTokenFileCredentialsProvider.create(),
- ProfileCredentialsProvider.create(),
- InstanceProfileCredentialsProvider.create());
+ return AnonymousCredentialsProvider.create();
}
private AwsCredentialsProvider getAwsCredentialsProviderV2() {
@@ -329,15 +332,10 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
if (StringUtils.isNotBlank(s3IAMRole)) {
StsClient stsClient = StsClient.builder()
.region(Region.of(region))
- .credentialsProvider(AwsCredentialsProviderChain.of(
- WebIdentityTokenFileCredentialsProvider.create(),
- ContainerCredentialsProvider.create(),
- InstanceProfileCredentialsProvider.create(),
- SystemPropertyCredentialsProvider.create(),
- EnvironmentVariableCredentialsProvider.create(),
- ProfileCredentialsProvider.create()))
+
.credentialsProvider(AwsCredentialsProviderFactory.createV2(
+ awsCredentialsProviderMode,
+ false))
.build();
-
return StsAssumeRoleCredentialsProvider.builder()
.stsClient(stsClient)
.refreshRequest(builder -> {
@@ -347,13 +345,9 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
}
}).build();
}
- return AwsCredentialsProviderChain.of(
- WebIdentityTokenFileCredentialsProvider.create(),
- ContainerCredentialsProvider.create(),
- InstanceProfileCredentialsProvider.create(),
- SystemPropertyCredentialsProvider.create(),
- EnvironmentVariableCredentialsProvider.create(),
- ProfileCredentialsProvider.create());
+ return AwsCredentialsProviderFactory.createV2(
+ awsCredentialsProviderMode,
+ true);
}
@Override
@@ -374,12 +368,26 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
hadoopStorageConfig.set("fs.s3a.assumed.role.arn", s3IAMRole);
hadoopStorageConfig.set("fs.s3a.aws.credentials.provider",
"org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider");
- hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider",
-
"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,com.amazonaws.auth.EnvironmentVar"
- +
"iableCredentialsProvider,com.amazonaws.auth.InstanceProfileCredentialsProvider");
+ if
(Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
+
hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider",
+ AwsCredentialsProviderFactory.getV2ClassName(
+ awsCredentialsProviderMode,
+ false));
+ } else {
+
hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider",
+ InstanceProfileCredentialsProvider.class.getName());
+ }
+
if (StringUtils.isNotBlank(s3ExternalId)) {
hadoopStorageConfig.set("fs.s3a.assumed.role.external.id",
s3ExternalId);
}
+ return;
+ }
+ if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
+ hadoopStorageConfig.set("fs.s3a.aws.credentials.provider",
+ AwsCredentialsProviderFactory.createV2(
+ awsCredentialsProviderMode,
+ true).getClass().getName());
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 0c279df3368..29f9a126ece 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -223,9 +223,10 @@ public class S3PropertiesTest {
Assertions.assertEquals("arn:aws:iam::123456789012:role/MyTestRole",
backendProperties.get("AWS_ROLE_ARN"));
}
+
@Test
public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked
StsClientBuilder mockBuilder,
- @Mocked
StsClient mockStsClient, @Mocked InstanceProfileCredentialsProvider
mockInstanceCreds) {
+ @Mocked
StsClient mockStsClient) {
new Expectations() {
{
@@ -235,8 +236,6 @@ public class S3PropertiesTest {
result = mockBuilder;
mockBuilder.build();
result = mockStsClient;
- InstanceProfileCredentialsProvider.create();
- result = mockInstanceCreds;
}
};
@@ -248,6 +247,29 @@ public class S3PropertiesTest {
AwsCredentialsProvider provider = s3Props.getAwsCredentialsProvider();
Assertions.assertNotNull(provider);
Assertions.assertTrue(provider instanceof
StsAssumeRoleCredentialsProvider);
+ origProps.put("s3.credentials_provider_type", "instance_profile");
+ s3Props = (S3Properties) StorageProperties.createPrimary(origProps);
+ provider = s3Props.getAwsCredentialsProvider();
+
Assertions.assertEquals(StsAssumeRoleCredentialsProvider.class.getName(),
provider.getClass().getName());
+
Assertions.assertEquals(InstanceProfileCredentialsProvider.class.getName(),
s3Props.getHadoopStorageConfig().get("fs.s3a.assumed.role.credentials.provider"));
+ origProps.remove("s3.external_id");
+ origProps.remove("s3.role_arn");
+ origProps.remove("s3.credentials_provider_type");
+ s3Props = (S3Properties) StorageProperties.createPrimary(origProps);
+ provider = s3Props.getAwsCredentialsProvider();
+ Assertions.assertNotNull(provider);
+ Assertions.assertTrue(provider instanceof AwsCredentialsProviderChain);
+ origProps.put("s3.credentials_provider_type", "instance_profile");
+ s3Props = (S3Properties) StorageProperties.createPrimary(origProps);
+ provider = s3Props.getAwsCredentialsProvider();
+ Assertions.assertNotNull(provider);
+ Assertions.assertTrue(provider instanceof
InstanceProfileCredentialsProvider);
+
Assertions.assertEquals(InstanceProfileCredentialsProvider.class.getName(),
s3Props.getHadoopStorageConfig().get("fs.s3a.aws.credentials.provider"));
+ origProps.put("s3.credentials_provider_type", "static");
+ ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class,
"Unsupported AWS credentials provider mode: static", () ->
StorageProperties.createPrimary(origProps));
+ origProps.put("s3.credentials_provider_type", "anonymous");
+ Assertions.assertDoesNotThrow(() ->
StorageProperties.createPrimary(origProps));
+
}
@Test
diff --git a/fe/pom.xml b/fe/pom.xml
index 5cc39c501a3..278f6c39673 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -230,7 +230,7 @@ under the License.
<gcs.version>3.1.9</gcs.version>
<obs.dependency.scope>compile</obs.dependency.scope>
<cos.dependency.scope>compile</cos.dependency.scope>
- <gcs.dependency.scope>compile</gcs.dependency.scope>
+ <gcs.dependency.scope>provided</gcs.dependency.scope>
<qcloud-java-sdk.version>2.0.1</qcloud-java-sdk.version>
<tencentcos.version>8.2.7</tencentcos.version>
<cos-api.version>5.6.211</cos-api.version>
diff --git
a/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy
b/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy
new file mode 100644
index 00000000000..f8877929b96
--- /dev/null
+++
b/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import com.google.common.base.Strings;
+
+suite("test_catalog_instance_profile_with_role") {
+
+ if
(Strings.isNullOrEmpty(context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName")))
{
+ return
+ }
+
+ String hiveGlueQueryTableName =
context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName")
+ String hiveGlueExpectCounts =
context.config.otherConfigs.get("hiveGlueInstanceProfileExpectCounts")
+ String icebergFsQueryTableName =
context.config.otherConfigs.get("icebergFsInstanceProfileQueryTableName")
+ String icebergFsExpectCounts =
context.config.otherConfigs.get("icebergFsInstanceProfileExpectCounts")
+ String icebergFsWarehouse =
context.config.otherConfigs.get("icebergFsInstanceProfileWarehouse")
+ String region = context.config.otherConfigs.get("awsInstanceProfileRegion")
+ //query method
+ def createCatalogAndQuery = { catalogProps, catalogName, queryTableName,
expectCounts ->
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ ${catalogProps}
+ """
+ def result = sql """
+ select count(1) from ${catalogName}.${queryTableName};
+ """
+ println("result: ${result}")
+ def countValue = result[0][0]
+ assertTrue(countValue == expectCounts.toInteger())
+ sql """drop catalog if exists ${catalogName}"""
+ }
+ def assertCatalogAndQueryException = { catalogProps, catalogName, errMsg ->
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ ${catalogProps}
+ """
+ try {
+ sql """
+ switch ${catalogName};
+ """
+ sql """
+ show databases;
+ """
+ throw new Exception("Expected exception was not thrown")
+ }catch (Exception e){
+ assertTrue(e.getMessage().contains(errMsg))
+ }
+ }
+ String hiveGlueCatalogProps = """
+ create catalog hive_glue_catalog properties(
+ "type"="hms",
+ "hive.metastore.type"="glue",
+ "glue.region"="${region}",
+ "glue.endpoint" = "https://glue.${region}.amazonaws.com";
+ );
+ """
+ createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog",
hiveGlueQueryTableName, hiveGlueExpectCounts)
+ hiveGlueCatalogProps = """
+ create catalog hive_glue_catalog properties(
+ "type"="hms",
+ "hive.metastore.type"="glue",
+ "glue.credentials_provider_type"="INSTANCE_PROFILE",
+ "glue.region"="${region}",
+ "glue.endpoint" = "https://glue.${region}.amazonaws.com";
+ );
+ """
+ createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog",
hiveGlueQueryTableName, hiveGlueExpectCounts)
+ hiveGlueCatalogProps = """
+ create catalog hive_glue_catalog properties(
+ "type"="hms",
+ "hive.metastore.type"="glue",
+ "glue.credentials_provider_type"="CONTAINER",
+ "glue.region"="${region}",
+ "glue.endpoint" = "https://glue.${region}.amazonaws.com";
+ );
+ """
+ assertCatalogAndQueryException(hiveGlueCatalogProps,"hive_glue_catalog",
"The environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")
+ String icebergFsCatalogProps = """
+ create catalog iceberg_fs_catalog properties(
+ "type"="iceberg",
+ "warehouse"="${icebergFsWarehouse}",
+ "iceberg.catalog.type"="hadoop",
+ "s3.region" = "${region}",
+ "s3.endpoint" = "https://s3.${region}.amazonaws.com";
+ );
+ """
+ createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog",
icebergFsQueryTableName, icebergFsExpectCounts)
+ icebergFsCatalogProps = """
+ create catalog iceberg_fs_catalog properties(
+ "type"="iceberg",
+ "warehouse"="${icebergFsWarehouse}",
+ "iceberg.catalog.type"="hadoop",
+ "s3.credentials_provider_type"="INSTANCE_PROFILE",
+ "s3.region" = "${region}",
+ "s3.endpoint" = "https://s3.${region}.amazonaws.com";
+ );
+ """
+ createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog",
icebergFsQueryTableName, icebergFsExpectCounts)
+ icebergFsCatalogProps = """
+ create catalog iceberg_fs_catalog properties(
+ "type"="iceberg",
+ "warehouse"="${icebergFsWarehouse}",
+ "iceberg.catalog.type"="hadoop",
+ "s3.credentials_provider_type"="CONTAINER",
+ "s3.region" = "${region}",
+ "s3.endpoint" = "https://s3.${region}.amazonaws.com";
+ );
+ """
+ assertCatalogAndQueryException(icebergFsCatalogProps,"iceberg_fs_catalog",
"No AWS Credentials provided by ContainerCredentialsProvider")
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
