This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 8391ed0466c branch-4.0: [feat](s3) support S3 storage vault
credentials provider without role ARN (#65011)
8391ed0466c is described below
commit 8391ed0466c0ecaaea3998c65e844e92c67ecc0d
Author: hui lai <[email protected]>
AuthorDate: Thu Jul 2 14:57:48 2026 +0800
branch-4.0: [feat](s3) support S3 storage vault credentials provider
without role ARN (#65011)
pick https://github.com/apache/doris/pull/64766
---
cloud/src/meta-service/meta_service_resource.cpp | 37 +++++++++++------
cloud/src/recycler/s3_accessor.cpp | 8 +++-
cloud/test/meta_service_test.cpp | 46 ++++++++++++++++++++++
.../datasource/property/storage/S3Properties.java | 37 +++++++++++++++--
.../property/storage/S3PropertiesTest.java | 19 +++++++++
5 files changed, 131 insertions(+), 16 deletions(-)
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 5afb92888ce..08fe741e61d 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -59,6 +59,11 @@ bool is_valid_storage_vault_name(const std::string& str) {
namespace doris::cloud {
+static CredProviderTypePB get_cred_provider_type(const ObjectStoreInfoPB& obj)
{
+ return obj.has_cred_provider_type() ? obj.cred_provider_type()
+ : CredProviderTypePB::INSTANCE_PROFILE;
+}
+
static std::string_view print_cluster_status(const ClusterStatus& status) {
switch (status) {
case ClusterStatus::UNKNOWN:
@@ -665,6 +670,14 @@ static int add_hdfs_storage_vault(InstanceInfoPB&
instance, Transaction* txn,
return 0;
}
+static bool has_non_empty_role_arn(const ObjectStoreInfoPB& obj) {
+ return obj.has_role_arn() && !obj.role_arn().empty();
+}
+
+static bool use_credential_provider(const ObjectStoreInfoPB& obj) {
+ return obj.has_cred_provider_type() || has_non_empty_role_arn(obj);
+}
+
static void create_object_info_with_encrypt(const InstanceInfoPB& instance,
ObjectStoreInfoPB* obj,
bool sse_enabled, MetaServiceCode&
code,
std::string& msg) {
@@ -1107,7 +1120,9 @@ static int extract_object_storage_info(const
AlterObjStoreInfoRequest* request,
auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region,
use_path_style, role_arn,
external_id] = obj_desc;
- if (!obj.has_role_arn()) {
+ bool use_credential_provider_for_add_vault =
+ request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT &&
obj.has_cred_provider_type();
+ if (!obj.has_role_arn() && !use_credential_provider_for_add_vault) {
if (!obj.has_ak() || !obj.has_sk()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 obj info err " + proto_to_json(*request);
@@ -1163,14 +1178,16 @@ static ObjectStoreInfoPB
object_info_pb_factory(ObjectStorageDesc& obj_desc,
last_item.set_user_id(obj.user_id());
}
- if (!obj.has_role_arn()) {
+ if (!use_credential_provider(obj)) {
last_item.set_ak(std::move(cipher_ak_sk_pair.first));
last_item.set_sk(std::move(cipher_ak_sk_pair.second));
last_item.mutable_encryption_info()->CopyFrom(encryption_info);
} else {
- last_item.set_role_arn(role_arn);
- last_item.set_external_id(external_id);
- last_item.set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+ if (has_non_empty_role_arn(obj)) {
+ last_item.set_role_arn(role_arn);
+ last_item.set_external_id(external_id);
+ }
+ last_item.set_cred_provider_type(get_cred_provider_type(obj));
}
last_item.set_bucket(bucket);
// format prefix, such as `/aa/bb/`, `aa/bb//`, `//aa/bb`, ` /aa/bb` ->
`aa/bb`
@@ -1322,19 +1339,17 @@ void
MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
return;
}
// ATTN: prefix may be empty
- if (((ak.empty() || sk.empty()) && role_arn.empty()) || bucket.empty()
||
+ if (((ak.empty() || sk.empty()) && !use_credential_provider(obj)) ||
bucket.empty() ||
endpoint.empty() || region.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 conf info err, please check it";
return;
}
- if (!role_arn.empty()) {
- if (!obj.has_cred_provider_type() ||
- obj.cred_provider_type() !=
CredProviderTypePB::INSTANCE_PROFILE ||
- !obj.has_provider() || obj.provider() !=
ObjectStoreInfoPB::S3) {
+ if (use_credential_provider(obj)) {
+ if (!obj.has_provider() || obj.provider() !=
ObjectStoreInfoPB::S3) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 conf info err with role_arn, please check it";
+ msg = "s3 conf info err with credentials_provider_type, please
check it";
return;
}
}
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 1b331ea5167..5bba3f4a070 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -235,10 +235,16 @@ std::optional<S3Conf> S3Conf::from_obj_store_info(const
ObjectStoreInfoPB& obj_i
}
}
+ if (obj_info.has_cred_provider_type()) {
+ s3_conf.cred_provider_type =
cred_provider_type_from_pb(obj_info.cred_provider_type());
+ }
+
if (obj_info.has_role_arn() && !obj_info.role_arn().empty()) {
s3_conf.role_arn = obj_info.role_arn();
s3_conf.external_id = obj_info.external_id();
- s3_conf.cred_provider_type = CredProviderType::InstanceProfile;
+ if (!obj_info.has_cred_provider_type()) {
+ s3_conf.cred_provider_type = CredProviderType::InstanceProfile;
+ }
}
}
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 0670dee914b..f914c2df845 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -10483,6 +10483,52 @@ TEST(MetaServiceTest, CreateS3VaultWithIamRole) {
}
}
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ADD_S3_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_obj_info()->set_endpoint("s3.us-east-1.amazonaws.com");
+ vault.mutable_obj_info()->set_region("us-east-1");
+
vault.mutable_obj_info()->set_bucket("test_credential_provider_bucket");
+
vault.mutable_obj_info()->set_prefix("test_credential_provider_prefix");
+ vault.mutable_obj_info()->set_provider(
+ ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3);
+
vault.mutable_obj_info()->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE);
+
+ vault.set_name("s3_vault_with_credential_provider");
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+ {
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(),
"5"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_TRUE(get_obj.obj_info().ak().empty()) <<
get_obj.obj_info().ak();
+ ASSERT_TRUE(get_obj.obj_info().sk().empty()) <<
get_obj.obj_info().sk();
+ ASSERT_FALSE(get_obj.obj_info().has_role_arn());
+ ASSERT_FALSE(get_obj.obj_info().has_external_id());
+ ASSERT_EQ(get_obj.obj_info().cred_provider_type(),
CredProviderTypePB::INSTANCE_PROFILE)
+ << get_obj.obj_info().cred_provider_type();
+ ASSERT_EQ(get_obj.obj_info().bucket(),
"test_credential_provider_bucket")
+ << get_obj.obj_info().bucket();
+ ASSERT_EQ(get_obj.obj_info().prefix(),
"test_credential_provider_prefix")
+ << get_obj.obj_info().prefix();
+ ASSERT_EQ(get_obj.id(), "5") << get_obj.id();
+ }
+ }
+
LOG(INFO) << "instance:" << instance.ShortDebugString();
SyncPoint::get_instance()->disable_processing();
SyncPoint::get_instance()->clear_all_call_backs();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 9b53191e228..6d997eab00a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.cloud.proto.Cloud;
-import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB;
import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -472,6 +471,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
public static final String ROLE_ARN = "AWS_ROLE_ARN";
public static final String EXTERNAL_ID = "AWS_EXTERNAL_ID";
+ public static final String CREDENTIALS_PROVIDER_TYPE =
"AWS_CREDENTIALS_PROVIDER_TYPE";
public static final List<String> REQUIRED_FIELDS =
Arrays.asList(ENDPOINT);
public static final List<String> FS_KEYS = Arrays.asList(ENDPOINT,
REGION, ACCESS_KEY, SECRET_KEY, TOKEN,
@@ -561,6 +561,9 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
if (properties.containsKey(Env.EXTERNAL_ID)) {
properties.putIfAbsent(EXTERNAL_ID,
properties.get(Env.EXTERNAL_ID));
}
+ if (properties.containsKey(Env.CREDENTIALS_PROVIDER_TYPE)) {
+ properties.putIfAbsent(CREDENTIALS_PROVIDER_TYPE,
properties.get(Env.CREDENTIALS_PROVIDER_TYPE));
+ }
}
private static final Pattern IPV4_PORT_PATTERN =
Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})");
@@ -593,6 +596,23 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
properties.putIfAbsent(Env.CONNECTION_TIMEOUT_MS,
Env.DEFAULT_CONNECTION_TIMEOUT_MS);
}
+ private static boolean hasCredentialsProviderType(Map<String, String>
properties) {
+ return properties.containsKey(CREDENTIALS_PROVIDER_TYPE)
+ || properties.containsKey(Env.CREDENTIALS_PROVIDER_TYPE);
+ }
+
+ private static Cloud.CredProviderTypePB getCredProviderTypePB(Map<String,
String> properties) {
+ String modeValue = properties.get(CREDENTIALS_PROVIDER_TYPE);
+ if (StringUtils.isBlank(modeValue)) {
+ modeValue = properties.get(Env.CREDENTIALS_PROVIDER_TYPE);
+ }
+ AwsCredentialsProviderMode mode =
AwsCredentialsProviderMode.fromString(
+ StringUtils.defaultIfBlank(modeValue,
AwsCredentialsProviderMode.INSTANCE_PROFILE.name()));
+ Preconditions.checkArgument(mode ==
AwsCredentialsProviderMode.INSTANCE_PROFILE,
+ "Unsupported AWS credentials provider mode for cloud storage
vault: %s", mode);
+ return Cloud.CredProviderTypePB.INSTANCE_PROFILE;
+ }
+
public static Cloud.ObjectStoreInfoPB.Builder
getObjStoreInfoPB(Map<String, String> properties) {
Cloud.ObjectStoreInfoPB.Builder builder =
Cloud.ObjectStoreInfoPB.newBuilder();
if (properties.containsKey(S3Properties.ENDPOINT)) {
@@ -632,12 +652,21 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
builder.setUsePathStyle(value.equalsIgnoreCase("true"));
}
+ if (hasCredentialsProviderType(properties)) {
+ builder.setCredProviderType(getCredProviderTypePB(properties));
+ }
+
if (properties.containsKey(S3Properties.ROLE_ARN)) {
- builder.setRoleArn(properties.get(S3Properties.ROLE_ARN));
- if (properties.containsKey(S3Properties.EXTERNAL_ID)) {
+ String roleArn = properties.get(S3Properties.ROLE_ARN);
+ if (!Strings.isNullOrEmpty(roleArn)) {
+ builder.setRoleArn(roleArn);
+ }
+ if (!Strings.isNullOrEmpty(roleArn) &&
properties.containsKey(S3Properties.EXTERNAL_ID)) {
builder.setExternalId(properties.get(S3Properties.EXTERNAL_ID));
}
- builder.setCredProviderType(CredProviderTypePB.INSTANCE_PROFILE);
+ if (!Strings.isNullOrEmpty(roleArn) &&
!builder.hasCredProviderType()) {
+ builder.setCredProviderType(getCredProviderTypePB(properties));
+ }
}
return builder;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index ae5bda12acb..2e716c96f06 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.property.storage;
+import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB;
import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
@@ -314,7 +315,25 @@ public class S3PropertiesTest {
Assertions.assertNotNull(provider);
Assertions.assertTrue(provider instanceof AwsCredentialsProviderChain);
Assertions.assertEquals("software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider,software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider,software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider",
s3Props.getHadoopStorageConfig().get("fs.s3a.aws.credentials.provider"));
+ }
+
+ @Test
+ public void testS3CredentialsProviderTypeWithoutIamRoleForCloud() {
+ origProps.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ origProps.put("s3.region", "us-west-2");
+ origProps.put("s3.bucket", "bucket");
+ origProps.put("s3.root.path", "root");
+ origProps.put("s3.credentials_provider_type", "instance_profile");
+ Assertions.assertEquals(CredProviderTypePB.INSTANCE_PROFILE,
+
S3Properties.getObjStoreInfoPB(origProps).getCredProviderType());
+
Assertions.assertFalse(S3Properties.getObjStoreInfoPB(origProps).hasRoleArn());
+
+ origProps.remove("s3.credentials_provider_type");
+ origProps.put("AWS_CREDENTIALS_PROVIDER_TYPE", "instance_profile");
+ Assertions.assertEquals(CredProviderTypePB.INSTANCE_PROFILE,
+
S3Properties.getObjStoreInfoPB(origProps).getCredProviderType());
+
Assertions.assertFalse(S3Properties.getObjStoreInfoPB(origProps).hasRoleArn());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
