This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new f03c9e41ccf branch-4.0: [fix](s3)Use anonymous credentials for
S3-compatible storage when credentials are absent #60443 (#60613)
f03c9e41ccf is described below
commit f03c9e41ccf9073365d2e10108d1a0082c032f48
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Feb 10 21:26:21 2026 +0800
branch-4.0: [fix](s3)Use anonymous credentials for S3-compatible storage
when credentials are absent #60443 (#60613)
Cherry-picked from #60443
Co-authored-by: Calvin Kirs <[email protected]>
---
.../storage/AbstractS3CompatibleProperties.java | 12 +++++++++++
.../datasource/property/storage/S3Properties.java | 10 ++++-----
.../property/storage/COSPropertiesTest.java | 4 ++++
.../property/storage/OBSPropertyTest.java | 4 ++++
.../property/storage/OSSPropertiesTest.java | 4 ++++
.../property/storage/S3PropertiesTest.java | 25 ++++++++++++++++++++++
6 files changed, 54 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index 9c5c01a2b3a..f8f7f6fad6c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -18,6 +18,7 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -108,9 +109,20 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
if (StringUtils.isNotBlank(getSessionToken())) {
s3Props.put("AWS_TOKEN", getSessionToken());
}
+ String credentialsProviderType =
getAwsCredentialsProviderTypeForBackend();
+ if (StringUtils.isNotBlank(credentialsProviderType)) {
+ s3Props.put("AWS_CREDENTIALS_PROVIDER_TYPE",
credentialsProviderType);
+ }
return s3Props;
}
+ protected String getAwsCredentialsProviderTypeForBackend() {
+ if (StringUtils.isBlank(getAccessKey()) &&
StringUtils.isBlank(getSecretKey())) {
+ return AwsCredentialsProviderMode.ANONYMOUS.name();
+ }
+ return null;
+ }
+
@Override
public Map<String, String> getBackendConfigProperties() {
return generateBackendS3Configuration();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 63cfe478a0e..df09803be61 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -291,13 +291,14 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
if (StringUtils.isNotBlank(s3ExternalId)) {
backendProperties.put("AWS_EXTERNAL_ID", s3ExternalId);
}
- // Pass credentials provider type to BE
- if (awsCredentialsProviderMode != null) {
- backendProperties.put("AWS_CREDENTIALS_PROVIDER_TYPE",
awsCredentialsProviderMode.getMode());
- }
return backendProperties;
}
+ @Override
+ protected String getAwsCredentialsProviderTypeForBackend() {
+ return awsCredentialsProviderMode == null ? null :
awsCredentialsProviderMode.getMode();
+ }
+
private void convertGlueToS3EndpointIfNeeded() {
if (this.endpoint.contains("glue")) {
this.endpoint = "https://s3."; + this.region + ".amazonaws.com";
@@ -671,4 +672,3 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
}
}
-
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
index d4b0277c946..de919e723c5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
@@ -173,10 +173,14 @@ public class COSPropertiesTest {
props.put("cos.endpoint", "cos.ap-beijing.myqcloud.com");
COSProperties obsStorageProperties = (COSProperties)
StorageProperties.createPrimary(props);
Assertions.assertEquals(AnonymousCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ Map<String, String> backendProps =
obsStorageProperties.getBackendConfigProperties();
+ Assertions.assertEquals("ANONYMOUS",
backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
props.put("cos.access_key", "myAccessKey");
props.put("cos.secret_key", "mySecretKey");
obsStorageProperties = (COSProperties)
StorageProperties.createPrimary(props);
Assertions.assertEquals(StaticCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ backendProps = obsStorageProperties.getBackendConfigProperties();
+
Assertions.assertNull(backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
index 06c2b36cf7d..f08168965d2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
@@ -149,10 +149,14 @@ public class OBSPropertyTest {
props.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
OBSProperties obsStorageProperties = (OBSProperties)
StorageProperties.createPrimary(props);
Assertions.assertEquals(AnonymousCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ Map<String, String> backendProps =
obsStorageProperties.getBackendConfigProperties();
+ Assertions.assertEquals("ANONYMOUS",
backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
props.put("obs.access_key", "myAccessKey");
props.put("obs.secret_key", "mySecretKey");
obsStorageProperties = (OBSProperties)
StorageProperties.createPrimary(props);
Assertions.assertEquals(StaticCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ backendProps = obsStorageProperties.getBackendConfigProperties();
+
Assertions.assertNull(backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 4be6414ae01..912f20b3144 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -246,10 +246,14 @@ public class OSSPropertiesTest {
ossProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
OSSProperties ossStorageProperties = (OSSProperties)
StorageProperties.createPrimary(ossProps);
Assertions.assertEquals(AnonymousCredentialsProvider.class,
ossStorageProperties.getAwsCredentialsProvider().getClass());
+ Map<String, String> backendProps =
ossStorageProperties.getBackendConfigProperties();
+ Assertions.assertEquals("ANONYMOUS",
backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
ossProps.put("oss.access_key", "myAccessKey");
ossProps.put("oss.secret_key", "mySecretKey");
ossStorageProperties = (OSSProperties)
StorageProperties.createPrimary(ossProps);
Assertions.assertEquals(StaticCredentialsProvider.class,
ossStorageProperties.getAwsCredentialsProvider().getClass());
+ backendProps = ossStorageProperties.getBackendConfigProperties();
+
Assertions.assertNull(backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 1882019127c..aa1c95b433d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -149,6 +149,7 @@ public class S3PropertiesTest {
Assertions.assertEquals("88", s3Props.get("AWS_MAX_CONNECTIONS"));
Assertions.assertEquals("6000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
Assertions.assertEquals("true", s3Props.get("use_path_style"));
+ Assertions.assertEquals("DEFAULT",
s3Props.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
origProps.remove("use_path_style");
origProps.remove("s3.connection.maximum");
origProps.remove("s3.connection.timeout");
@@ -159,6 +160,30 @@ public class S3PropertiesTest {
Assertions.assertEquals("6000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
}
+ @Test
+ public void testBackendCredentialsProviderType() throws UserException {
+ Map<String, String> props = new HashMap<>();
+ props.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ props.put("s3.region", "us-west-2");
+
+ String[][] cases = new String[][] {
+ {"default", "DEFAULT"},
+ {"env", "ENV"},
+ {"system_properties", "SYSTEM_PROPERTIES"},
+ {"web_identity", "WEB_IDENTITY"},
+ {"container", "CONTAINER"},
+ {"instance_profile", "INSTANCE_PROFILE"},
+ {"anonymous", "ANONYMOUS"}
+ };
+
+ for (String[] testCase : cases) {
+ props.put("s3.credentials_provider_type", testCase[0]);
+ S3Properties s3Props = (S3Properties)
StorageProperties.createPrimary(props);
+ Map<String, String> backendProps =
s3Props.getBackendConfigProperties();
+ Assertions.assertEquals(testCase[1],
backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
+ }
+ }
+
@Test
public void testGetRegion() throws UserException {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
