This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 3fed2e185e6 branch-4.0: [fix](S3)Downgrade the AWS SDK version to
avoid impacting other S3-compatible services. #58566 (#58608)
3fed2e185e6 is described below
commit 3fed2e185e6dd6e745dd7c9c6c3d259c6bae82b6
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Dec 2 17:23:47 2025 +0800
branch-4.0: [fix](S3)Downgrade the AWS SDK version to avoid impacting other
S3-compatible services. #58566 (#58608)
Cherry-picked from #58566
Co-authored-by: Calvin Kirs <[email protected]>
---
.../storage/AbstractS3CompatibleProperties.java | 30 ----------------------
.../property/storage/OSSPropertiesTest.java | 10 --------
fe/pom.xml | 3 ++-
3 files changed, 2 insertions(+), 41 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index ed03e1f0fcd..4b8997b6d56 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -29,7 +29,6 @@ import
software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.SdkSystemSetting;
import java.util.HashMap;
import java.util.Map;
@@ -251,7 +250,6 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
// storage is OSS, OBS, etc., users may still configure the schema as
"s3://".
// To ensure backward compatibility, we append S3-related properties
by default.
appendS3HdfsProperties(hadoopStorageConfig);
- setDefaultRequestChecksum();
}
private void appendS3HdfsProperties(Configuration hadoopStorageConfig) {
@@ -277,34 +275,6 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
hadoopStorageConfig.set("fs.s3a.path.style.access", getUsePathStyle());
}
- /**
- * Sets the AWS request checksum calculation property to "WHEN_REQUIRED"
- * only if it has not been explicitly set by the user.
- *
- * <p>
- * Background:
- * AWS SDK for Java v2 uses the system property
- * {@link SdkSystemSetting#AWS_REQUEST_CHECKSUM_CALCULATION} to determine
- * whether request payloads should have a checksum calculated.
- * <p>
- * According to the official AWS discussion:
- * https://github.com/aws/aws-sdk-java-v2/discussions/5802
- * - Default SDK behavior may calculate checksums automatically if the
property is not set.
- * - Automatic calculation can affect performance or cause unexpected
behavior for large requests.
- * <p>
- * This method ensures:
- * 1. The property is set to "WHEN_REQUIRED" only if the user has not
already set it.
- * 2. User-specified settings are never overridden.
- * 3. Aligns with AWS SDK recommended best practices.
- * </p>
- */
- public static void setDefaultRequestChecksum() {
- String key =
SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property();
- if (System.getProperty(key) == null) {
- System.setProperty(key, "WHEN_REQUIRED");
- }
- }
-
@Override
public String getStorageName() {
return "S3";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 5d6c5dae12f..0bc5e823f0e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -25,7 +25,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.SdkSystemSetting;
import java.util.HashMap;
import java.util.Map;
@@ -269,13 +268,4 @@ public class OSSPropertiesTest {
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache",
false));
}
- @Test
- public void testResuestCheckSum() throws UserException {
- Map<String, String> props = Maps.newHashMap();
- props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
- Assertions.assertEquals("WHEN_REQUIRED",
System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property()));
- System.setProperty("aws.requestChecksumCalculation", "ALWAYS");
- Assertions.assertEquals("ALWAYS",
System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property()));
- }
-
}
diff --git a/fe/pom.xml b/fe/pom.xml
index b8bf5dc3fe1..a648bef6e18 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -395,7 +395,8 @@ under the License.
<azure.sdk.batch.version>12.22.0</azure.sdk.batch.version>
<semver4j.version>5.3.0</semver4j.version>
<aliyun-sdk-oss.version>3.15.0</aliyun-sdk-oss.version>
- <awssdk.version>2.37.2</awssdk.version>
+ <!--Fixes the regression described in
https://github.com/aws/aws-sdk-java-v2/issues/5805 that forced us to downgrade
aws-s3 to version 2.29.x.-->
+ <awssdk.version>2.29.52</awssdk.version>
<s3tables.catalog.version>0.1.4</s3tables.catalog.version>
<mockito.version>4.11.0</mockito.version>
</properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
