This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 21d56a5c975 [fix](S3)Downgrade the AWS SDK version to avoid impacting
other S3-compatible services. (#58566)
21d56a5c975 is described below
commit 21d56a5c975855e6f30280e3152675cd6d1afc58
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Dec 2 12:55:51 2025 +0800
[fix](S3)Downgrade the AWS SDK version to avoid impacting other
S3-compatible services. (#58566)
…
This PR addresses an issue related to AWS SDK for Java v2 S3 client
behavior described in:
https://github.com/aws/aws-sdk-java-v2/issues/5805
Starting from newer versions of the SDK, the S3 client introduces
unexpected behavior that causes our application to fail during normal S3
operations. Due to this problem, we were forced to downgrade aws-s3 from
the latest version to 2.29.x, which restored correct functionality. The
regression affects stability in our production environment and prevents
us from upgrading the SDK.
#57226
---
.../storage/AbstractS3CompatibleProperties.java | 30 ----------------------
.../property/storage/OSSPropertiesTest.java | 10 --------
fe/pom.xml | 3 ++-
3 files changed, 2 insertions(+), 41 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index ed03e1f0fcd..4b8997b6d56 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -29,7 +29,6 @@ import
software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.SdkSystemSetting;
import java.util.HashMap;
import java.util.Map;
@@ -251,7 +250,6 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
// storage is OSS, OBS, etc., users may still configure the schema as
"s3://".
// To ensure backward compatibility, we append S3-related properties
by default.
appendS3HdfsProperties(hadoopStorageConfig);
- setDefaultRequestChecksum();
}
private void appendS3HdfsProperties(Configuration hadoopStorageConfig) {
@@ -277,34 +275,6 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
hadoopStorageConfig.set("fs.s3a.path.style.access", getUsePathStyle());
}
- /**
- * Sets the AWS request checksum calculation property to "WHEN_REQUIRED"
- * only if it has not been explicitly set by the user.
- *
- * <p>
- * Background:
- * AWS SDK for Java v2 uses the system property
- * {@link SdkSystemSetting#AWS_REQUEST_CHECKSUM_CALCULATION} to determine
- * whether request payloads should have a checksum calculated.
- * <p>
- * According to the official AWS discussion:
- * https://github.com/aws/aws-sdk-java-v2/discussions/5802
- * - Default SDK behavior may calculate checksums automatically if the
property is not set.
- * - Automatic calculation can affect performance or cause unexpected
behavior for large requests.
- * <p>
- * This method ensures:
- * 1. The property is set to "WHEN_REQUIRED" only if the user has not
already set it.
- * 2. User-specified settings are never overridden.
- * 3. Aligns with AWS SDK recommended best practices.
- * </p>
- */
- public static void setDefaultRequestChecksum() {
- String key =
SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property();
- if (System.getProperty(key) == null) {
- System.setProperty(key, "WHEN_REQUIRED");
- }
- }
-
@Override
public String getStorageName() {
return "S3";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 5d6c5dae12f..0bc5e823f0e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -25,7 +25,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.SdkSystemSetting;
import java.util.HashMap;
import java.util.Map;
@@ -269,13 +268,4 @@ public class OSSPropertiesTest {
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache",
false));
}
- @Test
- public void testResuestCheckSum() throws UserException {
- Map<String, String> props = Maps.newHashMap();
- props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
- Assertions.assertEquals("WHEN_REQUIRED",
System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property()));
- System.setProperty("aws.requestChecksumCalculation", "ALWAYS");
- Assertions.assertEquals("ALWAYS",
System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property()));
- }
-
}
diff --git a/fe/pom.xml b/fe/pom.xml
index b8bf5dc3fe1..a648bef6e18 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -395,7 +395,8 @@ under the License.
<azure.sdk.batch.version>12.22.0</azure.sdk.batch.version>
<semver4j.version>5.3.0</semver4j.version>
<aliyun-sdk-oss.version>3.15.0</aliyun-sdk-oss.version>
- <awssdk.version>2.37.2</awssdk.version>
+ <!--Fixes the regression described in
https://github.com/aws/aws-sdk-java-v2/issues/5805 that forced us to downgrade
aws-s3 to version 2.29.x.-->
+ <awssdk.version>2.29.52</awssdk.version>
<s3tables.catalog.version>0.1.4</s3tables.catalog.version>
<mockito.version>4.11.0</mockito.version>
</properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
