This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 07ba0d1e448 [improvement](s3) Default scheme-less S3 endpoints to
HTTPS (#65379)
07ba0d1e448 is described below
commit 07ba0d1e448be58bef46e17c01498cb3d9864891
Author: Refrain <[email protected]>
AuthorDate: Tue Aug 4 23:26:01 2026 +0800
[improvement](s3) Default scheme-less S3 endpoints to HTTPS (#65379)
Problem Summary:
S3 endpoint scheme handling was inconsistent across FE, BE, and Cloud,
and the previous implementation distinguished Doris internal buckets
from user-configured external buckets. That classification leaked into
S3Conf construction and client-cache identity even though the desired
transport rule is the same for every AWS/S3-compatible endpoint.
This PR applies one rule across S3 client construction and
Resource/Storage Vault persistence boundaries:
- Keep the existing FE, BE, and Cloud s3_client_http_scheme
compatibility setting.
- Default every endpoint without an explicit scheme to HTTPS.
- If s3_client_http_scheme is explicitly set to http, use HTTP for
endpoints without a scheme.
- Preserve endpoints that already contain an explicit http:// or
https:// scheme.
- Remove the internal/external bucket classification from BE and Cloud
S3 configuration.
- Apply the same behavior to FE native S3 clients, the S3 filesystem
plugin, connectivity checks, stages, resources, and Iceberg S3 FileIO
handoffs.
- Normalize scheme-less S3 Resource and Storage Vault endpoint metadata
before persistence using the same configured default.
Azure is intentionally unchanged in this PR. Its existing
provider-specific behavior already defaults a bare endpoint to HTTPS and
preserves an explicit scheme; whether Azure should share the S3
compatibility setting is a separate policy decision.
### Release note
S3 endpoints without an explicit scheme now use HTTPS by default across
FE, BE, and Cloud. Set s3_client_http_scheme = http for compatibility
with scheme-less HTTP endpoints. Explicit HTTP and HTTPS endpoint
schemes are unchanged.
---
be/src/common/config.cpp | 4 +-
be/src/common/config.h | 2 +-
be/src/util/s3_util.cpp | 4 +-
be/test/io/s3_client_factory_test.cpp | 29 +++++++++++
cloud/src/common/config.h | 4 +-
cloud/src/recycler/s3_accessor.cpp | 4 +-
cloud/test/util_test.cpp | 30 +++++++++++
common/cpp/aws_common.cpp | 10 ++++
common/cpp/aws_common.h | 38 ++++++++------
.../main/java/org/apache/doris/common/Config.java | 5 ++
.../java/org/apache/doris/catalog/S3Resource.java | 29 ++++++-----
.../java/org/apache/doris/common/util/S3Util.java | 22 +++++---
.../doris/catalog/S3ResourcePersistParityTest.java | 13 +++--
.../org/apache/doris/catalog/S3ResourceTest.java | 22 ++++++--
.../org/apache/doris/common/util/S3UtilTest.java | 58 +++++++++++++++++++++-
.../pipeline/vault_p0/conf/be_custom.conf | 1 +
.../pipeline/vault_p0/conf/fe_custom.conf | 1 +
.../pipeline/vault_p0/conf/recycler_custom.conf | 1 +
.../cold_heat_separation/policy/alter.groovy | 2 +-
19 files changed, 222 insertions(+), 57 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index dede5133117..f365dab488a 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1653,8 +1653,8 @@ DEFINE_Validator(paimon_file_system_scheme_mappings,
DEFINE_mInt32(thrift_client_open_num_tries, "1");
-// http scheme in S3Client to use. E.g. http or https
-DEFINE_String(s3_client_http_scheme, "http");
+// Default HTTP scheme used by S3Client when the endpoint has no scheme.
+DEFINE_String(s3_client_http_scheme, "https");
DEFINE_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
return config == "http" || config == "https";
});
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 9954f3b3e41..699bbdbee47 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1714,7 +1714,7 @@ DECLARE_Strings(paimon_file_system_scheme_mappings);
// Retry the Open num_retries time waiting 100 milliseconds between retries.
DECLARE_mInt32(thrift_client_open_num_tries);
-// http scheme in S3Client to use. E.g. http or https
+// Default HTTP scheme used by S3Client when the endpoint has no scheme.
DECLARE_String(s3_client_http_scheme);
DECLARE_mBool(ignore_schema_change_check);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 1e03308fae4..fe9af8f582e 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -446,9 +446,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_s3_client(
aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms;
}
- if (config::s3_client_http_scheme == "http") {
- aws_config.scheme = Aws::Http::Scheme::HTTP;
- }
+ set_s3_client_default_http_scheme(aws_config,
config::s3_client_http_scheme);
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry /*scaleFactor = 25*/,
/*retry_slow_down=*/true);
diff --git a/be/test/io/s3_client_factory_test.cpp
b/be/test/io/s3_client_factory_test.cpp
index de8149380cb..53787e7150e 100644
--- a/be/test/io/s3_client_factory_test.cpp
+++ b/be/test/io/s3_client_factory_test.cpp
@@ -17,6 +17,7 @@
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/auth/STSCredentialsProvider.h>
+#include <aws/core/client/ClientConfiguration.h>
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
#include <gtest/gtest.h>
@@ -26,6 +27,7 @@
#include <vector>
#include "cloud/config.h"
+#include "cpp/aws_common.h"
#include "cpp/custom_aws_credentials_provider_chain.h"
#include "io/fs/rate_limited_obj_storage_client.h"
#include "io/fs/s3_obj_storage_client.h"
@@ -232,6 +234,32 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) {
config::aws_credentials_provider_version = "v2";
}
+TEST_F(S3ClientFactoryTest, SetS3ClientDefaultHttpScheme) {
+ S3ClientFactory::instance();
+ Aws::Client::ClientConfiguration client_config;
+ client_config.endpointOverride = "example.com:9000";
+
+ set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+
+ client_config.endpointOverride = "http://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTP;
+ set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "http://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ client_config.endpointOverride = "https://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTPS;
+ set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "https://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+}
+
TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRoleArnProviderType) {
std::map<std::string, std::string> properties {
{"AWS_ENDPOINT", "s3.us-west-2.amazonaws.com"},
@@ -244,6 +272,7 @@ TEST_F(S3ClientFactoryTest,
ConvertPropertiesToS3ConfRoleArnProviderType) {
S3Conf s3_conf;
ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties,
s3_uri, &s3_conf).ok());
+ ASSERT_EQ(s3_conf.client_conf.endpoint, properties.at("AWS_ENDPOINT"));
ASSERT_EQ(s3_conf.client_conf.cred_provider_type,
CredProviderType::Default);
properties["AWS_CREDENTIALS_PROVIDER_TYPE"] = "WEB_IDENTITY";
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 37069faa706..070cdd4e44e 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -327,8 +327,8 @@ CONF_String(priority_networks, "");
CONF_Bool(enable_cluster_name_check, "false");
-// http scheme in S3Client to use. E.g. http or https
-CONF_String(s3_client_http_scheme, "http");
+// Default HTTP scheme used by S3Client when the endpoint has no scheme.
+CONF_String(s3_client_http_scheme, "https");
CONF_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
return config == "http" || config == "https";
});
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 7c4fde97041..0c3713d92d1 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -428,9 +428,7 @@ int S3Accessor::init() {
config::instance_recycler_worker_pool_size),
(long)aws_config.maxConnections);
- if (config::s3_client_http_scheme == "http") {
- aws_config.scheme = Aws::Http::Scheme::HTTP;
- }
+ set_s3_client_default_http_scheme(aws_config,
config::s3_client_http_scheme);
// Recycler should fail fast on S3 SlowDown instead of retrying and
blocking worker threads.
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry, /*retry_slow_down=*/false);
diff --git a/cloud/test/util_test.cpp b/cloud/test/util_test.cpp
index 435feb6e0c3..3ef53ad358f 100644
--- a/cloud/test/util_test.cpp
+++ b/cloud/test/util_test.cpp
@@ -17,6 +17,8 @@
#include "cpp/util.h"
+#include <aws/core/client/ClientConfiguration.h>
+
#include <chrono>
#include <stdexcept>
#include <string>
@@ -33,6 +35,7 @@
#include "cpp/sync_point.h"
#include "gtest/gtest.h"
#include "recycler/recycler.h"
+#include "recycler/s3_accessor.h"
#include "recycler/sync_executor.h"
using namespace doris::cloud;
@@ -438,6 +441,33 @@ TEST(UtilTest, test_normalize_http_uri) {
"https://example.com/path?query=value#fragment";);
}
+TEST(UtilTest, test_set_s3_client_default_http_scheme) {
+ doris::cloud::S3Environment::getInstance();
+ Aws::Client::ClientConfiguration client_config =
+ doris::cloud::S3Environment::getClientConfiguration();
+
+ client_config.endpointOverride = "example.com:9000";
+ doris::set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ doris::set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+
+ client_config.endpointOverride = "http://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTP;
+ doris::set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "http://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ client_config.endpointOverride = "https://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTPS;
+ doris::set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "https://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+}
+
TEST(UtilTest, test_long_normalize_http_uri) {
std::string longPath = "https://example.com";;
for (int i = 0; i < 100; i++) {
diff --git a/common/cpp/aws_common.cpp b/common/cpp/aws_common.cpp
index 3c7f5a0eda5..7f11d8cb95e 100644
--- a/common/cpp/aws_common.cpp
+++ b/common/cpp/aws_common.cpp
@@ -17,6 +17,7 @@
#include "aws_common.h"
+#include <aws/core/client/ClientConfiguration.h>
#include <glog/logging.h>
namespace doris {
@@ -84,4 +85,13 @@ std::string get_valid_ca_cert_path(const
std::vector<std::string>& ca_cert_file_
}
return "";
}
+
+void set_s3_client_default_http_scheme(Aws::Client::ClientConfiguration&
client_config,
+ const std::string& scheme) {
+ if (client_config.endpointOverride.starts_with("http://";) ||
+ client_config.endpointOverride.starts_with("https://";)) {
+ return;
+ }
+ client_config.scheme = scheme == "http" ? Aws::Http::Scheme::HTTP :
Aws::Http::Scheme::HTTPS;
}
+} // namespace doris
diff --git a/common/cpp/aws_common.h b/common/cpp/aws_common.h
index cc48e23c2f9..d977dd245b0 100644
--- a/common/cpp/aws_common.h
+++ b/common/cpp/aws_common.h
@@ -21,23 +21,31 @@
#include <filesystem>
+namespace Aws::Client {
+struct ClientConfiguration;
+}
+
namespace doris {
- //AWS Credentials Provider Type
- enum class CredProviderType {
- Default = 0,
- Simple = 1,
- InstanceProfile = 2,
- Env = 3,
- SystemProperties = 4,
- WebIdentity = 5,
- Container = 6,
- Anonymous = 7
- };
+//AWS Credentials Provider Type
+enum class CredProviderType {
+ Default = 0,
+ Simple = 1,
+ InstanceProfile = 2,
+ Env = 3,
+ SystemProperties = 4,
+ WebIdentity = 5,
+ Container = 6,
+ Anonymous = 7
+};
+
+CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB
cred_provider_type);
- CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB
cred_provider_type);
+CredProviderType cred_provider_type_from_string(const std::string& type);
- CredProviderType cred_provider_type_from_string(const std::string& type);
+std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths);
- std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths);
+// Configures the default S3 client transport scheme for endpoints without an
explicit scheme.
+void set_s3_client_default_http_scheme(Aws::Client::ClientConfiguration&
client_config,
+ const std::string& scheme);
- } // namespace doris
\ No newline at end of file
+} // namespace doris
\ No newline at end of file
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 133ee2114f7..765baf5393d 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3278,6 +3278,11 @@ public class Config extends ConfigBase {
+ "it is intentionally not modifiable at runtime via ADMIN
SET FRONTEND CONFIG."})
public static String[] s3_load_endpoint_white_list = {};
+ @ConfField(description = {
+ "The default scheme for S3 endpoints without an explicit scheme.
Valid values are http and https."},
+ options = {"http", "https"})
+ public static String s3_client_http_scheme = "https";
+
@ConfField(mutable = true, description = {
"For deterministic S3 paths (without wildcards like *, ?), use
HEAD requests instead of "
+ "ListObjects to avoid requiring ListBucket permission.
Brace patterns {1,2,3} and "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index fce8d09a837..0c70cd67ce4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -21,6 +21,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.common.util.DatasourcePrintableMap;
+import org.apache.doris.common.util.S3Util;
import org.apache.doris.datasource.storage.S3ResourceCompat;
import org.apache.doris.filesystem.UploadPartResult;
import org.apache.doris.filesystem.spi.ObjFileSystem;
@@ -102,20 +103,19 @@ public class S3Resource extends Resource {
LOG.debug("s3 info need check validity : {}", needCheck);
}
- // the endpoint for ping need add uri scheme.
- String pingEndpoint = properties.get(S3ResourceCompat.ENDPOINT);
- if (!pingEndpoint.startsWith("http://";) &&
!pingEndpoint.startsWith("https://";)) {
- pingEndpoint = "http://"; +
properties.get(S3ResourceCompat.ENDPOINT);
- properties.put(S3ResourceCompat.ENDPOINT, pingEndpoint);
- properties.put(S3ResourceCompat.Env.ENDPOINT, pingEndpoint);
- }
- String region = S3ResourceCompat.getRegionOfEndpoint(pingEndpoint);
+ String endpoint = properties.get(S3ResourceCompat.ENDPOINT);
+ properties.put(S3ResourceCompat.Env.ENDPOINT, endpoint);
+ String region = S3ResourceCompat.getRegionOfEndpoint(endpoint);
properties.putIfAbsent(S3ResourceCompat.REGION, region);
if (needCheck) {
+ Map<String, String> pingProperties = new HashMap<>(properties);
+ String pingEndpoint = S3Util.buildEndpointUrl(endpoint);
+ pingProperties.put(S3ResourceCompat.ENDPOINT, pingEndpoint);
+ pingProperties.put(S3ResourceCompat.Env.ENDPOINT, pingEndpoint);
String bucketName = properties.get(S3ResourceCompat.BUCKET);
String rootPath = properties.get(S3ResourceCompat.ROOT_PATH);
- pingS3(bucketName, rootPath, properties);
+ pingS3(bucketName, rootPath, pingProperties);
}
// optional
S3ResourceCompat.optionalS3Property(properties);
@@ -233,6 +233,9 @@ public class S3Resource extends Resource {
}
// compatible with old version, Need convert if modified properties
map uses old properties.
S3ResourceCompat.convertToStdProperties(properties);
+ if (!Strings.isNullOrEmpty(properties.get(S3ResourceCompat.ENDPOINT)))
{
+ properties.put(S3ResourceCompat.Env.ENDPOINT,
properties.get(S3ResourceCompat.ENDPOINT));
+ }
boolean needCheck = isNeedCheck(properties);
if (LOG.isDebugEnabled()) {
LOG.debug("s3 info need check validity : {}", needCheck);
@@ -241,6 +244,9 @@ public class S3Resource extends Resource {
S3ResourceCompat.requiredS3PingProperties(this.properties);
Map<String, String> changedProperties = new
HashMap<>(this.properties);
changedProperties.putAll(properties);
+ String endpoint =
S3Util.buildEndpointUrl(changedProperties.get(S3ResourceCompat.ENDPOINT));
+ changedProperties.put(S3ResourceCompat.ENDPOINT, endpoint);
+ changedProperties.put(S3ResourceCompat.Env.ENDPOINT, endpoint);
String bucketName =
properties.getOrDefault(S3ResourceCompat.BUCKET,
this.properties.get(S3ResourceCompat.BUCKET));
String rootPath =
properties.getOrDefault(S3ResourceCompat.ROOT_PATH,
@@ -289,10 +295,9 @@ public class S3Resource extends Resource {
this.properties.get(S3ResourceCompat.SESSION_TOKEN));
String endpoint = properties.getOrDefault(S3ResourceCompat.ENDPOINT,
this.properties.get(S3ResourceCompat.ENDPOINT));
- String pingEndpoint = "http://"; + endpoint;
- String region = S3ResourceCompat.getRegionOfEndpoint(pingEndpoint);
+ String region = S3ResourceCompat.getRegionOfEndpoint(endpoint);
properties.putIfAbsent(S3ResourceCompat.REGION, region);
- return new CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk,
token);
+ return new CloudCredentialWithEndpoint(endpoint, region, ak, sk,
token);
}
private boolean isNeedCheck(Map<String, String> newProperties) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index 3e4f4e7a62f..f6c2fcf9c7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -89,7 +89,7 @@ public class S3Util {
}
@Deprecated
- public static S3Client buildS3Client(URI endpoint, String region,
CloudCredential credential,
+ public static S3Client buildS3Client(String endpoint, String region,
CloudCredential credential,
boolean isUsePathStyle) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
@@ -112,7 +112,7 @@ public class S3Util {
return S3Client.builder()
.httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
.connectionTimeout(Duration.ofSeconds(30)).build())
- .endpointOverride(endpoint)
+ .endpointOverride(URI.create(buildEndpointUrl(endpoint)))
.credentialsProvider(getAwsCredencialsProvider(credential))
.region(Region.of(region))
.overrideConfiguration(clientConf)
@@ -215,7 +215,7 @@ public class S3Util {
sessionToken, roleArn, externalId);
}
- public static S3Client buildS3Client(URI endpoint, String region, boolean
isUsePathStyle,
+ public static S3Client buildS3Client(String endpoint, String region,
boolean isUsePathStyle,
AwsCredentialsProvider credential) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
@@ -238,7 +238,7 @@ public class S3Util {
return S3Client.builder()
.httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
.connectionTimeout(Duration.ofSeconds(30)).build())
- .endpointOverride(endpoint)
+ .endpointOverride(URI.create(buildEndpointUrl(endpoint)))
.credentialsProvider(credential)
.region(Region.of(region))
.overrideConfiguration(clientConf)
@@ -250,7 +250,7 @@ public class S3Util {
.build();
}
- public static S3Client buildS3Client(URI endpoint, String region, boolean
isUsePathStyle, String accessKey,
+ public static S3Client buildS3Client(String endpoint, String region,
boolean isUsePathStyle, String accessKey,
String secretKey, String sessionToken, String roleArn, String
externalId) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
@@ -270,11 +270,12 @@ public class S3Util {
// using AwsS3V4Signer
.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
AwsS3V4Signer.create())
.build();
+ URI endpointUri = URI.create(buildEndpointUrl(endpoint));
return S3Client.builder()
.httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
.connectionTimeout(Duration.ofSeconds(30)).build())
- .endpointOverride(endpoint)
- .credentialsProvider(getAwsCredencialsProvider(endpoint,
region, accessKey, secretKey,
+ .endpointOverride(endpointUri)
+ .credentialsProvider(getAwsCredencialsProvider(endpointUri,
region, accessKey, secretKey,
sessionToken, roleArn, externalId))
.region(Region.of(region))
.overrideConfiguration(clientConf)
@@ -434,6 +435,13 @@ public class S3Util {
}
}
+ public static String buildEndpointUrl(String endpoint) {
+ if (endpoint.startsWith("http://";) || endpoint.startsWith("https://";))
{
+ return endpoint;
+ }
+ return Config.s3_client_http_scheme + "://" + endpoint;
+ }
+
/**
* Check if a path pattern is deterministic, meaning all file paths can be
determined
* without listing. A pattern is deterministic if it contains no true
wildcard characters
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourcePersistParityTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourcePersistParityTest.java
index a6f62f0a904..68ea4046de4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourcePersistParityTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourcePersistParityTest.java
@@ -53,9 +53,9 @@ public class S3ResourcePersistParityTest {
resource.setProperties(ImmutableMap.copyOf(input));
Map<String, String> golden = new HashMap<>();
- // Dual-namespace write: the ping endpoint gets an http:// prefix and
lands under BOTH keys.
- golden.put("s3.endpoint", "http://s3.us-east-1.amazonaws.com";);
- golden.put("AWS_ENDPOINT", "http://s3.us-east-1.amazonaws.com";);
+ // Dual-namespace write preserves the user-provided endpoint under
BOTH keys.
+ golden.put("s3.endpoint", "s3.us-east-1.amazonaws.com");
+ golden.put("AWS_ENDPOINT", "s3.us-east-1.amazonaws.com");
// Region derived from the endpoint host (second dot-segment).
golden.put("s3.region", "us-east-1");
golden.put("s3.access_key", "myAk");
@@ -85,10 +85,9 @@ public class S3ResourcePersistParityTest {
resource.setProperties(ImmutableMap.copyOf(input));
Map<String, String> golden = new HashMap<>();
- // convertToStdProperties mirrors legacy AWS_* keys into the s3.*
namespace first;
- // the endpoint then gets the http:// prefix under both keys.
- golden.put("AWS_ENDPOINT", "http://s3.us-east-1.amazonaws.com";);
- golden.put("s3.endpoint", "http://s3.us-east-1.amazonaws.com";);
+ // convertToStdProperties mirrors the original legacy endpoint into
the s3.* namespace.
+ golden.put("AWS_ENDPOINT", "s3.us-east-1.amazonaws.com");
+ golden.put("s3.endpoint", "s3.us-east-1.amazonaws.com");
golden.put("s3.region", "us-east-1");
golden.put("AWS_ACCESS_KEY", "myAk");
golden.put("s3.access_key", "myAk");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
index ac2bf9fc1ff..acdd7dedec9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.storage.CloudObjectStoreAdapter;
import org.apache.doris.datasource.storage.S3ResourceCompat;
import org.apache.doris.meta.MetaContext;
import org.apache.doris.mysql.privilege.AccessControllerManager;
@@ -194,7 +195,9 @@ public class S3ResourceTest {
Assert.assertEquals("s3_1", rS3Resource1.getName());
Assert.assertEquals("s3_2", rS3Resource2.getName());
-
Assert.assertEquals(rS3Resource2.getProperty(S3ResourceCompat.ENDPOINT),
"http://aaa";);
+ Assert.assertEquals("aaa",
rS3Resource2.getProperty(S3ResourceCompat.ENDPOINT));
+ Assert.assertEquals("aaa",
+
CloudObjectStoreAdapter.getObjStoreInfoPB(rS3Resource2.getCopiedProperties()).getEndpoint());
Assert.assertEquals(rS3Resource2.getProperty(S3ResourceCompat.REGION),
"bbb");
Assert.assertEquals(rS3Resource2.getProperty(S3ResourceCompat.ROOT_PATH),
"/path/to/root");
Assert.assertEquals(rS3Resource2.getProperty(S3ResourceCompat.ACCESS_KEY),
"xxx");
@@ -226,11 +229,24 @@ public class S3ResourceTest {
Map<String, String> modify = new HashMap<>();
modify.put("s3.access_key", "aaa");
s3Resource.modifyProperties(modify);
+
+ modify.clear();
+ modify.put(S3ResourceCompat.ENDPOINT, "new-endpoint");
+ s3Resource.modifyProperties(modify);
+ Assert.assertEquals("new-endpoint",
s3Resource.getProperty(S3ResourceCompat.ENDPOINT));
+ Assert.assertEquals("new-endpoint",
s3Resource.getProperty(S3ResourceCompat.Env.ENDPOINT));
+ Assert.assertEquals("new-endpoint",
+
CloudObjectStoreAdapter.getObjStoreInfoPB(s3Resource.getCopiedProperties()).getEndpoint());
+
+ modify.clear();
+ modify.put(S3ResourceCompat.Env.ENDPOINT, "http://other-endpoint";);
+ s3Resource.modifyProperties(modify);
+ Assert.assertEquals("http://other-endpoint";,
s3Resource.getProperty(S3ResourceCompat.ENDPOINT));
+ Assert.assertEquals("http://other-endpoint";,
s3Resource.getProperty(S3ResourceCompat.Env.ENDPOINT));
}
@Test
- public void testHttpScheme() throws DdlException {
- // if https:// is set, it should be replaced with http://
+ public void testExplicitSchemeIsPreserved() throws DdlException {
ImmutableMap<String, String> properties = ImmutableMap.of(
"AWS_ENDPOINT", "https://aaa";,
"AWS_REGION", "bbb",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
index 4b976ed86cd..ff155a0b1f8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
@@ -17,13 +17,70 @@
package org.apache.doris.common.util;
+import org.apache.doris.common.Config;
+
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3Client;
+import java.net.URI;
import java.util.Arrays;
import java.util.List;
public class S3UtilTest {
+ private String originalS3ClientHttpScheme;
+
+ @Before
+ public void setUp() {
+ originalS3ClientHttpScheme = Config.s3_client_http_scheme;
+ }
+
+ @After
+ public void tearDown() {
+ Config.s3_client_http_scheme = originalS3ClientHttpScheme;
+ }
+
+ @Test
+ public void testBuildEndpointUrlDefaultsToHttps() {
+ Config.s3_client_http_scheme = "https";
+ Assert.assertEquals("https://s3.us-east-1.amazonaws.com";,
+ S3Util.buildEndpointUrl("s3.us-east-1.amazonaws.com"));
+ }
+
+ @Test
+ public void testBuildEndpointUrlUsesConfiguredHttpScheme() {
+ Config.s3_client_http_scheme = "http";
+ Assert.assertEquals("http://127.0.0.1:9000";,
+ S3Util.buildEndpointUrl("127.0.0.1:9000"));
+ }
+
+ @Test
+ public void testBuildEndpointUrlPreservesExplicitSchemes() {
+ Config.s3_client_http_scheme = "https";
+ Assert.assertEquals("http://127.0.0.1:9000";,
+ S3Util.buildEndpointUrl("http://127.0.0.1:9000";));
+
+ Config.s3_client_http_scheme = "http";
+ Assert.assertEquals("https://s3.us-east-1.amazonaws.com";,
+ S3Util.buildEndpointUrl("https://s3.us-east-1.amazonaws.com";));
+ }
+
+ @Test
+ public void testBuildS3ClientAppliesDefaultSchemeAtClientCreation() {
+ Config.s3_client_http_scheme = "https";
+ try (S3Client client = S3Util.buildS3Client(
+ "127.0.0.1:9000",
+ "us-east-1",
+ true,
+
StaticCredentialsProvider.create(AwsBasicCredentials.create("ak", "sk")))) {
+ Assert.assertEquals(URI.create("https://127.0.0.1:9000";),
+
client.serviceClientConfiguration().endpointOverride().orElseThrow());
+ }
+ }
@Test
public void testExtendGlobNumberRange_simpleRange() {
@@ -457,4 +514,3 @@ public class S3UtilTest {
Assert.assertEquals("file[abc.csv",
S3Util.expandBracketPatterns("file[abc.csv"));
}
}
-
diff --git a/regression-test/pipeline/vault_p0/conf/be_custom.conf
b/regression-test/pipeline/vault_p0/conf/be_custom.conf
index 779eb5a77a5..3a2a0e16213 100644
--- a/regression-test/pipeline/vault_p0/conf/be_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/be_custom.conf
@@ -11,6 +11,7 @@ enable_file_cache=true
enable_file_cache_query_limit=true
file_cache_max_file_segment_size=1048576
s3_write_buffer_whole_size=52428800
+s3_client_http_scheme = http
enable_vertical_compaction=true
fuzzy_vertical_compaction=true
vacuum_stale_rowsets_interval_seconds=60
diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf
b/regression-test/pipeline/vault_p0/conf/fe_custom.conf
index f62ffa19fc7..9b622795168 100644
--- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf
@@ -12,6 +12,7 @@ enable_mtmv = true
remote_fragment_exec_timeout_ms=60000
dynamic_partition_check_interval_seconds=10
use_fuzzy_session_variable=true
+s3_client_http_scheme = http
enable_cloud_snapshot_version = true
enable_auto_collect_statistics = false
diff --git a/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
b/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
index 62deff2b870..3c3dc627122 100644
--- a/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
@@ -1,2 +1,3 @@
# below lines will be appended to the default doris_cloud.conf when deploying
recycler
brpc_listen_port = 6000
+s3_client_http_scheme = http
diff --git a/regression-test/suites/cold_heat_separation/policy/alter.groovy
b/regression-test/suites/cold_heat_separation/policy/alter.groovy
index cfd8e0da6fc..22903e672ba 100644
--- a/regression-test/suites/cold_heat_separation/policy/alter.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/alter.groovy
@@ -174,7 +174,7 @@ suite("alter_policy") {
// AWS_CONNECTION_TIMEOUT_MS
assertEquals(show_alter_result[5][3], "2222")
// AWS_ENDPOINT
- assertEquals(show_alter_result[6][3], "http://bj.s3.comaaaa";)
+ assertEquals(show_alter_result[6][3], "bj.s3.comaaaa")
// AWS_REGION
assertEquals(show_alter_result[7][3], "bj")
// s3_rootpath
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
