github-actions[bot] commented on code in PR #70443:
URL: https://github.com/apache/doris/pull/70443#discussion_r4599556065


##########
be/test/io/object_storage/obj_storage_client_factory_test.cpp:
##########
@@ -0,0 +1,1886 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-storage/client/obj_storage_client_factory.h"
+
+#include <aws/core/auth/AWSCredentialsProviderChain.h>
+#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
+#include <aws/core/auth/STSCredentialsProvider.h>
+#include <aws/core/auth/signer/AWSAuthV4Signer.h>
+#include <aws/core/client/ClientConfiguration.h>
+#include <aws/core/http/HttpClientFactory.h>
+#include <aws/core/utils/stream/ResponseStream.h>
+#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/AbortMultipartUploadRequest.h>
+#include <aws/s3/model/AbortMultipartUploadResult.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadResult.h>
+#include <aws/s3/model/CreateMultipartUploadRequest.h>
+#include <aws/s3/model/CreateMultipartUploadResult.h>
+#include <aws/s3/model/DeleteObjectRequest.h>
+#include <aws/s3/model/DeleteObjectResult.h>
+#include <aws/s3/model/DeleteObjectsRequest.h>
+#include <aws/s3/model/DeleteObjectsResult.h>
+#include <aws/s3/model/GetBucketLifecycleConfigurationRequest.h>
+#include <aws/s3/model/GetBucketLifecycleConfigurationResult.h>
+#include <aws/s3/model/GetBucketVersioningRequest.h>
+#include <aws/s3/model/GetBucketVersioningResult.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/GetObjectResult.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/HeadObjectResult.h>
+#include <aws/s3/model/ListObjectsV2Request.h>
+#include <aws/s3/model/ListObjectsV2Result.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/PutObjectResult.h>
+#include <aws/s3/model/UploadPartRequest.h>
+#include <aws/s3/model/UploadPartResult.h>
+#include <gen_cpp/AgentService_types.h>
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <typeinfo>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "cloud/config.h"
+#include "common/config.h"
+#include "cpp/custom_aws_credentials_provider_chain.h"
+#include "cpp/obj-storage/auth/aws/aws_credential_factory.h"
+#include "cpp/obj-storage/auth/ca_bundle_cache.h"
+#include "cpp/obj-storage/auth/gcp/gcp_auth.h"
+#include "cpp/obj-storage/auth/gcp/gcp_oauth_s3_client.h"
+#include "cpp/obj-storage/auth/gcp/gcs_signed_url.h"
+#include "cpp/obj-storage/client/bos_obj_storage_client.h"
+#include "cpp/obj-storage/client/cos_obj_storage_client.h"
+#include "cpp/obj-storage/client/gcp_obj_storage_client.h"
+#include "cpp/obj-storage/client/obs_obj_storage_client.h"
+#include "cpp/obj-storage/client/oss_obj_storage_client.h"
+#include "cpp/obj-storage/client/s3_obj_storage_client.h"
+#include "cpp/obj-storage/client/s3_util.h"
+#include "cpp/obj-storage/client/tos_obj_storage_client.h"
+#include "cpp/obj-storage/parser/obj_storage_config_parser.h"
+#include "cpp/obj-storage/rate_limit/rate_limited_obj_storage_client.h"
+#include "cpp/sync_point.h"
+#include "io/fs/err_utils.h"
+#include "io/fs/obj_storage_file_system.h"
+#include "io/object_storage/be_obj_storage_config_parser.h"
+#include "io/object_storage/obj_storage_client_holder.h"
+#include "io/object_storage/obj_storage_rate_limiter_manager.h"
+#include "testutil/container_credentials_endpoint.h"
+#include "util/s3_uri.h"
+
+#ifdef USE_AZURE
+#include <azure/core/http/curl_transport.hpp>
+
+#include "cpp/obj-storage/client/azure_obj_storage_client.h"
+#endif
+
+namespace doris {
+
+class S3ClientFactoryTest : public testing::Test {
+protected:
+    using Properties = std::map<std::string, std::string>;
+
+    static AwsCredentialResult create_credentials_provider(
+            Properties properties, const std::string& ca_path = 
be_ca_bundle_path()) {
+        properties.emplace("AWS_ENDPOINT", 
"https://s3.us-east-1.amazonaws.com";);
+        properties.emplace("AWS_REGION", "us-east-1");
+        S3URI uri("s3://credentials/test");
+        if (auto status = uri.parse(); !status.ok()) {
+            return {.error = status.to_string()};
+        }
+        auto parsed = ObjStorageConfigParser::from_properties(
+                properties, uri.get_bucket(),
+                
aws_credential_implementation(config::aws_credentials_provider_version),
+                EmptyCredentialsBehavior::ANONYMOUS);
+        if (auto* error = std::get_if<std::string>(&parsed)) {
+            return {.error = *error};
+        }
+        auto resource = std::get<ObjStorageResourceConfig>(std::move(parsed));
+        apply_be_obj_storage_defaults(resource.client);
+        auto& config = std::get<AwsObjStorageConfig>(resource.client);
+        config.connection.ca_cert_file = ca_path;
+        return AwsCredentialFactory::create(config.auth, config.connection);
+    }
+
+    std::string previous_version = config::aws_credentials_provider_version;
+
+    void TearDown() override {
+        config::aws_credentials_provider_version = previous_version;
+        ObjStorageClientFactory::instance().clear_client_creator_for_test();
+    }
+};
+
+namespace {
+
+constexpr size_t kNoThrottleBytesPerSecond = 1ULL << 40;
+
+AwsObjStorageConfig make_factory_conf(std::string endpoint) {
+    return {.connection = {.endpoint = std::move(endpoint)}, .region = 
"us-east-1"};
+}
+
+AwsObjStorageConfig make_timeout_conf(std::string endpoint, bool swapped) {
+    auto config = make_factory_conf(std::move(endpoint));
+    config.connection.connect_timeout = std::chrono::milliseconds(swapped ? 
200 : 100);
+    config.connection.request_timeout = std::chrono::milliseconds(swapped ? 
100 : 200);
+    return config;
+}
+
+template <typename Client, typename Config>
+void check_s3_provider_client() {
+    Config config;
+    config.connection.endpoint = "https://objects.example.com";;
+    config.region = "us-east-1";
+    auto result = ObjStorageClientFactory::instance().create(config);
+    ASSERT_NE(result.client, nullptr) << result.error;
+    auto* client = result.client.get();
+    EXPECT_EQ(typeid(*client), typeid(Client));
+}
+
+class CloudModeConfigGuard {
+public:
+    explicit CloudModeConfigGuard(bool cloud_mode)
+            : _deploy_mode(config::deploy_mode), 
_cloud_unique_id(config::cloud_unique_id) {
+        config::deploy_mode = cloud_mode ? "cloud" : "";
+        config::cloud_unique_id.clear();
+    }
+
+    ~CloudModeConfigGuard() {
+        config::deploy_mode = _deploy_mode;
+        config::cloud_unique_id = _cloud_unique_id;
+    }
+
+private:
+    std::string _deploy_mode;
+    std::string _cloud_unique_id;
+};
+
+class RateLimiterConfigGuard {
+public:
+    RateLimiterConfigGuard()
+            : _enabled(config::enable_s3_rate_limiter),
+              _qps_max_speed(_qps()->get_max_speed()),
+              _qps_max_burst(_qps()->get_max_burst()),
+              _qps_limit(_qps()->get_limit()),
+              _bytes_max_speed(_bytes()->get_max_speed()),
+              _bytes_max_burst(_bytes()->get_max_burst()),
+              _bytes_limit(_bytes()->get_limit()) {}
+
+    ~RateLimiterConfigGuard() {
+        config::enable_s3_rate_limiter = _enabled;
+        _qps()->reset(_qps_max_speed, _qps_max_burst, _qps_limit);
+        _bytes()->reset(_bytes_max_speed, _bytes_max_burst, _bytes_limit);
+    }
+
+private:
+    static S3RateLimiterHolder* _qps() {
+        return 
ObjStorageRateLimiterManager::instance().qps_limiter(ObjStorageRateLimitType::GET);
+    }
+    static S3RateLimiterHolder* _bytes() {
+        return 
ObjStorageRateLimiterManager::instance().bytes_limiter(ObjStorageRateLimitType::GET);
+    }
+
+    bool _enabled;
+    size_t _qps_max_speed;
+    size_t _qps_max_burst;
+    size_t _qps_limit;
+    size_t _bytes_max_speed;
+    size_t _bytes_max_burst;
+    size_t _bytes_limit;
+};
+
+GcpObjStorageConfig make_gcp_native_conf(std::optional<std::string> account = 
std::nullopt) {
+    return {.connection = {.endpoint = "https://storage.googleapis.com"},
+            .auth = GcpNativeObjStorageAuth {.impersonation_service_account = 
std::move(account)},
+            .region = "us-east1"};
+}
+
+class FixedTokenGcpOAuthS3Client final : public GcpOAuthS3Client {
+public:
+    explicit FixedTokenGcpOAuthS3Client(std::string endpoint = 
"https://storage.googleapis.com";)
+            : GcpOAuthS3Client(GcpCredentialConfig {}, "", 
make_config(endpoint),
+                               
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true) {}
+
+    using GcpOAuthS3Client::BuildHttpRequest;
+    using GcpOAuthS3Client::authorize_request;
+
+    std::optional<std::string> token = "test-gcp-token";
+    mutable int token_requests = 0;
+
+protected:
+    std::optional<std::string> fetch_token() const override {
+        ++token_requests;
+        return token;
+    }
+
+private:
+    static Aws::Client::ClientConfiguration make_config(const std::string& 
endpoint) {
+        Aws::Client::ClientConfiguration config;
+        config.endpointOverride = endpoint;
+        return config;
+    }
+};
+
+class SyncPointProcessingGuard {
+public:
+    SyncPointProcessingGuard() : 
_was_enabled(SyncPoint::get_instance()->get_enable()) {
+        SyncPoint::get_instance()->enable_processing();
+    }
+    ~SyncPointProcessingGuard() {
+        if (!_was_enabled) {
+            SyncPoint::get_instance()->disable_processing();
+        }
+    }
+
+private:
+    bool _was_enabled;
+};
+
+} // namespace
+
+TEST_F(S3ClientFactoryTest, CreatesClientForEachS3CompatibleProvider) {
+    check_s3_provider_client<S3ObjStorageClient, AwsObjStorageConfig>();
+    check_s3_provider_client<OssObjStorageClient, OssObjStorageConfig>();
+    check_s3_provider_client<CosObjStorageClient, CosObjStorageConfig>();
+    check_s3_provider_client<BosObjStorageClient, BosObjStorageConfig>();
+    check_s3_provider_client<ObsObjStorageClient, ObsObjStorageConfig>();
+    check_s3_provider_client<GcpObjStorageClient, GcpObjStorageConfig>();
+    check_s3_provider_client<TosObjStorageClient, TosObjStorageConfig>();
+}
+
+TEST_F(S3ClientFactoryTest, CreatesS3CompatibleClientsFromProperties) {
+    SyncPointProcessingGuard sync_point_guard;
+    SyncPoint::CallbackGuard head_object_callback;
+    SyncPoint::get_instance()->set_call_back(
+            "s3_file_system::head_object",
+            [](auto&& args) {
+                auto result = 
try_any_cast_ret<Aws::S3::Model::HeadObjectOutcome>(args);
+                Aws::S3::Model::HeadObjectResult head;
+                head.SetContentLength(37);
+                result->first = 
Aws::S3::Model::HeadObjectOutcome(std::move(head));
+                result->second = true;
+            },
+            &head_object_callback);
+
+    S3URI uri("s3://bucket/key");
+    ASSERT_TRUE(uri.parse().ok());
+    const struct {
+        const char* provider;
+        const char* endpoint;
+        const std::type_info& expected_client;
+    } cases[] = {
+            {"OSS", "objects.example.com", typeid(OssObjStorageClient)},
+            {"GCP", "storage.googleapis.com", typeid(GcpObjStorageClient)},
+            {nullptr, "oss-cn-hangzhou.aliyuncs.com", 
typeid(S3ObjStorageClient)},
+            {" gCs ", "oss-cn-hangzhou.aliyuncs.com", 
typeid(GcpObjStorageClient)},
+    };
+    for (const auto& entry : cases) {
+        SCOPED_TRACE(entry.provider ? entry.provider : "default S3");
+        std::map<std::string, std::string> props {{"AWS_ENDPOINT", 
entry.endpoint},
+                                                  {"AWS_REGION", "region"},
+                                                  {"AWS_ACCESS_KEY", 
"test-key"},
+                                                  {"AWS_SECRET_KEY", 
"test-secret"}};
+        if (entry.provider) {
+            props["provider"] = entry.provider;
+        }
+        auto parsed_result = ObjStorageConfigParser::from_properties(
+                props, uri.get_bucket(),
+                
aws_credential_implementation(config::aws_credentials_provider_version),
+                EmptyCredentialsBehavior::ANONYMOUS);
+        
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(parsed_result))
+                << std::get<std::string>(parsed_result);
+        auto parsed = 
std::get<ObjStorageResourceConfig>(std::move(parsed_result));
+        apply_be_obj_storage_defaults(parsed.client);
+        auto result =
+                ObjStorageClientFactory::instance().create(parsed.client, 
true, parsed.bucket);
+        ASSERT_NE(result.client, nullptr) << result.error;
+        const auto* client = result.client.get();
+        EXPECT_EQ(typeid(*client), entry.expected_client);
+        const auto head =
+                result.client->head_object({.bucket = uri.get_bucket(), .key = 
uri.get_key()});
+        ASSERT_TRUE(head.resp.ok());
+        EXPECT_EQ(37, head.file_size);
+    }
+}
+
+TEST_F(S3ClientFactoryTest, LegacyS3TimeoutsPreserveSdkDefaults) {
+    const auto defaults = be_default_aws_client_configuration();
+    for (int timeout_ms : {-1, 0, 5000}) {
+        SCOPED_TRACE(timeout_ms);
+        Properties properties {{"AWS_ENDPOINT", "https://endpoint"},
+                               {"AWS_REGION", "region"},
+                               {"AWS_REQUEST_TIMEOUT_MS", 
std::to_string(timeout_ms)},
+                               {"AWS_CONNECTION_TIMEOUT_MS", 
std::to_string(timeout_ms)}};
+        TS3StorageParam param;
+        param.__set_provider(TObjStorageType::AWS);
+        param.__set_bucket("bucket");
+        param.__set_endpoint("https://endpoint";);
+        param.__set_region("region");
+        param.__set_request_timeout_ms(timeout_ms);
+        param.__set_conn_timeout_ms(timeout_ms);
+        for (const auto& parsed : {parse_be_obj_storage_config(properties, 
"bucket"),
+                                   parse_be_obj_storage_config(param)}) {
+            ASSERT_TRUE(parsed.has_value()) << parsed.error();
+            const auto& config = std::get<AwsObjStorageConfig>(parsed->client);
+            const auto options =
+                    make_s3_sdk_config(config.connection, config.region, 
config.retry_slow_down);
+            EXPECT_EQ(options.requestTimeoutMs, timeout_ms > 0 ? timeout_ms : 
30000);
+            EXPECT_EQ(options.connectTimeoutMs,
+                      timeout_ms > 0 ? timeout_ms : defaults.connectTimeoutMs);
+        }
+    }
+}
+
+#ifdef USE_AZURE
+TEST_F(S3ClientFactoryTest, AzureLegacyConnectTimeoutKeepsSdkDefault) {
+    TS3StorageParam param;
+    param.__set_provider(TObjStorageType::AZURE);
+    param.__set_bucket("container");
+    param.__set_endpoint("https://account.blob.core.windows.net";);
+    param.__set_ak("account");
+    param.__set_sk("dGVzdA==");
+    // Omitted Thrift fields still carry the 1000ms default sent by existing 
resources.
+    EXPECT_EQ(param.conn_timeout_ms, 1000);
+    SyncPointProcessingGuard sync_point_guard;
+    for (int timeout_ms : {param.conn_timeout_ms, 0, 3000}) {
+        SCOPED_TRACE(timeout_ms);
+        param.__set_conn_timeout_ms(timeout_ms);
+        for (bool with_ca : {false, true}) {
+            SCOPED_TRACE(with_ca);
+            auto parsed = parse_be_obj_storage_config(param);
+            ASSERT_TRUE(parsed.has_value()) << parsed.error();
+            auto& config = std::get<AzureObjStorageConfig>(parsed->client);
+            config.connection.ca_cert_file =
+                    with_ca ? std::make_optional<std::string>("/test/ca.pem") 
: std::nullopt;
+            bool checked_options = false;
+            SyncPoint::CallbackGuard options_callback;
+            SyncPoint::get_instance()->set_call_back(
+                    "AzureObjStorageClient::client_options",
+                    [&](auto&& args) {
+                        checked_options = true;
+                        const auto* options =
+                                
try_any_cast<Azure::Storage::Blobs::BlobClientOptions*>(args[0]);
+                        if (with_ca) {
+                            auto transport =
+                                    
std::dynamic_pointer_cast<Azure::Core::Http::CurlTransport>(
+                                            options->Transport.Transport);
+                            ASSERT_NE(transport, nullptr);
+                            EXPECT_EQ(transport->m_options.CAInfo, 
"/test/ca.pem");
+                            EXPECT_EQ(transport->m_options.ConnectionTimeout,
+                                      std::chrono::minutes(5));
+                        } else {
+                            EXPECT_EQ(options->Transport.Transport, nullptr);
+                        }
+                    },
+                    &options_callback);
+            auto built = AzureObjStorageClient::create(config);
+            ASSERT_NE(built.client, nullptr) << built.error;
+            EXPECT_TRUE(checked_options);
+        }
+    }
+}
+#endif
+
+TEST_F(S3ClientFactoryTest, AzureResourcesCacheServiceClientsPerContainer) {
+    AzureObjStorageConfig azure {
+            .connection = {.endpoint = 
"https://account.blob.core.windows.net"},
+            .auth = {.account_name = "account", .account_key = "dGVzdA=="}};
+    ObjStorageResourceConfig first {.bucket = "one", .client = azure};
+    ObjStorageResourceConfig second {.bucket = "two", .prefix = "data", 
.client = azure};
+    auto& factory = ObjStorageClientFactory::instance();
+    auto result = factory.create(first.client, true, first.bucket);
+#ifdef USE_AZURE
+    ASSERT_NE(result.client, nullptr) << result.error;
+    EXPECT_NE(std::dynamic_pointer_cast<AzureObjStorageClient>(result.client), 
nullptr);
+    auto reused = factory.create(first.client, true, first.bucket);
+    ASSERT_NE(reused.client, nullptr) << reused.error;
+    EXPECT_EQ(result.client, reused.client);
+    auto separate = factory.create(second.client, true, second.bucket);
+    ASSERT_NE(separate.client, nullptr) << separate.error;
+    EXPECT_NE(result.client, separate.client);
+#else
+    ASSERT_EQ(result.client, nullptr);
+    EXPECT_TRUE(result.unsupported);
+    EXPECT_FALSE(result.error.empty());
+#endif
+}
+
+TEST_F(S3ClientFactoryTest, AzureDiscardsS3AddressingAndRoleProperties) {
+    S3URI uri("s3://container/blob");
+    ASSERT_TRUE(uri.parse().ok());
+    Properties props {{"provider", "AZURE"},
+                      {"AWS_ENDPOINT", 
"https://account.blob.core.windows.net"},
+                      {"AWS_ACCESS_KEY", "account"},
+                      {"AWS_SECRET_KEY", "dGVzdA=="}};
+    auto first_result = ObjStorageConfigParser::from_properties(
+            props, uri.get_bucket(),
+            
aws_credential_implementation(config::aws_credentials_provider_version),
+            EmptyCredentialsBehavior::ANONYMOUS);
+    ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(first_result))
+            << std::get<std::string>(first_result);
+    auto first = std::get<ObjStorageResourceConfig>(std::move(first_result));
+    apply_be_obj_storage_defaults(first.client);
+    props.insert({{"AWS_ROLE_ARN", "unused"},
+                  {"AWS_EXTERNAL_ID", "unused"},
+                  {"AWS_REGION", "dummy_region"},
+                  {"use_path_style", "true"},
+                  {"AWS_NEED_OVERRIDE_ENDPOINT", "false"}});
+    auto second_result = ObjStorageConfigParser::from_properties(
+            props, uri.get_bucket(),
+            
aws_credential_implementation(config::aws_credentials_provider_version),
+            EmptyCredentialsBehavior::ANONYMOUS);
+    
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(second_result))
+            << std::get<std::string>(second_result);
+    auto second = std::get<ObjStorageResourceConfig>(std::move(second_result));
+    apply_be_obj_storage_defaults(second.client);
+    EXPECT_EQ(first.client, second.client);
+    auto invalid = std::get<AzureObjStorageConfig>(second.client);
+    invalid.auth.account_key.clear();
+    EXPECT_EQ(ObjStorageClientFactory::instance().create(invalid).client, 
nullptr);
+}
+
+TEST_F(S3ClientFactoryTest, OssPublicSigningPreservesLegacyAwsAuthentication) {
+    class RecordingSignedUrlClient final : public S3ObjStorageClient {
+    public:
+        RecordingSignedUrlClient() : S3ObjStorageClient(nullptr) {}
+
+        std::string generate_presigned_url(const ObjStoragePath& path,
+                                           int64_t expiration_secs) override {
+            signed_path = path;
+            signed_expiration_secs = expiration_secs;
+            return "signed-public-url";
+        }
+
+        ObjStoragePath signed_path;
+        int64_t signed_expiration_secs = 0;
+    };
+
+    for (bool assume_role : {false, true}) {
+        SCOPED_TRACE(assume_role);
+        auto config = 
make_factory_conf("https://oss-cn-beijing-internal.aliyuncs.com";);
+        config.connection.ca_cert_file = "/test/ca-bundle.pem";
+        config.connection.max_connections = 123;
+        config.connection.connect_timeout = std::chrono::milliseconds(456);
+        config.connection.request_timeout = std::chrono::milliseconds(789);
+        config.connection.max_retries = 4;
+        config.auth.source = AwsCredentialChainType::Environment;
+        config.auth.implementation = AwsCredentialImplementation::V2;
+        config.use_path_style = true;
+        config.retry_slow_down = true;
+        if (assume_role) {
+            config.auth.assume_role =
+                    AwsAssumeRole {.role_arn = 
"arn:aws:iam::123456789012:role/legacy-oss",
+                                   .external_id = "legacy-external-id",
+                                   .request_timeout = 
std::chrono::milliseconds(321)};
+        }
+        auto expected = config;
+        expected.connection.endpoint = "https://oss-cn-beijing.aliyuncs.com";;
+
+        // Capture the factory input instead of acquiring ENV credentials or 
contacting STS.
+        // The public endpoint must not reclassify this historical S3 vault as 
OSS auth.
+        std::optional<ObjStorageClientConfig> signing_config;
+        auto recording_client = std::make_shared<RecordingSignedUrlClient>();
+        ObjStorageClientFactory::instance().set_client_creator_for_test(
+                [&](const ObjStorageClientConfig& created) {
+                    signing_config = created;
+                    return recording_client;
+                });
+        auto sign = make_oss_public_presigned_url_generator(config);
+        ASSERT_TRUE(sign);
+        EXPECT_EQ(sign({.bucket = "legacy-bucket", .key = "errors/load-id"}, 
600),
+                  "signed-public-url");
+        ASSERT_TRUE(signing_config.has_value());
+        
ASSERT_TRUE(std::holds_alternative<AwsObjStorageConfig>(*signing_config));
+        EXPECT_EQ(std::get<AwsObjStorageConfig>(*signing_config), expected);
+        EXPECT_EQ(config.connection.endpoint, 
"https://oss-cn-beijing-internal.aliyuncs.com";);
+        EXPECT_EQ(recording_client->signed_path.bucket, "legacy-bucket");
+        EXPECT_EQ(recording_client->signed_path.key, "errors/load-id");
+        EXPECT_EQ(recording_client->signed_expiration_secs, 600);
+    }
+}
+
+TEST_F(S3ClientFactoryTest, ReusesS3ClientsWithinEachBucket) {
+    ObjStorageResourceConfig first {.bucket = "first",
+                                    .client = 
make_factory_conf("https://shared.example.com";)};
+    auto other = first;
+    other.prefix = "warehouse";
+    other.sse_enabled = true;
+    other.is_internal_bucket = true;
+    auto& factory = ObjStorageClientFactory::instance();
+    auto raw = factory.create(first.client, true, first.bucket);
+    auto reused = factory.create(other.client, true, other.bucket);
+    ASSERT_NE(raw.client, nullptr) << raw.error;
+    ASSERT_NE(reused.client, nullptr) << reused.error;
+    EXPECT_EQ(raw.client, reused.client);
+
+    other.bucket = "second";
+    auto separate = factory.create(other.client, true, other.bucket);
+    ASSERT_NE(separate.client, nullptr) << separate.error;
+    EXPECT_NE(raw.client, separate.client);
+    EXPECT_EQ(factory.create(other.client, true, other.bucket).client, 
separate.client);
+
+    auto bypassed = factory.create(first.client, /*use_cache=*/false, 
first.bucket);
+    ASSERT_NE(bypassed.client, nullptr) << bypassed.error;
+    EXPECT_NE(raw.client, bypassed.client);
+    EXPECT_EQ(factory.create(first.client, true, first.bucket).client, 
raw.client);
+}
+
+TEST_F(S3ClientFactoryTest, CacheFlagControlsClientReuse) {
+    auto& factory = ObjStorageClientFactory::instance();
+    auto config = make_factory_conf("https://cache-policy-reuse.example.com";);
+
+    auto cached = factory.create(config);
+    ASSERT_NE(cached.client, nullptr) << cached.error;
+    auto reused = factory.create(config, /*use_cache=*/true);
+    ASSERT_NE(reused.client, nullptr) << reused.error;
+    EXPECT_EQ(reused.client, cached.client);
+
+    auto bypassed = factory.create(config, /*use_cache=*/false);
+    ASSERT_NE(bypassed.client, nullptr) << bypassed.error;
+    EXPECT_NE(bypassed.client, cached.client);
+    auto bypassed_again = factory.create(config, /*use_cache=*/false);
+    ASSERT_NE(bypassed_again.client, nullptr) << bypassed_again.error;
+    EXPECT_NE(bypassed_again.client, bypassed.client);
+
+    // Disabling the cache must leave an existing cached client in place.
+    auto still_cached = factory.create(config, /*use_cache=*/true);
+    ASSERT_NE(still_cached.client, nullptr) << still_cached.error;
+    EXPECT_EQ(still_cached.client, cached.client);
+
+    // A bypassed first creation must not populate an empty cache entry either.
+    config.connection.endpoint = "https://cache-policy-empty.example.com";;
+    auto uncached = factory.create(config, /*use_cache=*/false);
+    ASSERT_NE(uncached.client, nullptr) << uncached.error;
+    auto fresh_cached = factory.create(config, /*use_cache=*/true);
+    ASSERT_NE(fresh_cached.client, nullptr) << fresh_cached.error;
+    EXPECT_NE(fresh_cached.client, uncached.client);
+    auto fresh_reused = factory.create(config);
+    ASSERT_NE(fresh_reused.client, nullptr) << fresh_reused.error;
+    EXPECT_EQ(fresh_reused.client, fresh_cached.client);
+}
+
+TEST_F(S3ClientFactoryTest, CacheFlagValidatesBeforeCreating) {
+    auto& factory = ObjStorageClientFactory::instance();
+    int create_count = 0;
+    factory.set_client_creator_for_test(
+            [&](const ObjStorageClientConfig&) -> 
std::shared_ptr<io::ObjStorageClient> {
+                ++create_count;
+                return std::make_shared<io::S3ObjStorageClient>(
+                        std::shared_ptr<Aws::S3::S3Client> {});
+            });
+    auto config = 
make_factory_conf("https://cache-policy-validation.example.com";);
+    config.connection.max_connections = 0;
+    for (bool use_cache : {true, false}) {
+        auto invalid = factory.create(config, use_cache);
+        EXPECT_EQ(invalid.client, nullptr);
+        EXPECT_EQ(invalid.error, "max_connections must be positive");
+        EXPECT_FALSE(invalid.unsupported);
+    }
+    EXPECT_EQ(create_count, 0);
+
+    config.connection.max_connections = 1;
+    for (bool use_cache : {true, false}) {
+        auto valid = factory.create(config, use_cache);
+        ASSERT_NE(valid.client, nullptr) << valid.error;
+    }
+    EXPECT_EQ(create_count, 2);
+}
+
+TEST_F(S3ClientFactoryTest, DistinguishesTimeoutOptions) {
+    auto external_conf = 
make_timeout_conf("cloud-rate-limit-timeout.example.com", false);
+    auto internal_conf = 
make_timeout_conf("cloud-rate-limit-timeout.example.com", true);
+    ASSERT_NE(external_conf, internal_conf);
+
+    auto& factory = ObjStorageClientFactory::instance();
+    auto external_result = factory.create(external_conf);
+    auto internal_result = factory.create(internal_conf);
+    ASSERT_NE(external_result.client, nullptr) << external_result.error;
+    ASSERT_NE(internal_result.client, nullptr) << internal_result.error;
+    auto external_client = std::move(external_result.client);
+    auto internal_client = std::move(internal_result.client);
+
+    EXPECT_NE(external_client, internal_client);
+    auto cached_external = factory.create(external_conf);
+    auto cached_internal = factory.create(internal_conf);
+    ASSERT_NE(cached_external.client, nullptr) << cached_external.error;
+    ASSERT_NE(cached_internal.client, nullptr) << cached_internal.error;
+    EXPECT_EQ(cached_external.client, external_client);
+    EXPECT_EQ(cached_internal.client, internal_client);
+}
+
+TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesTimeoutOptions) {
+    auto external_conf = 
make_timeout_conf("s3-client-holder-timeout.example.com", false);
+    auto internal_conf = 
make_timeout_conf("s3-client-holder-timeout.example.com", true);
+
+    CloudModeConfigGuard cloud_mode_guard(true);
+    auto external_client =
+            
std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+    auto internal_client =
+            
std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+    int create_count = 0;
+    ObjStorageClientFactory::instance().set_client_creator_for_test(
+            [&](const ObjStorageClientConfig& conf) -> 
std::shared_ptr<io::ObjStorageClient> {
+                ++create_count;
+                return conf == ObjStorageClientConfig {internal_conf} ? 
internal_client
+                                                                      : 
external_client;
+            });
+
+    io::ObjStorageClientHolder holder(external_conf);
+    ASSERT_TRUE(holder.init().ok());
+    EXPECT_EQ(create_count, 1);
+    EXPECT_EQ(holder.get(), external_client);
+
+    ASSERT_TRUE(holder.reset(internal_conf).ok());
+    EXPECT_EQ(create_count, 2);
+    EXPECT_EQ(holder.get(), internal_client);
+    EXPECT_EQ(holder.config(), ObjStorageClientConfig {internal_conf});
+}
+
+TEST_F(S3ClientFactoryTest, 
ObjClientHolderKeepsBucketScopeWhenCredentialsChange) {
+    auto config = make_factory_conf("https://bucket-holder-cache.example.com";);
+    io::ObjStorageClientHolder first(config, nullptr, "first");
+    io::ObjStorageClientHolder second(config, nullptr, "second");
+    ASSERT_TRUE(first.init().ok());
+    ASSERT_TRUE(second.init().ok());
+    auto first_client = first.get();
+    auto second_client = second.get();
+    EXPECT_NE(first_client, second_client);
+    auto& factory = ObjStorageClientFactory::instance();
+    EXPECT_EQ(first_client, factory.create(config, true, "first").client);
+
+    config.auth.source = AccessKeyObjStorageAuth {.access_key = "new-ak", 
.secret_key = "new-sk"};
+    ASSERT_TRUE(first.reset(config).ok());
+    EXPECT_NE(first.get(), first_client);
+    EXPECT_EQ(first.get(), factory.create(config, true, "first").client);
+    auto separate = factory.create(config, true, "second");
+    ASSERT_NE(separate.client, nullptr) << separate.error;
+    EXPECT_NE(first.get(), separate.client);
+    EXPECT_EQ(second.get(), second_client);
+}
+
+TEST_F(S3ClientFactoryTest, SelectsRateLimiterByDeploymentAndBucketType) {
+    RateLimiterConfigGuard rate_limiter_guard;
+    SyncPointProcessingGuard sync_point_guard;
+    auto* sync_point = SyncPoint::get_instance();
+    SyncPoint::CallbackGuard create_client_callback;
+    sync_point->set_call_back(
+            "s3_client_factory::create",
+            [](auto&& args) {
+                auto result = 
try_any_cast_ret<doris::ObjStorageClientBuildResult>(args);
+                result->second = true;
+            },
+            &create_client_callback);
+    SyncPoint::CallbackGuard head_object_callback;
+    sync_point->set_call_back(
+            "s3_file_system::head_object",
+            [](auto&& args) {
+                auto result = 
try_any_cast_ret<Aws::S3::Model::HeadObjectOutcome>(args);
+                result->first =
+                        
Aws::S3::Model::HeadObjectOutcome(Aws::S3::Model::HeadObjectResult {});
+                result->second = true;
+            },
+            &head_object_callback);
+
+    config::enable_s3_rate_limiter = true;
+    auto check_selection = [&](bool cloud_mode, bool internal_bucket, bool 
expect_limited,
+                               std::string endpoint) {
+        CloudModeConfigGuard cloud_mode_guard(cloud_mode);
+        auto& manager = ObjStorageRateLimiterManager::instance();
+        manager.qps_limiter(ObjStorageRateLimitType::GET)->reset(0, 0, 1);
+        manager.bytes_limiter(ObjStorageRateLimitType::GET)->reset(0, 0, 0);
+
+        auto conf = make_factory_conf(std::move(endpoint));
+        io::ObjStorageClientHolder holder(
+                conf, 
ObjStorageRateLimiterManager::policy_for(internal_bucket));
+        ASSERT_TRUE(holder.init().ok());
+        auto client = holder.get();
+        EXPECT_TRUE(client->head_object({.bucket = "bucket", .key = 
"key"}).resp.ok());
+        auto second = client->head_object({.bucket = "bucket", .key = "key"});
+        if (expect_limited) {
+            EXPECT_EQ(second.resp.status.code, 
static_cast<int>(ErrorCode::EXCEEDED_LIMIT));
+            EXPECT_EQ(second.resp.http_code, 0);
+            EXPECT_NE(second.resp.status.msg.find("s3 get request exceeds QPS 
limit"),
+                      std::string::npos);
+        } else {
+            EXPECT_TRUE(second.resp.ok());
+        }
+    };
+
+    // In cloud mode both resources share one raw client but independently 
bind their policy.
+    check_selection(false, false, true, 
"non-cloud-external-rate-limit.example.com");
+    check_selection(true, true, true, "shared-cloud-rate-limit.example.com");
+    check_selection(true, false, false, "shared-cloud-rate-limit.example.com");
+}
+
+TEST_F(S3ClientFactoryTest, 
RateLimitResponseDistinguishesBytesFromProviderThrottling) {
+    RateLimiterConfigGuard rate_limiter_guard;
+    SyncPointProcessingGuard sync_point_guard;
+    SyncPoint::CallbackGuard create_client_callback;
+    SyncPoint::get_instance()->set_call_back(
+            "s3_client_factory::create",
+            [](auto&& args) {
+                auto result = 
try_any_cast_ret<doris::ObjStorageClientBuildResult>(args);
+                result->second = true;
+            },
+            &create_client_callback);
+
+    CloudModeConfigGuard cloud_mode_guard(false);
+    config::enable_s3_rate_limiter = true;
+    auto& manager = ObjStorageRateLimiterManager::instance();
+    manager.qps_limiter(ObjStorageRateLimitType::GET)->reset(0, 0, 0);
+    manager.bytes_limiter(ObjStorageRateLimitType::GET)
+            ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1);
+
+    io::ObjStorageClientHolder 
holder(make_factory_conf("be-bytes-rate-limit-response.example.com"),
+                                      
ObjStorageRateLimiterManager::policy_for(false));
+    ASSERT_TRUE(holder.init().ok());
+    auto client = holder.get();
+    char buffer[2];
+    size_t size_return = 0;
+    auto response = client->get_object({.bucket = "bucket", .key = "key"}, 
buffer, 0,
+                                       sizeof(buffer), &size_return);
+
+    EXPECT_EQ(response.status.code, 
static_cast<int>(ErrorCode::EXCEEDED_LIMIT));
+    EXPECT_EQ(response.http_code, 0);
+    EXPECT_NE(response.status.msg.find("s3 get request exceeds bytes limit"), 
std::string::npos);
+}
+
+TEST_F(S3ClientFactoryTest, ObjClientHolderResetRefreshesNativeCredential) {
+    auto initial_conf = make_gcp_native_conf();
+    auto rotated_conf = 
make_gcp_native_conf("[email protected]");
+    std::vector<ObjStorageClientConfig> created_confs;
+    auto client = 
std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+    ObjStorageClientFactory::instance().set_client_creator_for_test(
+            [&](const ObjStorageClientConfig& conf) -> 
std::shared_ptr<io::ObjStorageClient> {
+                created_confs.push_back(conf);
+                return client;
+            });
+
+    io::ObjStorageClientHolder holder(initial_conf);
+    ASSERT_TRUE(holder.init().ok());
+    ASSERT_TRUE(holder.reset(rotated_conf).ok());
+
+    ASSERT_EQ(created_confs.size(), 2);
+    EXPECT_EQ(created_confs.back(), ObjStorageClientConfig {rotated_conf});
+    EXPECT_EQ(holder.config(), ObjStorageClientConfig {rotated_conf});
+}
+
+TEST_F(S3ClientFactoryTest, AzureLegacyPoolSizeAllowsHolderCredentialRotation) 
{
+    TS3StorageParam param;
+    param.__set_provider(TObjStorageType::AZURE);
+    param.__set_endpoint("https://account.blob.core.windows.net";);
+    param.__set_bucket("container");
+    param.__set_ak("account");
+    param.__set_sk("old-key");
+    param.__set_max_conn(100);
+    auto parse = [&] {
+        return ObjStorageConfigParser::from_storage_param(param, 
AwsCredentialImplementation::V2,
+                                                          
EmptyCredentialsBehavior::ANONYMOUS);
+    };
+    int create_count = 0;
+    ObjStorageClientFactory::instance().set_client_creator_for_test(
+            [&](const ObjStorageClientConfig&) {
+                ++create_count;
+                return 
std::make_shared<S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+            });
+
+    auto initial = parse();
+    ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(initial));
+    auto holder = std::make_shared<io::ObjStorageClientHolder>(
+            std::get<ObjStorageResourceConfig>(initial).client);
+    ASSERT_TRUE(holder->init().ok());
+    auto existing_holder = holder;
+    auto old_client = holder->get();
+
+    param.__set_sk("new-key");
+    auto rotated = parse();
+    ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(rotated));
+    const auto& rotated_conf = 
std::get<ObjStorageResourceConfig>(rotated).client;
+    ASSERT_TRUE(holder->reset(rotated_conf).ok());
+    EXPECT_EQ(create_count, 2);
+    EXPECT_NE(existing_holder->get(), old_client);
+    EXPECT_EQ(existing_holder->config(), rotated_conf);
+    
EXPECT_EQ(std::get<AzureObjStorageConfig>(existing_holder->config()).auth.account_key,
+              "new-key");
+    
EXPECT_FALSE(std::get<AzureObjStorageConfig>(rotated_conf).connection.max_connections);
+}
+
+TEST_F(S3ClientFactoryTest, CredentialRotationPreservesQuotaAndInFlightClient) 
{
+    CloudModeConfigGuard cloud_mode_guard(true);
+    RateLimiterConfigGuard rate_limiter_guard;
+    config::enable_s3_rate_limiter = true;
+    auto& manager = ObjStorageRateLimiterManager::instance();
+    manager.qps_limiter(ObjStorageRateLimitType::GET)->reset(0, 0, 1);
+    manager.bytes_limiter(ObjStorageRateLimitType::GET)->reset(0, 0, 0);
+
+    auto conf = make_factory_conf("rotate.example.com");
+    auto raw = 
std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+    ObjStorageClientFactory::instance().set_client_creator_for_test(
+            [&](const ObjStorageClientConfig&) { return raw; });
+    io::ObjStorageClientHolder holder(conf, 
ObjStorageRateLimiterManager::policy_for(true));
+    ASSERT_TRUE(holder.init().ok());
+    auto old_client = holder.get();
+    SyncPointProcessingGuard sync_point_guard;
+    SyncPoint::CallbackGuard head_callback;
+    SyncPoint::get_instance()->set_call_back(
+            "s3_file_system::head_object",
+            [](auto&& args) {
+                auto result = 
try_any_cast_ret<Aws::S3::Model::HeadObjectOutcome>(args);
+                result->first =
+                        
Aws::S3::Model::HeadObjectOutcome(Aws::S3::Model::HeadObjectResult {});
+                result->second = true;
+            },
+            &head_callback);
+    EXPECT_TRUE(old_client->head_object({.bucket = "bucket", .key = 
"key"}).resp.ok());
+
+    conf.auth.source = AccessKeyObjStorageAuth {.access_key = "new-ak"};
+    EXPECT_FALSE(holder.reset(conf).ok());
+    EXPECT_EQ(holder.get(), old_client);
+    EXPECT_TRUE(std::holds_alternative<AnonymousObjStorageAuth>(
+            std::get<AwsObjStorageConfig>(holder.config()).auth.source));
+
+    std::get<AccessKeyObjStorageAuth>(conf.auth.source).secret_key = "new-sk";
+    ASSERT_TRUE(holder.reset(conf).ok());
+    EXPECT_EQ(holder.config(), ObjStorageClientConfig {conf});
+    EXPECT_NE(holder.get(), old_client);
+    for (const auto& client : {old_client, holder.get()}) {
+        auto response = client->head_object({.bucket = "bucket", .key = 
"key"}).resp;
+        EXPECT_EQ(response.status.code, 
static_cast<int>(ErrorCode::EXCEEDED_LIMIT));
+    }
+}
+
+TEST_F(S3ClientFactoryTest, ObjClientHolderKeepsClientWhenTargetChanges) {
+    auto native_conf = make_gcp_native_conf();
+    AwsObjStorageConfig s3_conf {.connection = native_conf.connection,
+                                 .region = native_conf.region};
+
+    std::vector<ObjStorageClientConfig> created_confs;
+    auto client = 
std::make_shared<io::S3ObjStorageClient>(std::shared_ptr<Aws::S3::S3Client> {});
+    ObjStorageClientFactory::instance().set_client_creator_for_test(
+            [&](const ObjStorageClientConfig& conf) -> 
std::shared_ptr<io::ObjStorageClient> {
+                created_confs.push_back(conf);
+                return client;
+            });
+
+    io::ObjStorageClientHolder holder(s3_conf);
+    ASSERT_TRUE(holder.init().ok());
+    auto old_client = holder.get();
+    EXPECT_FALSE(holder.reset(native_conf).ok());

Review Comment:
   [P2] Use a different physical target in the rejection test
   
   `s3_conf` copies the native GCP endpoint and region, and both configurations 
use an explicit endpoint. `same_client_target` now intentionally accepts this 
provider-only switch, so `holder.reset(native_conf)` succeeds, publishes the 
GCP configuration, and calls the test creator a second time. The rejection, 
old-configuration, and single-creation expectations here therefore fail 
whenever this new unit test runs. Change the endpoint or region to exercise 
target rejection; same-target provider switching should continue to succeed as 
required by the retained-holder credential-rotation fix.



##########
be/test/io/object_storage/obj_storage_config_test.cpp:
##########
@@ -0,0 +1,1040 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/cloud.pb.h>
+#include <gtest/gtest.h>
+
+#include <initializer_list>
+#include <map>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+#include "common/config.h"
+#include "cpp/obj-storage/parser/obj_storage_config_parser.h"
+#include "io/object_storage/be_obj_storage_config_parser.h"
+#include "util/s3_uri.h"
+
+namespace doris {
+
+TEST(ObjStorageConfigTest, CredentialChangesPreserveTarget) {
+    AwsObjStorageConfig aws;
+    aws.region = "region";
+    aws.connection.endpoint = "https://endpoint";;
+    aws.auth.source = AccessKeyObjStorageAuth {"ak", "sk", "token"};
+    auto update = aws;
+    std::get<AccessKeyObjStorageAuth>(update.auth.source).session_token = 
"rotated";
+    EXPECT_TRUE(same_client_target(aws, update));
+    EXPECT_NE(aws, update);
+    update.region = "another-region";
+    EXPECT_FALSE(same_client_target(aws, update));
+    update = aws;
+    update.connection.ca_cert_file = "/etc/ssl/ca.pem";
+    EXPECT_TRUE(same_client_target(aws, update));
+    EXPECT_NE(aws, update);
+    update.connection.endpoint = "another-endpoint";
+    EXPECT_FALSE(same_client_target(aws, update));
+}
+
+TEST(ObjStorageConfigTest, ResourceFieldsDoNotChangeClientIdentity) {
+    AzureObjStorageConfig azure;
+    azure.connection.endpoint = "https://account.blob.core.windows.net";;
+    azure.auth = {"account", "key"};
+    ObjStorageResourceConfig first {.bucket = "container-1", .prefix = 
"/root/", .client = azure};
+    auto second = first;
+    second.bucket = "container-2";
+    second.prefix = "archive";
+    second.is_internal_bucket = true;
+    second.sse_enabled = true;
+    EXPECT_EQ(first.normalized_prefix(), "root");
+    EXPECT_EQ(first.client, second.client);
+    EXPECT_EQ(ObjStorageClientHash {}(first.client), ObjStorageClientHash 
{}(second.client));
+    std::get<AzureObjStorageConfig>(second.client).auth.account_name = 
"other-account";
+    EXPECT_FALSE(same_client_target(first.client, second.client));
+}
+
+TEST(ObjStorageConfigTest, LegacyResourceWithoutProviderUsesAws) {
+    TS3StorageParam param;
+    param.__set_bucket("legacy");
+    param.__set_root_path("/tablets");
+    param.__set_endpoint("https://s3.example";);
+    param.__set_region("region");
+    auto parsed_result = ObjStorageConfigParser::from_storage_param(
+            param, 
aws_credential_implementation(config::aws_credentials_provider_version),
+            EmptyCredentialsBehavior::ANONYMOUS);
+    
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(parsed_result))
+            << std::get<std::string>(parsed_result);
+    auto parsed = std::get<ObjStorageResourceConfig>(std::move(parsed_result));
+    EXPECT_EQ(parsed.bucket, "legacy");
+    EXPECT_EQ(parsed.prefix, "/tablets");
+    const auto& aws = std::get<AwsObjStorageConfig>(parsed.client);
+    EXPECT_EQ(aws.connection.endpoint, "https://s3.example";);
+    EXPECT_EQ(aws.connection.max_connections, 50);
+    EXPECT_EQ(aws.connection.request_timeout, std::chrono::milliseconds(3000));
+    EXPECT_EQ(aws.connection.connect_timeout, std::chrono::milliseconds(1000));
+}
+
+TEST(ObjStorageConfigTest, UnknownThriftUsesAwsWhileVaultInfersEndpoint) {
+    const struct {
+        const char* endpoint;
+        ObjStorageProvider expected;
+    } cases[] = {
+            {"oss-cn-hangzhou.aliyuncs.com", ObjStorageProvider::OSS},
+            {"https://bucket.OSS-CN-HANGZHOU-INTERNAL.aliyuncs.com:443/path";,
+             ObjStorageProvider::OSS},
+            {"s3.cn-hangzhou.aliyuncs.com", ObjStorageProvider::OSS},
+            {"cn-shanghai.oss-dls.aliyuncs.com", ObjStorageProvider::AWS},
+            {"ecs.cn-hangzhou.aliyuncs.com", ObjStorageProvider::AWS},
+            {"bucket.cos.ap-guangzhou.myqcloud.com", ObjStorageProvider::COS},
+            {"obs.cn-north-4.myhuaweicloud.com", ObjStorageProvider::OBS},
+            {"bj.bcebos.com", ObjStorageProvider::BOS},
+            {"tos-s3-cn-beijing.volces.com", ObjStorageProvider::TOS},
+            {"account.blob.core.windows.net", ObjStorageProvider::AZURE},
+            {"https://storage.googleapis.com";, ObjStorageProvider::GCP},
+            {"https://objects.example.com";, ObjStorageProvider::AWS},
+            {"https://oss-cn-hangzhou.aliyuncs.com.example.com";, 
ObjStorageProvider::AWS},
+            {"https://[email protected]";, 
ObjStorageProvider::AWS},
+            {"https://objects.example.com/oss-cn-hangzhou.aliyuncs.com";, 
ObjStorageProvider::AWS},
+    };
+    for (const auto& entry : cases) {
+        SCOPED_TRACE(entry.endpoint);
+        TS3StorageParam param;
+        param.__set_bucket("bucket");
+        param.__set_endpoint(entry.endpoint);
+        auto thrift_result = ObjStorageConfigParser::from_storage_param(
+                param, 
aws_credential_implementation(config::aws_credentials_provider_version),
+                EmptyCredentialsBehavior::ANONYMOUS);
+        
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(thrift_result))
+                << std::get<std::string>(thrift_result);
+        auto thrift = 
std::get<ObjStorageResourceConfig>(std::move(thrift_result));
+        EXPECT_EQ(ObjStorageProvider::AWS, storage_provider(thrift.client));
+        cloud::ObjectStoreInfoPB vault;
+        vault.set_bucket("bucket");
+        vault.set_endpoint(entry.endpoint);
+        auto pb_result = ObjStorageConfigParser::from_vault(
+                vault, 
aws_credential_implementation(config::aws_credentials_provider_version),
+                EmptyCredentialsBehavior::ANONYMOUS);
+        
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(pb_result))
+                << std::get<std::string>(pb_result);
+        auto pb = std::get<ObjStorageResourceConfig>(std::move(pb_result));
+        EXPECT_EQ(entry.expected, storage_provider(pb.client));
+    }
+}
+
+TEST(ObjStorageConfigTest, 
UnknownThriftProviderPreservesLegacyCredentialChain) {
+    TS3StorageParam param;
+    param.__set_bucket("private-bucket");
+    param.__set_endpoint("https://oss-cn-hangzhou.aliyuncs.com";);
+    param.__set_region("cn-hangzhou");
+    for (auto mode : {TCredProviderType::DEFAULT, TCredProviderType::ENV}) {
+        param.__set_cred_provider_type(mode);
+        auto parsed = ObjStorageConfigParser::from_storage_param(
+                param, AwsCredentialImplementation::V2, 
EmptyCredentialsBehavior::ANONYMOUS);
+        ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(parsed));
+        auto& client = std::get<ObjStorageResourceConfig>(parsed).client;
+        ASSERT_TRUE(std::holds_alternative<AwsObjStorageConfig>(client));
+        auto& source = std::get<AwsObjStorageConfig>(client).auth.source;
+        ASSERT_TRUE(std::holds_alternative<AwsCredentialChainType>(source));
+        EXPECT_EQ(std::get<AwsCredentialChainType>(source),
+                  mode == TCredProviderType::ENV ? 
AwsCredentialChainType::Environment
+                                                 : 
AwsCredentialChainType::Default);
+    }
+}
+
+TEST(ObjStorageConfigTest, MissingPropertyProviderPreservesAwsCredentialChain) 
{
+    const struct {
+        const char* name;
+        AwsCredentialChainType expected;
+    } modes[] = {{"ENV", AwsCredentialChainType::Environment},
+                 {"DEFAULT", AwsCredentialChainType::Default}};
+    for (const char* endpoint :
+         {"oss-cn-hangzhou.aliyuncs.com", "cos.ap-guangzhou.myqcloud.com",
+          "obs.cn-north-4.myhuaweicloud.com", "bj.bcebos.com", 
"tos-s3-cn-beijing.volces.com",
+          "https://storage.googleapis.com";, "account.blob.core.windows.net"}) {
+        SCOPED_TRACE(endpoint);
+        for (const auto& mode : modes) {
+            SCOPED_TRACE(mode.name);
+            // FE's generic S3 provider emits these fields without a provider 
hint,
+            // including when users select S3 explicitly at another cloud's 
endpoint.
+            auto result = ObjStorageConfigParser::from_properties(
+                    {{"AWS_ENDPOINT", endpoint},
+                     {"AWS_REGION", "region"},
+                     {"AWS_ACCESS_KEY", ""},
+                     {"AWS_SECRET_KEY", ""},
+                     {"AWS_CREDENTIALS_PROVIDER_TYPE", mode.name}},
+                    "bucket", AwsCredentialImplementation::V2, 
EmptyCredentialsBehavior::ANONYMOUS);
+            
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(result))
+                    << std::get<std::string>(result);
+            const auto& resource = std::get<ObjStorageResourceConfig>(result);
+            
ASSERT_TRUE(std::holds_alternative<AwsObjStorageConfig>(resource.client));
+            const auto& aws = std::get<AwsObjStorageConfig>(resource.client);
+            EXPECT_EQ(aws.auth.source, AwsCredentialSource {mode.expected});
+            EXPECT_EQ(aws.connection.endpoint, endpoint);
+            EXPECT_FALSE(aws.validate());
+        }
+    }
+}
+
+TEST(ObjStorageConfigTest, 
ExplicitPropertyProviderPreservesGcpAndAzureAuthentication) {
+    auto gcp_result = ObjStorageConfigParser::from_properties(
+            {{"provider", "GCP"},
+             {"AWS_ENDPOINT", "https://storage.googleapis.com"},
+             {"AWS_REGION", "us-east1"},
+             {"gs.credential_provider_type", "COMPUTE_ENGINE"}},
+            "bucket", AwsCredentialImplementation::V2, 
EmptyCredentialsBehavior::ANONYMOUS);
+    ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(gcp_result))
+            << std::get<std::string>(gcp_result);
+    const auto& gcp_resource = std::get<ObjStorageResourceConfig>(gcp_result);
+    
ASSERT_TRUE(std::holds_alternative<GcpObjStorageConfig>(gcp_resource.client));
+    const auto& gcp = std::get<GcpObjStorageConfig>(gcp_resource.client);
+    ASSERT_TRUE(std::holds_alternative<GcpNativeObjStorageAuth>(gcp.auth));
+    EXPECT_EQ(std::get<GcpNativeObjStorageAuth>(gcp.auth).source,
+              GcpCredentialSource::ComputeEngine);
+    EXPECT_FALSE(gcp.validate());
+
+    auto azure_result = ObjStorageConfigParser::from_properties(
+            {{"provider", "azure"},
+             {"AWS_ENDPOINT", "https://account.blob.core.windows.net"},
+             {"AWS_ACCESS_KEY", "account"},
+             {"AWS_SECRET_KEY", "key"}},
+            "container", AwsCredentialImplementation::V2, 
EmptyCredentialsBehavior::ANONYMOUS);
+    ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(azure_result))
+            << std::get<std::string>(azure_result);
+    const auto& azure_resource = 
std::get<ObjStorageResourceConfig>(azure_result);
+    
ASSERT_TRUE(std::holds_alternative<AzureObjStorageConfig>(azure_resource.client));
+    const auto& azure = std::get<AzureObjStorageConfig>(azure_resource.client);
+    EXPECT_EQ(azure.auth.account_name, "account");
+    EXPECT_EQ(azure.auth.account_key, "key");
+    EXPECT_FALSE(azure.validate());
+}
+
+TEST(ObjStorageConfigTest, ParsesLegacyAwsProperties) {
+    S3URI uri("s3://bucket/key");
+    ASSERT_TRUE(uri.parse().ok());
+    std::map<std::string, std::string> props {{"AWS_ACCESS_KEY", "ak"},
+                                              {"AWS_SECRET_KEY", "sk"},
+                                              {"AWS_TOKEN", "token"},
+                                              {"aws_endpoint", "endpoint"},
+                                              {"AWS_REGION", "region"},
+                                              {"AWS_NEED_OVERRIDE_ENDPOINT", 
"false"},
+                                              {"AWS_MAX_CONNECTIONS", "42"},
+                                              {"AWS_REQUEST_TIMEOUT_MS", 
"2345"},
+                                              {"AWS_CONNECTION_TIMEOUT_MS", 
"678"},
+                                              {"USE_PATH_STYLE", "true"},
+                                              {"AWS_ROLE_ARN", "role"},
+                                              {"AWS_EXTERNAL_ID", "external"},
+                                              
{"AWS_CREDENTIALS_PROVIDER_TYPE", "ENV"}};
+    auto parsed_result = ObjStorageConfigParser::from_properties(
+            props, uri.get_bucket(),
+            
aws_credential_implementation(config::aws_credentials_provider_version),
+            EmptyCredentialsBehavior::ANONYMOUS);
+    
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(parsed_result))
+            << std::get<std::string>(parsed_result);
+    auto parsed = std::get<ObjStorageResourceConfig>(std::move(parsed_result));
+    apply_be_obj_storage_defaults(parsed.client);
+    EXPECT_EQ(parsed.bucket, "bucket");
+    const auto& aws = std::get<AwsObjStorageConfig>(parsed.client);
+    EXPECT_TRUE(aws.use_sdk_endpoint);
+    EXPECT_TRUE(aws.use_path_style);
+    EXPECT_TRUE(aws.connection.endpoint.empty());
+    EXPECT_EQ(aws.region, "region");
+    EXPECT_EQ(aws.connection.max_connections, 42);
+    EXPECT_EQ(aws.connection.request_timeout, std::chrono::milliseconds(2345));
+    EXPECT_EQ(aws.connection.connect_timeout, std::chrono::milliseconds(678));
+    EXPECT_EQ(std::get<AccessKeyObjStorageAuth>(aws.auth.source),
+              (AccessKeyObjStorageAuth {"ak", "sk", "token"}));
+    EXPECT_FALSE(aws.auth.assume_role);
+    props["aws_endpoint"] = "unused-second-override";
+    auto second_result = ObjStorageConfigParser::from_properties(
+            props, uri.get_bucket(),
+            
aws_credential_implementation(config::aws_credentials_provider_version),
+            EmptyCredentialsBehavior::ANONYMOUS);
+    
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(second_result))
+            << std::get<std::string>(second_result);
+    auto second = std::get<ObjStorageResourceConfig>(std::move(second_result));
+    apply_be_obj_storage_defaults(second.client);
+    EXPECT_EQ(parsed.client, second.client);
+    props["provider"] = "BOS";
+    auto unsupported = ObjStorageConfigParser::from_properties(
+            props, uri.get_bucket(),
+            
aws_credential_implementation(config::aws_credentials_provider_version),
+            EmptyCredentialsBehavior::ANONYMOUS);
+    ASSERT_TRUE(std::holds_alternative<std::string>(unsupported));
+    EXPECT_NE(std::get<std::string>(unsupported).find("requires 
provider=AWS"), std::string::npos);
+}
+
+TEST(ObjStorageConfigTest, 
NonAwsCredentialRestrictionsApplyToEveryInputFormat) {
+    const struct {
+        const char* name;
+        TObjStorageType::type thrift;
+        cloud::ObjectStoreInfoPB::Provider pb;
+    } providers[] = {
+            {"OSS", TObjStorageType::OSS, cloud::ObjectStoreInfoPB::OSS},
+            {"COS", TObjStorageType::COS, cloud::ObjectStoreInfoPB::COS},
+            {"BOS", TObjStorageType::BOS, cloud::ObjectStoreInfoPB::BOS},
+            {"OBS", TObjStorageType::OBS, cloud::ObjectStoreInfoPB::OBS},
+            {"TOS", TObjStorageType::TOS, cloud::ObjectStoreInfoPB::TOS},
+            {"GCP", TObjStorageType::GCP, cloud::ObjectStoreInfoPB::GCP},
+    };
+    for (const auto& provider : providers) {
+        SCOPED_TRACE(provider.name);
+        ObjStorageProperties properties {{"provider", provider.name},
+                                         {"AWS_ENDPOINT", "endpoint"},
+                                         {"AWS_REGION", "region"}};
+        TS3StorageParam param;
+        param.__set_provider(provider.thrift);
+        param.__set_bucket("bucket");
+        param.__set_endpoint("endpoint");
+        param.__set_region("region");
+        cloud::ObjectStoreInfoPB vault;
+        vault.set_provider(provider.pb);
+        vault.set_bucket("bucket");
+        vault.set_endpoint("endpoint");
+        vault.set_region("region");
+        // Recycler's AWS default chain must not turn another cloud's empty 
keys
+        // into an AWS metadata or profile lookup.
+        for (const auto& result :
+             {ObjStorageConfigParser::from_properties(properties, "bucket",
+                                                      
AwsCredentialImplementation::V1,
+                                                      
EmptyCredentialsBehavior::DEFAULT_CHAIN),
+              ObjStorageConfigParser::from_storage_param(param, 
AwsCredentialImplementation::V1,
+                                                         
EmptyCredentialsBehavior::DEFAULT_CHAIN),
+              ObjStorageConfigParser::from_vault(vault, 
AwsCredentialImplementation::V1,
+                                                 
EmptyCredentialsBehavior::DEFAULT_CHAIN)}) {
+            
ASSERT_TRUE(std::holds_alternative<ObjStorageResourceConfig>(result))
+                    << std::get<std::string>(result);
+            std::visit(
+                    [](const auto& client) {
+                        using C = std::decay_t<decltype(client)>;
+                        if constexpr (!std::is_same_v<C, AwsObjStorageConfig> 
&&
+                                      !std::is_same_v<C, 
AzureObjStorageConfig>) {
+                            EXPECT_TRUE(
+                                    
std::holds_alternative<AnonymousObjStorageAuth>(client.auth));

Review Comment:
   [P2] Preserve the default HMAC expectation in the GCP matrix
   
   The GCP row calls all three parser entry points with `V1` and 
`DEFAULT_CHAIN`, so `apply_aws_credential_defaults` selects `DEFAULT` and 
`parse_gcp_obj_storage_config` returns `GcpDefaultHmacObjStorageAuth`. This 
blanket anonymous assertion therefore fails for the properties, Thrift, and 
protobuf cases whenever this new unit test runs. The later 
`GcpDefaultHmacKeepsHostPolicyAcrossInputFormats` test correctly expects the 
HMAC chain for the same inputs. Give GCP its chain expectation here (or exclude 
it from this anonymous-only assertion) while preserving the production 
compatibility fix.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to