github-actions[bot] commented on code in PR #66350:
URL: https://github.com/apache/doris/pull/66350#discussion_r3788968617
##########
be/src/util/s3_util.cpp:
##########
@@ -291,132 +293,59 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_azure_client(
VLOG_DEBUG << "uri:" << uri << ", normalized_uri:" << normalized_uri;
std::string tls_debug_context =
build_azure_tls_debug_context(_ca_cert_file_path);
- auto containerClient =
std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(
- uri, cred, std::move(options));
+ auto built = AzureAuthFactory::create(uri,
+ {
+ .type =
AzureCredentialType::SHARED_KEY,
+ .account_name = s3_conf.ak,
+ .account_key = s3_conf.sk,
+ },
+ std::move(options));
+ if (!built) {
+ return ResultError(
+ Status::InvalidArgument("failed to create Azure client: {}",
built.error));
+ }
LOG_INFO("create one azure client with {}", s3_conf.to_string());
- return
std::make_shared<io::AzureObjStorageClient>(std::move(containerClient),
-
std::move(tls_debug_context));
+ return std::make_shared<io::AzureObjStorageClient>(
+ std::move(built.container_client),
+ ObjStorageEndpointInfo {
+ .endpoint = s3_conf.endpoint,
+ .ak = s3_conf.ak,
+ .sk = s3_conf.sk,
+ .tls_debug_context = std::move(tls_debug_context),
+ },
+ std::move(built.shared_key_credential));
#else
- LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON
before building");
- return nullptr;
+ return ResultError(Status::NotSupported(
+ "BE is not compiled with azure support, export BUILD_AZURE=ON
before building"));
#endif
}
-std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
-S3ClientFactory::_get_aws_credentials_provider_v1(const S3ClientConf& s3_conf)
{
- if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
- Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
- DCHECK(!aws_cred.IsExpiredOrEmpty());
- if (!s3_conf.token.empty()) {
- aws_cred.SetSessionToken(s3_conf.token);
- }
- return
std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(std::move(aws_cred));
- }
-
- if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) {
- if (s3_conf.role_arn.empty()) {
- return
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
- }
-
- Aws::Client::ClientConfiguration clientConfiguration =
- S3ClientFactory::getClientConfiguration();
-
- if (_ca_cert_file_path.empty()) {
- _ca_cert_file_path =
-
get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
- }
- if (!_ca_cert_file_path.empty()) {
- clientConfiguration.caFile = _ca_cert_file_path;
- }
-
- auto stsClient = std::make_shared<Aws::STS::STSClient>(
-
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>(),
- clientConfiguration);
-
- return std::make_shared<Aws::Auth::STSAssumeRoleCredentialsProvider>(
- s3_conf.role_arn, Aws::String(), s3_conf.external_id,
- Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient);
- }
-
- // Support anonymous access for public datasets when no credentials are
provided
- if (s3_conf.ak.empty() && s3_conf.sk.empty()) {
- return std::make_shared<Aws::Auth::AnonymousAWSCredentialsProvider>();
- }
-
- return std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
-}
-
-std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::_create_credentials_provider(
- CredProviderType type) {
- switch (type) {
- case CredProviderType::Env:
- return
std::make_shared<Aws::Auth::EnvironmentAWSCredentialsProvider>();
- case CredProviderType::SystemProperties:
- return
std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>();
- case CredProviderType::WebIdentity:
- return
std::make_shared<Aws::Auth::STSAssumeRoleWebIdentityCredentialsProvider>();
- case CredProviderType::Container:
- return std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(
-
Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str());
- case CredProviderType::InstanceProfile:
- return
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
- case CredProviderType::Anonymous:
- return std::make_shared<Aws::Auth::AnonymousAWSCredentialsProvider>();
- case CredProviderType::Default:
- default:
- return std::make_shared<CustomAwsCredentialsProviderChain>();
- }
-}
-
-std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
-S3ClientFactory::_get_aws_credentials_provider_v2(const S3ClientConf& s3_conf)
{
- if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
- Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
- DCHECK(!aws_cred.IsExpiredOrEmpty());
- if (!s3_conf.token.empty()) {
- aws_cred.SetSessionToken(s3_conf.token);
- }
- return
std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(std::move(aws_cred));
- }
-
- // Handle role_arn for assume role scenario
- if (!s3_conf.role_arn.empty()) {
- Aws::Client::ClientConfiguration clientConfiguration =
- S3ClientFactory::getClientConfiguration();
-
- if (_ca_cert_file_path.empty()) {
- _ca_cert_file_path =
-
get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
- }
- if (!_ca_cert_file_path.empty()) {
- clientConfiguration.caFile = _ca_cert_file_path;
- }
-
- auto baseProvider =
_create_credentials_provider(s3_conf.cred_provider_type);
- auto stsClient = std::make_shared<Aws::STS::STSClient>(baseProvider,
clientConfiguration);
-
- return std::make_shared<Aws::Auth::STSAssumeRoleCredentialsProvider>(
- s3_conf.role_arn, Aws::String(), s3_conf.external_id,
- Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient);
- }
-
- // Return provider based on cred_provider_type
- return _create_credentials_provider(s3_conf.cred_provider_type);
-}
-
-std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::get_aws_credentials_provider(
- const S3ClientConf& s3_conf) {
- if (config::aws_credentials_provider_version == "v2") {
- return _get_aws_credentials_provider_v2(s3_conf);
- }
- return _get_aws_credentials_provider_v1(s3_conf);
+AwsCredentialResult S3ClientFactory::create_aws_credentials_provider(const
S3ClientConf& s3_conf) {
+ auto sts_config = S3ClientFactory::getClientConfiguration();
+ if (!_ca_cert_file_path.empty()) {
Review Comment:
[P2] Refresh the CA before building the Kinesis STS client
This helper now copies `_ca_cert_file_path` only when the factory's
constructor found a valid bundle. The deleted V1/V2 assumed-role branches
retried `get_valid_ca_cert_path(config::ca_cert_file_paths)` whenever that
cache was empty. `KinesisDataConsumer::_create_kinesis_client` still resolves
the current CA for the Kinesis client and then calls this helper directly, so
if the singleton was initialized while the bundle was absent and the mutable
config/path becomes valid later, Kinesis trusts the CA but the STS client used
to obtain its role credentials does not and role authentication can fail.
Please resolve a fresh local CA before building `sts_config` (or synchronize
any cache refresh), and cover late CA availability for an assumed-role Kinesis
client.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
