github-actions[bot] commented on code in PR #70657:
URL: https://github.com/apache/doris/pull/70657#discussion_r4597061452
##########
common/cpp/obj-storage/client/azure_obj_storage_client.cpp:
##########
@@ -436,9 +580,9 @@ ObjStorageResponse
AzureObjStorageClient::delete_objects(const ObjStoragePath& o
std::advance(chunk_end,
std::min(batch_size,
static_cast<size_t>(std::distance(begin, end))));
- auto deleter = AzureBatchDeleter(_client.get(), opts, *begin,
_config.tls_debug_context);
- std::ranges::for_each(std::ranges::subrange(begin, chunk_end),
- [&](const std::string& obj) {
deleter.delete_blob(obj); });
+ auto container = _service.GetBlobContainerClient(opts.bucket);
+ auto deleter = AzureBatchDeleter(&container, opts, *begin,
_config.tls_debug_context);
Review Comment:
[P2] Convert OAuth failures from deferred batch responses
With the new ClientSecret authentication, a cached token can cross the SDK's
proactive refresh boundary while a batch request is in flight (for example,
start one second before the boundary and receive the response two seconds
later). The SDK then refreshes while [replaying
subresponses](https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.16.0/sdk/storage/azure-storage-blobs/src/blob_batch.cpp#L404-L419),
storing any `AuthenticationException` in the deferred future.
`AzureBatchDeleter::execute()` subsequently rethrows it at `GetResponse()`,
outside the `SubmitBatch` conversion; its deferred loop catches only
`RequestFailedException`, whereas `AuthenticationException` derives directly
from `std::exception`. Thus this supported client API throws instead of
returning an error `ObjStorageResponse`. Include deferred response consumption
in the redacted standard-exception boundary as well.
##########
common/cpp/obj-storage/auth/azure/azure_auth.cpp:
##########
@@ -0,0 +1,301 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-storage/auth/azure/azure_auth.h"
+
+#include <algorithm>
+#include <chrono>
+#include <string>
+
+namespace doris {
+namespace {
+
+std::optional<std::string> validate_method(std::string_view account,
+ const AzureSharedKeyConfig&
shared_key) {
+ if (account.empty() || shared_key.account_key.empty()) {
+ return "incomplete Azure shared-key credentials";
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> validate_method(std::string_view /*account*/,
+ const AzureSasConfig& sas) {
+ if (sas.token.empty() || sas.token == "?") {
+ return "Invalid azure conf, empty SAS token";
+ }
+ if (sas.token.find_first_of("#\r\n") != std::string::npos) {
+ return "Azure SAS token contains an invalid query character";
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> validate_method(std::string_view /*account*/,
+ const AzureClientSecretConfig&
client_secret) {
+ if (client_secret.tenant_id.empty()) {
+ return "Invalid azure conf, empty tenant_id for client secret";
+ }
+ if (client_secret.client_id.empty()) {
+ return "Invalid azure conf, empty client_id for client secret";
+ }
+ if (client_secret.client_secret.empty()) {
+ return "Invalid azure conf, empty client_secret for client secret";
+ }
+ const std::string_view authority = client_secret.authority_host;
+ if (!authority.empty()) {
+ // Authorities select where the SDK sends the client secret. Require a
TLS
+ // origin (optionally a trailing slash), not a URL with userinfo or a
path.
+ if (!authority.starts_with("https://")) {
+ return "Azure OAuth authority must be an HTTPS origin";
+ }
+ auto host = authority.substr(8);
+ if (host.ends_with('/')) {
+ host.remove_suffix(1);
+ }
+ if (host.empty() || host.find_first_of("/?#@\r\n\t ") !=
std::string_view::npos) {
+ return "Azure OAuth authority must be an HTTPS origin";
+ }
+ }
+ return std::nullopt;
+}
+
+int sas_hex_value(char c) {
+ if (c >= '0' && c <= '9') {
+ return c - '0';
+ }
+ if (c >= 'a' && c <= 'f') {
+ return c - 'a' + 10;
+ }
+ if (c >= 'A' && c <= 'F') {
+ return c - 'A' + 10;
+ }
+ return -1;
+}
+
+// Decodes %XX escapes into *decoded. SAS values never encode a space as '+',
so '+' is kept
+// as is.
+std::optional<std::string> sas_url_decode(std::string_view value, std::string*
decoded) {
+ decoded->reserve(value.size());
+ for (size_t i = 0; i < value.size(); ++i) {
+ if (value[i] != '%') {
+ decoded->push_back(value[i]);
+ continue;
+ }
+ if (i + 2 >= value.size() || sas_hex_value(value[i + 1]) < 0 ||
+ sas_hex_value(value[i + 2]) < 0) {
+ return "Azure SAS token has an invalid escape";
+ }
+ decoded->push_back(
+ static_cast<char>(sas_hex_value(value[i + 1]) * 16 +
sas_hex_value(value[i + 2])));
+ i += 2;
+ }
+ return std::nullopt;
+}
+
+// Parses value[pos, pos + n) as decimal digits.
+bool sas_parse_digits(std::string_view value, size_t pos, size_t n, int64_t*
out) {
+ if (pos + n > value.size()) {
+ return false;
+ }
+ int64_t result = 0;
+ for (size_t i = pos; i < pos + n; ++i) {
+ if (value[i] < '0' || value[i] > '9') {
+ return false;
+ }
+ result = result * 10 + (value[i] - '0');
+ }
+ *out = result;
+ return true;
+}
+
+// Days since 1970-01-01 of a proleptic Gregorian date (Howard Hinnant's
days_from_civil).
+// Done by hand so that the result never depends on the process time zone.
+int64_t sas_days_from_civil(int64_t year, int64_t month, int64_t day) {
+ year -= month <= 2 ? 1 : 0;
+ const int64_t era = (year >= 0 ? year : year - 399) / 400;
+ const int64_t year_of_era = year - era * 400;
+ const int64_t shifted_month = month > 2 ? month - 3 : month + 9;
+ const int64_t day_of_year = (153 * shifted_month + 2) / 5 + day - 1;
+ const int64_t day_of_era =
+ year_of_era * 365 + year_of_era / 4 - year_of_era / 100 +
day_of_year;
+ return era * 146097 + day_of_era - 719468;
+}
+
+// Parses the UTC forms of the SAS "se" field into Unix milliseconds:
+// YYYY-MM-DD, YYYY-MM-DDThh:mmZ, YYYY-MM-DDThh:mm:ssZ and
YYYY-MM-DDThh:mm:ss.fffffffZ.
+AzureAuthExpiry parse_sas_time(std::string_view value) {
+ const AzureAuthExpiry invalid {
+ .expires_at_ms = std::nullopt,
+ .error = "Azure SAS token has an invalid expiry",
+ };
+ int64_t year = 0;
+ int64_t month = 0;
+ int64_t day = 0;
+ if (value.size() < 10 || value[4] != '-' || value[7] != '-' ||
+ !sas_parse_digits(value, 0, 4, &year) || !sas_parse_digits(value, 5,
2, &month) ||
+ !sas_parse_digits(value, 8, 2, &day) || month < 1 || month > 12 || day
< 1 || day > 31) {
+ return invalid;
+ }
+ // Reject impossible calendar dates (e.g. February 30) instead of rolling
them
+ // into the next month and accidentally extending the local credential
lifetime.
+ if (!std::chrono::year_month_day
{std::chrono::year(static_cast<int>(year)),
+
std::chrono::month(static_cast<unsigned>(month)),
+
std::chrono::day(static_cast<unsigned>(day))}
+ .ok()) {
+ return invalid;
+ }
+ int64_t hour = 0;
+ int64_t minute = 0;
+ int64_t second = 0;
+ int64_t millis = 0;
+ if (value.size() > 10) {
+ if (value[10] != 'T' || value.back() != 'Z') {
Review Comment:
[P2] Accept numeric timezone offsets in SAS expiry
Azure permits numeric UTC offsets in the SAS `se` field ([accepted
formats](https://learn.microsoft.com/en-us/rest/api/storageservices/formatting-datetime-values)).
An otherwise valid, unexpired signed token containing
`se=2099-01-01T00%3A00%3A00%2B00%3A00` is therefore rejected here as an invalid
expiry. Both `from_azure_client_config()` and the client factory reach this
check, so the token never reaches Azure. Parse the supported suffixes into UTC
milliseconds while preserving the original signed token; the existing expiry
tests cover only date/`Z` forms.
##########
common/cpp/obj-storage/parser/obj_storage_config_parser.cpp:
##########
@@ -0,0 +1,741 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-storage/parser/obj_storage_config_parser.h"
+
+#include <brpc/uri.h>
+#include <fmt/format.h>
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+
+#include <algorithm>
+#include <cctype>
+#include <charconv>
+#include <type_traits>
+#include <utility>
+
+#include "cpp/obj-storage/auth/gcp/gcp_auth.h"
+
+namespace doris {
+namespace {
+using Vault = cloud::ObjectStoreInfoPB;
+
+struct ProviderMapping {
+ std::string_view name;
+ TObjStorageType::type resource;
+ cloud::ObjectStoreInfoPB::Provider vault;
+};
+
+constexpr ProviderMapping PROVIDERS[] = {
+ {"AWS", TObjStorageType::AWS, cloud::ObjectStoreInfoPB::S3},
+ {"S3", TObjStorageType::AWS, cloud::ObjectStoreInfoPB::S3},
+ {"OSS", TObjStorageType::OSS, cloud::ObjectStoreInfoPB::OSS},
+ {"COS", TObjStorageType::COS, cloud::ObjectStoreInfoPB::COS},
+ {"BOS", TObjStorageType::BOS, cloud::ObjectStoreInfoPB::BOS},
+ {"OBS", TObjStorageType::OBS, cloud::ObjectStoreInfoPB::OBS},
+ {"GCP", TObjStorageType::GCP, cloud::ObjectStoreInfoPB::GCP},
+ {"GCS", TObjStorageType::GCP, cloud::ObjectStoreInfoPB::GCP},
+ {"TOS", TObjStorageType::TOS, cloud::ObjectStoreInfoPB::TOS},
+ {"AZURE", TObjStorageType::AZURE, cloud::ObjectStoreInfoPB::AZURE},
+};
+
+std::optional<std::string_view>
credential_provider_name(TCredProviderType::type provider) {
+ switch (provider) {
+ case TCredProviderType::DEFAULT:
+ return "DEFAULT";
+ case TCredProviderType::SIMPLE:
+ return "SIMPLE";
+ case TCredProviderType::INSTANCE_PROFILE:
+ return "INSTANCE_PROFILE";
+ case TCredProviderType::ENV:
+ return "ENV";
+ case TCredProviderType::SYSTEM_PROPERTIES:
+ return "SYSTEM_PROPERTIES";
+ case TCredProviderType::WEB_IDENTITY:
+ return "WEB_IDENTITY";
+ case TCredProviderType::CONTAINER:
+ return "CONTAINER";
+ case TCredProviderType::ANONYMOUS:
+ return "ANONYMOUS";
+ default:
+ return std::nullopt;
+ }
+}
+
+std::optional<std::string_view> gcp_credential_provider_name(
+ TGcpCredentialProviderType::type provider) {
+ switch (provider) {
+ case TGcpCredentialProviderType::DEFAULT:
+ return "DEFAULT";
+ case TGcpCredentialProviderType::COMPUTE_ENGINE:
+ return "COMPUTE_ENGINE";
+ default:
+ return std::nullopt;
+ }
+}
+
+std::string to_upper(std::string text) {
+ std::transform(text.begin(), text.end(), text.begin(),
+ [](unsigned char c) { return
static_cast<char>(std::toupper(c)); });
+ return text;
+}
+
+std::string normalize_provider(std::string text) {
+ const auto non_space = [](unsigned char c) { return !std::isspace(c); };
+ text.erase(text.begin(), std::find_if(text.begin(), text.end(),
non_space));
+ text.erase(std::find_if(text.rbegin(), text.rend(), non_space).base(),
text.end());
+ return to_upper(std::move(text));
+}
+
+std::string_view pop_dns_label(std::string_view& name) {
+ const auto dot = name.rfind('.');
+ if (dot == std::string_view::npos) {
+ return std::exchange(name, std::string_view {});
+ }
+ auto label = name.substr(dot + 1);
+ name = name.substr(0, dot);
+ return label;
+}
+
+} // namespace
+
+cloud::ObjectStoreInfoPB::Provider vault_storage_provider(const
cloud::ObjectStoreInfoPB& info) {
+ if (info.has_provider() || info.endpoint().empty()) {
+ return info.provider();
+ }
+ return storage_provider_from_endpoint(info.endpoint());
+}
+
+cloud::ObjectStoreInfoPB::Provider
storage_provider_from_endpoint(std::string_view endpoint) {
+ std::string url(endpoint);
+ std::transform(url.begin(), url.end(), url.begin(),
+ [](unsigned char c) { return std::tolower(c); });
+ brpc::URI uri;
+ if (url.find('\0') != std::string::npos || uri.SetHttpURL(url.c_str()) !=
0 ||
+ (!uri.scheme().empty() && uri.scheme() != "http" && uri.scheme() !=
"https")) {
+ return Vault::S3;
+ }
+ std::string_view host = uri.host();
+ if (host.ends_with('.')) {
+ host.remove_suffix(1);
+ }
+ if (host.empty() ||
host.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-.") !=
+ std::string_view::npos) {
+ return Vault::S3;
+ }
+
+ // Match from the DNS root so virtual-hosted buckets cannot change the
service or cloud.
+ auto labels = host;
+ auto tld = pop_dns_label(labels);
+ auto domain = pop_dns_label(labels);
+ auto last = pop_dns_label(labels);
+ auto previous = pop_dns_label(labels);
+ auto is_region = [](std::string_view name) {
+ return name.size() > 3 && name[2] == '-' && name[0] >= 'a' && name[0]
<= 'z' &&
+ name[1] >= 'a' && name[1] <= 'z';
+ };
+ if (domain == "aliyuncs" && tld == "com") {
+ if ((last.starts_with("oss-") && is_region(last.substr(4))) || last ==
"oss-accelerate" ||
+ last == "oss-accelerate-overseas" || (previous == "s3" &&
is_region(last)) ||
+ (last == "oss" && is_region(previous))) {
+ return Vault::OSS;
+ }
+ } else if (domain == "googleapis" && tld == "com" && last == "storage") {
+ return Vault::GCP;
+ } else if (domain == "myqcloud" && tld == "com" && previous == "cos" &&
!last.empty()) {
+ return Vault::COS;
+ } else if (domain == "myhuaweicloud" && tld == "com" && previous == "obs"
&& !last.empty()) {
+ return Vault::OBS;
+ } else if (domain == "bcebos" && tld == "com" && !last.empty()) {
+ return Vault::BOS;
+ } else if ((domain == "volces" || domain == "ivolces") && tld == "com" &&
+ ((last.starts_with("tos-") && is_region(last.substr(4))) ||
+ (last.starts_with("tos-s3-") && is_region(last.substr(7))))) {
+ return Vault::TOS;
+ } else if (last == "core" && (previous == "blob" || previous == "dfs") &&
!labels.empty() &&
+ ((domain == "windows" && tld == "net") ||
+ (domain == "chinacloudapi" && tld == "cn") ||
+ (domain == "usgovcloudapi" && tld == "net") ||
+ (domain == "cloudapi" && tld == "de"))) {
+ return Vault::AZURE;
+ }
+ return Vault::S3;
+}
+
+AwsCredentialImplementation aws_credential_implementation(std::string_view
version) {
+ return version == "v2" ? AwsCredentialImplementation::V2 :
AwsCredentialImplementation::V1;
+}
+
+void normalize_obj_storage_endpoint(ObjStorageClientConfig& client,
std::string_view scheme) {
+ auto& endpoint = obj_storage_connection(client).endpoint;
+ if (auto* aws = std::get_if<AwsObjStorageConfig>(&client); aws &&
aws->use_sdk_endpoint) {
+ endpoint.clear();
+ aws->sdk_endpoint_http = scheme == "http";
+ return;
+ }
+ if (!endpoint.empty() && endpoint.find("://") == std::string::npos) {
+ endpoint.insert(0,
+ scheme == "http" &&
!std::holds_alternative<AzureObjStorageConfig>(client)
+ ? "http://"
+ : "https://");
+ }
+}
+
+namespace {
+using Props = ObjStorageProperties;
+using Error = std::optional<std::string>;
+template <class T>
+using Parsed = ObjStorageConfigParseResult<T>;
+
+std::string value(const Props& properties, const char* key) {
+ auto it = properties.find(key);
+ return it == properties.end() ? std::string {} : it->second;
+}
+
+template <class T>
+Error read_number(const Props& props, const char* key, std::optional<T>&
output) {
+ auto it = props.find(key);
+ if (it == props.end()) {
+ return {};
+ }
+ const auto& text = it->second;
+ int number;
+ auto [end, error] = std::from_chars(text.data(), text.data() +
text.size(), number);
+ if (error != std::errc {} || end != text.data() + text.size() || number <
-1) {
+ return fmt::format("invalid {} value \"{}\"", key, text);
+ }
+ // Legacy resources use -1 or 0 for host defaults. Passing a zero request
timeout
+ // to the SDK would disable stalled-transfer detection instead of keeping
the BE default.
+ if (number > 0) {
+ output = T(number);
+ }
+ return {};
+}
+Error read_timeouts(const Props& props, ObjStorageConnectionConfig& conn) {
+ if (auto error = read_number(props, "AWS_REQUEST_TIMEOUT_MS",
conn.request_timeout)) {
+ return error;
+ }
+ return read_number(props, "AWS_CONNECTION_TIMEOUT_MS",
conn.connect_timeout);
+}
+Error read_bool(const Props& props, const char* key, bool& output) {
+ auto it = props.find(key);
+ if (it == props.end()) {
+ return {};
+ }
+ auto text = it->second;
+ std::transform(text.begin(), text.end(), text.begin(),
+ [](unsigned char c) { return
static_cast<char>(std::tolower(c)); });
+ if (text == "true") {
+ output = true;
+ } else if (text == "false") {
+ output = false;
+ } else {
+ return fmt::format("invalid {} value \"{}\"", key, it->second);
+ }
+ return {};
+}
+template <class C>
+Error read_regional_options(const Props& props, C& client) {
+ client.region = value(props, "AWS_REGION");
+ client.connection.endpoint = value(props, "AWS_ENDPOINT");
+ if (auto error = read_bool(props, "use_path_style",
client.use_path_style)) {
+ return error;
+ }
+ bool override_endpoint = true;
+ if (auto error = read_bool(props, "AWS_NEED_OVERRIDE_ENDPOINT",
override_endpoint)) {
+ return error;
+ }
+ if (!override_endpoint) {
+ if constexpr (std::is_same_v<C, AwsObjStorageConfig>) {
+ client.use_sdk_endpoint = true;
+ } else {
+ return "AWS SDK automatic endpoint selection requires
provider=AWS";
+ }
+ }
+ if (auto error = read_number(props, "AWS_MAX_CONNECTIONS",
client.connection.max_connections)) {
+ return error;
+ }
+ return read_timeouts(props, client.connection);
+}
+
+AwsCredentialChainType chain_source(CredProviderType mode) {
+ switch (mode) {
+ case CredProviderType::Env:
+ return AwsCredentialChainType::Environment;
+ case CredProviderType::SystemProperties:
+ return AwsCredentialChainType::Profile;
+ case CredProviderType::WebIdentity:
+ return AwsCredentialChainType::WebIdentity;
+ case CredProviderType::Container:
+ return AwsCredentialChainType::Container;
+ case CredProviderType::InstanceProfile:
+ return AwsCredentialChainType::InstanceProfile;
+ default:
+ return AwsCredentialChainType::Default;
+ }
+}
+
+Parsed<AccessKeyOrAnonymousObjStorageAuth> parse_access_key_credentials(const
Props& props) {
+ AccessKeyObjStorageAuth keys {value(props, "AWS_ACCESS_KEY"), value(props,
"AWS_SECRET_KEY"),
+ value(props, "AWS_TOKEN")};
+ if (!keys.access_key.empty() && !keys.secret_key.empty()) {
+ // Some persisted resources also carry AWS role/provider fields.
Static keys
+ // have always taken precedence; they include temporary credentials
issued by FE.
+ return AccessKeyOrAnonymousObjStorageAuth {std::move(keys)};
+ }
+ if (keys.access_key.empty() != keys.secret_key.empty()) {
+ return std::string("Incomplete access key credentials");
+ }
+ if (!keys.session_token.empty()) {
+ return std::string("Session token requires access key credentials");
+ }
+ const auto mode = to_upper(value(props, "AWS_CREDENTIALS_PROVIDER_TYPE"));
+ if ((!mode.empty() && mode != "DEFAULT" && mode != "SIMPLE" && mode !=
"ANONYMOUS") ||
+ !value(props, "AWS_ROLE_ARN").empty() || !value(props,
"AWS_EXTERNAL_ID").empty()) {
+ return std::string(
+ "AWS credential chains and AssumeRole are not supported for
this provider");
+ }
+ return AccessKeyOrAnonymousObjStorageAuth {AnonymousObjStorageAuth {}};
+}
+
+template <class C>
+Parsed<C> parse_access_key_client(const Props& props) {
+ C client;
+ if (auto error = read_regional_options(props, client)) {
+ return *error;
+ }
+ auto credentials = parse_access_key_credentials(props);
+ if (auto* error = std::get_if<std::string>(&credentials)) {
+ return std::move(*error);
+ }
+ client.auth =
std::get<AccessKeyOrAnonymousObjStorageAuth>(std::move(credentials));
+ return client;
+}
+} // namespace
+
+ObjStorageConfigParseResult<AwsObjStorageConfig> parse_aws_obj_storage_config(
+ const ObjStorageProperties& props, AwsCredentialImplementation impl) {
+ AwsObjStorageConfig client;
+ if (auto error = read_regional_options(props, client)) {
+ return *error;
+ }
+ client.auth.implementation = impl;
+ AccessKeyObjStorageAuth keys {value(props, "AWS_ACCESS_KEY"), value(props,
"AWS_SECRET_KEY"),
+ value(props, "AWS_TOKEN")};
+ if (!keys.access_key.empty() && !keys.secret_key.empty()) {
+ client.auth.source = std::move(keys);
+ return client;
+ }
+ auto arn = value(props, "AWS_ROLE_ARN");
+ bool partial = keys.access_key.empty() != keys.secret_key.empty();
+ if (partial && arn.empty()) {
+ return std::string("Incomplete access key credentials");
+ }
+ auto mode = cred_provider_type_from_string(value(props,
"AWS_CREDENTIALS_PROVIDER_TYPE"));
+ if (impl == AwsCredentialImplementation::V1 && mode !=
CredProviderType::InstanceProfile) {
+ // Legacy v1 only supports AssumeRole through the instance profile.
Host
+ // defaults are expressed as an explicit mode by the unified entry
points.
+ if (partial || mode != CredProviderType::Anonymous) {
+ client.auth.source = AwsCredentialChainType::Default;
+ }
+ } else {
+ if (mode != CredProviderType::Anonymous) {
+ client.auth.source = chain_source(mode);
+ }
+ if (!arn.empty()) {
+ AwsAssumeRole role;
+ role.role_arn = std::move(arn);
+ if (auto ext = value(props, "AWS_EXTERNAL_ID"); !ext.empty()) {
+ role.external_id = std::move(ext);
+ }
+ client.auth.assume_role = std::move(role);
+ }
+ }
+ return client;
+}
+
+ObjStorageConfigParseResult<OssObjStorageConfig> parse_oss_obj_storage_config(
+ const ObjStorageProperties& props) {
+ return parse_access_key_client<OssObjStorageConfig>(props);
+}
+ObjStorageConfigParseResult<CosObjStorageConfig> parse_cos_obj_storage_config(
+ const ObjStorageProperties& props) {
+ return parse_access_key_client<CosObjStorageConfig>(props);
+}
+ObjStorageConfigParseResult<BosObjStorageConfig> parse_bos_obj_storage_config(
+ const ObjStorageProperties& props) {
+ return parse_access_key_client<BosObjStorageConfig>(props);
+}
+ObjStorageConfigParseResult<ObsObjStorageConfig> parse_obs_obj_storage_config(
+ const ObjStorageProperties& props) {
+ return parse_access_key_client<ObsObjStorageConfig>(props);
+}
+ObjStorageConfigParseResult<TosObjStorageConfig> parse_tos_obj_storage_config(
+ const ObjStorageProperties& props) {
+ return parse_access_key_client<TosObjStorageConfig>(props);
+}
+
+ObjStorageConfigParseResult<GcpObjStorageConfig> parse_gcp_obj_storage_config(
+ const ObjStorageProperties& props) {
+ GcpObjStorageConfig client;
+ if (auto e = read_regional_options(props, client)) {
+ return *e;
+ }
+ // Capture the explicit timeout before BE/Recycler fill object-I/O
defaults.
+ // A vault without a timeout must keep IAM signing at 10s even when BE I/O
uses 30s.
+ client.signing_timeout =
client.connection.request_timeout.value_or(std::chrono::seconds(10));
+ auto native = parse_gcp_credential_properties(props);
+ if (native.error) {
+ return *native.error;
+ }
+ if (native.credential) {
+ auto mode = cred_provider_type_from_string(value(props,
"AWS_CREDENTIALS_PROVIDER_TYPE"));
+ if (!value(props, "AWS_ACCESS_KEY").empty() || !value(props,
"AWS_SECRET_KEY").empty() ||
+ !value(props, "AWS_TOKEN").empty() || !value(props,
"AWS_ROLE_ARN").empty() ||
+ !value(props, "AWS_EXTERNAL_ID").empty() || mode !=
CredProviderType::Default) {
+ return std::string("GCP credentials cannot be combined with other
credentials");
+ }
+ GcpNativeObjStorageAuth auth;
+ auth.source = native.credential->provider_type ==
GcpCredentialProviderType::ComputeEngine
+ ? GcpCredentialSource::ComputeEngine
+ : GcpCredentialSource::Default;
+ if (!native.credential->impersonation_service_account.empty()) {
+ auth.impersonation_service_account =
+
std::move(native.credential->impersonation_service_account);
+ }
+ client.auth = std::move(auth);
+ } else {
+ auto credentials = parse_access_key_credentials(props);
+ if (auto* error = std::get_if<std::string>(&credentials)) {
+ return std::move(*error);
+ }
+ std::visit(
+ [&]<class T>(T&& selected) {
+ if constexpr (std::is_same_v<std::decay_t<T>,
AccessKeyObjStorageAuth>) {
+ client.auth = GcpHmacObjStorageAuth
{std::move(selected.access_key),
+
std::move(selected.secret_key),
+
std::move(selected.session_token)};
+ } else {
+ client.auth = std::move(selected);
+ }
+ },
+
std::get<AccessKeyOrAnonymousObjStorageAuth>(std::move(credentials)));
+ }
+ return client;
+}
+ObjStorageConfigParseResult<AzureObjStorageConfig>
parse_azure_obj_storage_config(
+ const ObjStorageProperties& props) {
+ AzureObjStorageConfig client;
+ client.connection.endpoint = value(props, "AWS_ENDPOINT");
+ client.auth = {value(props, "AWS_ACCESS_KEY"),
+ AzureSharedKeyConfig {value(props, "AWS_SECRET_KEY")}};
+ if (client.auth.account_name.empty() !=
+
std::get<AzureSharedKeyConfig>(client.auth.credential).account_key.empty()) {
+ return std::string("Incomplete Azure shared key credentials");
+ }
+ if (auto it = props.find("AWS_MAX_CONNECTIONS"); it != props.end()) {
+ // Legacy properties checked integer conversion (including numeric
prefixes), but Azure
+ // ignored the pool size. Preserve that check without adding it to the
effective config.
+ const auto& text = it->second;
+ int ignored_pool_size;
+ if (std::from_chars(text.data(), text.data() + text.size(),
ignored_pool_size).ec !=
+ std::errc {}) {
+ return fmt::format("invalid AWS_MAX_CONNECTIONS value \"{}\"",
text);
+ }
+ }
+ if (auto error = read_timeouts(props, client.connection)) {
+ return *error;
+ }
+ return client;
+}
+
+namespace {
+template <class T>
+Parsed<ObjStorageClientConfig> as_client_config(Parsed<T> parsed) {
+ if (auto* error = std::get_if<std::string>(&parsed)) {
+ return std::move(*error);
+ }
+ return ObjStorageClientConfig {std::get<T>(std::move(parsed))};
+}
+
+Parsed<ObjStorageClientConfig> parse_obj_storage_client_config(Vault::Provider
provider,
+ const Props&
props,
+
AwsCredentialImplementation impl) {
+ if (provider != Vault::GCP &&
+ (props.contains(std::string(GCP_CREDENTIAL_PROVIDER_TYPE)) ||
+ props.contains(std::string(GCP_IMPERSONATION_SERVICE_ACCOUNT)))) {
+ return std::string("GCP credentials require provider=GCP");
+ }
+ switch (provider) {
+ case Vault::S3:
+ return as_client_config(parse_aws_obj_storage_config(props, impl));
+ case Vault::OSS:
+ return as_client_config(parse_oss_obj_storage_config(props));
+ case Vault::COS:
+ return as_client_config(parse_cos_obj_storage_config(props));
+ case Vault::BOS:
+ return as_client_config(parse_bos_obj_storage_config(props));
+ case Vault::OBS:
+ return as_client_config(parse_obs_obj_storage_config(props));
+ case Vault::TOS:
+ return as_client_config(parse_tos_obj_storage_config(props));
+ case Vault::GCP:
+ return as_client_config(parse_gcp_obj_storage_config(props));
+ case Vault::AZURE:
+ return as_client_config(parse_azure_obj_storage_config(props));
+ default:
+ return fmt::format("unknown object storage provider {}",
static_cast<int>(provider));
+ }
+}
+
+void apply_aws_credential_defaults(Props& props, AwsCredentialImplementation
implementation,
+ EmptyCredentialsBehavior empty_credentials)
{
+ if (implementation == AwsCredentialImplementation::V1 &&
+ cred_provider_type_from_string(value(props,
"AWS_CREDENTIALS_PROVIDER_TYPE")) !=
+ CredProviderType::InstanceProfile) {
+ // BE v1 historically uses anonymous access for an empty key pair,
while
+ // Recycler v1 uses the SDK default chain. Keep this host policy
AWS-only.
+ props["AWS_CREDENTIALS_PROVIDER_TYPE"] =
+ empty_credentials == EmptyCredentialsBehavior::ANONYMOUS ?
"ANONYMOUS" : "DEFAULT";
+ }
+}
+
+Parsed<ObjStorageResourceConfig> parse_resource(ObjStorageProperties
properties,
+ Vault::Provider provider,
std::string bucket,
+ std::string prefix,
+ AwsCredentialImplementation
implementation,
+ EmptyCredentialsBehavior
aws_empty_credentials) {
+ if (provider == Vault::S3) {
+ apply_aws_credential_defaults(properties, implementation,
aws_empty_credentials);
+ }
+ auto parsed = parse_obj_storage_client_config(provider, properties,
implementation);
+ if (auto* error = std::get_if<std::string>(&parsed)) {
+ return std::move(*error);
+ }
+ if (bucket.empty()) {
+ return std::string("object storage bucket is not specified");
+ }
+ return ObjStorageResourceConfig {.bucket = std::move(bucket),
+ .prefix = std::move(prefix),
+ .client =
std::get<ObjStorageClientConfig>(std::move(parsed))};
+}
+} // namespace
+
+ObjStorageConfigParseResult<ObjStorageResourceConfig>
+ObjStorageConfigParser::from_azure_client_config(const TAzureClientConfig&
config) {
+ // A query here would mix credentials with the SAS query added by the SDK.
+ // Keep malformed URLs and their possible credentials out of error
messages.
+ brpc::URI endpoint;
+ if ((!config.endpoint.starts_with("http://") &&
!config.endpoint.starts_with("https://")) ||
+ config.endpoint.find('\0') != std::string::npos ||
+ config.endpoint.find_first_of("?#@\r\n\t ") != std::string::npos ||
+ endpoint.SetHttpURL(config.endpoint.c_str()) != 0 ||
endpoint.host().empty() ||
+ (endpoint.scheme() != "http" && endpoint.scheme() != "https")) {
+ return std::string(
+ "Azure endpoint must be an HTTP(S) service URL without "
+ "credentials, query or fragment");
+ }
+ if (config.container.empty()) {
+ return std::string("Azure container is not specified");
+ }
+ const auto& credential = config.credential;
+ const int credential_count =
static_cast<int>(credential.__isset.shared_key) +
+ static_cast<int>(credential.__isset.sas) +
+
static_cast<int>(credential.__isset.client_secret);
+ if (!config.__isset.credential || credential_count != 1) {
+ return std::string("Azure client config must carry exactly one
credential");
+ }
+ AzureObjStorageConfig client;
+ client.connection.endpoint = config.endpoint;
+ client.auth.account_name = config.account_name;
+ if (credential.__isset.shared_key) {
+ client.auth.credential = AzureSharedKeyConfig
{credential.shared_key.account_key};
+ } else if (credential.__isset.sas) {
+ std::string_view token = credential.sas.token;
+ if (!token.empty() && token.front() == '?') {
+ token.remove_prefix(1);
+ }
+ client.auth.credential = AzureSasConfig {
+ .token = std::string(token),
+ .expires_at_ms = credential.sas.__isset.expires_at_ms
+ ?
std::optional<int64_t>(credential.sas.expires_at_ms)
+ : std::nullopt};
+ } else {
+ const auto& secret = credential.client_secret;
+ client.auth.credential = AzureClientSecretConfig {.tenant_id =
secret.tenant_id,
+ .client_id =
secret.client_id,
+ .client_secret =
secret.client_secret,
+ .authority_host =
secret.authority_host};
+ }
+ if (auto error = client.validate()) {
+ return std::move(*error);
+ }
+ return ObjStorageResourceConfig {.bucket = config.container, .client =
std::move(client)};
+}
+
+ObjStorageConfigParseResult<ObjStorageResourceConfig>
ObjStorageConfigParser::from_vault(
+ const cloud::ObjectStoreInfoPB& info, AwsCredentialImplementation
implementation,
+ EmptyCredentialsBehavior aws_empty_credentials) {
+ ObjStorageProperties properties;
+ if (info.has_endpoint()) {
+ properties["AWS_ENDPOINT"] = info.endpoint();
+ }
+ if (info.has_region()) {
+ properties["AWS_REGION"] = info.region();
+ }
+ if (info.has_ak()) {
+ properties["AWS_ACCESS_KEY"] = info.ak();
+ }
+ if (info.has_sk()) {
+ properties["AWS_SECRET_KEY"] = info.sk();
+ }
+ if (info.has_role_arn()) {
+ properties["AWS_ROLE_ARN"] = info.role_arn();
+ }
+ if (info.has_external_id()) {
+ properties["AWS_EXTERNAL_ID"] = info.external_id();
+ }
+
+ if (info.has_use_path_style()) {
+ properties["use_path_style"] = info.use_path_style() ? "true" :
"false";
+ }
+ if (info.has_cred_provider_type()) {
+ properties["AWS_CREDENTIALS_PROVIDER_TYPE"] =
+ cloud::CredProviderTypePB_Name(info.cred_provider_type());
+ }
+ if (info.credential().has_gcp_credential()) {
+ const auto& gcp = info.credential().gcp_credential();
+ properties[std::string(GCP_CREDENTIAL_PROVIDER_TYPE)] =
+
cloud::GcpCredentialPB::CredentialProviderType_Name(gcp.credential_provider_type());
+ if (gcp.has_impersonation_service_account()) {
+ properties[std::string(GCP_IMPERSONATION_SERVICE_ACCOUNT)] =
+ gcp.impersonation_service_account();
+ }
+ }
+ if (info.bucket().empty()) {
+ return "Object storage bucket is not specified";
+ }
+ auto parsed = parse_resource(std::move(properties),
vault_storage_provider(info), info.bucket(),
+ info.prefix(), implementation,
aws_empty_credentials);
+ if (auto* resource = std::get_if<ObjStorageResourceConfig>(&parsed)) {
+ resource->sse_enabled = info.sse_enabled();
+ }
+ return parsed;
+}
+
+ObjStorageConfigParseResult<ObjStorageResourceConfig>
ObjStorageConfigParser::from_properties(
+ const ObjStorageProperties& input, std::string bucket,
+ AwsCredentialImplementation implementation,
+ EmptyCredentialsBehavior aws_empty_credentials) {
+ ObjStorageProperties properties;
+ for (const auto& [key, v] : input) {
+ auto normalized = to_upper(key);
+ if (!normalized.starts_with("AWS_")) {
+ std::transform(key.begin(), key.end(), normalized.begin(),
+ [](unsigned char c) { return
static_cast<char>(std::tolower(c)); });
+ }
+ properties.try_emplace(std::move(normalized), v);
+ }
+ // FE's generic S3 backend map omits provider even for explicitly selected
S3
+ // access to another cloud. Inferring GCP from storage.googleapis.com here
+ // would reject its ENV credentials instead of preserving the S3 auth path.
+ auto provider = Vault::S3;
+ const auto name = normalize_provider(value(properties, "provider"));
+ if (!name.empty()) {
+ const auto entry = std::ranges::find(PROVIDERS, name,
&ProviderMapping::name);
+ if (entry != std::end(PROVIDERS)) {
+ provider = entry->vault;
+ }
+ }
+ return parse_resource(std::move(properties), provider, std::move(bucket),
{}, implementation,
+ aws_empty_credentials);
+}
+
+ObjStorageConfigParseResult<ObjStorageResourceConfig>
ObjStorageConfigParser::from_storage_param(
+ const TS3StorageParam& param, AwsCredentialImplementation
implementation,
+ EmptyCredentialsBehavior aws_empty_credentials) {
+ Vault::Provider provider;
+ if (param.provider == TObjStorageType::UNKNOWN) {
+ provider = storage_provider_from_endpoint(param.endpoint);
Review Comment:
[P2] Preserve credential-chain auth for legacy UNKNOWN resources
An existing cold-storage resource with an OSS endpoint,
`s3_validity_check=false`, no `provider` or inline AK/SK, and BE
`aws_credentials_provider_version=v2` can authenticate using OSS keys from the
AWS environment/profile chain. `S3ThriftAdapter` still sends `UNKNOWN` for that
resource. Previously `get_s3_conf()` mapped it to AWS and used that chain; this
inference now selects OSS, whose no-key parser returns
`AnonymousObjStorageAuth`. After a resource push or BE restart, private-bucket
reads therefore fail instead of using the configured credentials. Preserve the
legacy credential resolution for unspecified providers.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]