github-actions[bot] commented on code in PR #70657:
URL: https://github.com/apache/doris/pull/70657#discussion_r4609103239


##########
common/cpp/obj-storage/parser/obj_storage_config_parser.cpp:
##########
@@ -0,0 +1,751 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cpp/obj-storage/parser/obj_storage_config_parser.h"
+
+#include <brpc/uri.h>
+#include <fmt/format.h>
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/ObjectStorage_types.h>
+
+#include <algorithm>
+#include <cctype>
+#include <charconv>
+#include <type_traits>
+#include <utility>
+
+#include "cpp/obj-storage/auth/gcp/gcp_auth.h"
+
+namespace doris {
+namespace {
+using Vault = cloud::ObjectStoreInfoPB;
+
+struct ProviderMapping {
+    std::string_view name;
+    TObjStorageType::type resource;
+    cloud::ObjectStoreInfoPB::Provider vault;
+};
+
+constexpr ProviderMapping PROVIDERS[] = {
+        {"AWS", TObjStorageType::AWS, cloud::ObjectStoreInfoPB::S3},
+        {"S3", TObjStorageType::AWS, cloud::ObjectStoreInfoPB::S3},
+        {"OSS", TObjStorageType::OSS, cloud::ObjectStoreInfoPB::OSS},
+        {"COS", TObjStorageType::COS, cloud::ObjectStoreInfoPB::COS},
+        {"BOS", TObjStorageType::BOS, cloud::ObjectStoreInfoPB::BOS},
+        {"OBS", TObjStorageType::OBS, cloud::ObjectStoreInfoPB::OBS},
+        {"GCP", TObjStorageType::GCP, cloud::ObjectStoreInfoPB::GCP},
+        {"GCS", TObjStorageType::GCP, cloud::ObjectStoreInfoPB::GCP},
+        {"TOS", TObjStorageType::TOS, cloud::ObjectStoreInfoPB::TOS},
+        {"AZURE", TObjStorageType::AZURE, cloud::ObjectStoreInfoPB::AZURE},
+};
+
+std::optional<std::string_view> 
credential_provider_name(TCredProviderType::type provider) {
+    switch (provider) {
+    case TCredProviderType::DEFAULT:
+        return "DEFAULT";
+    case TCredProviderType::SIMPLE:
+        return "SIMPLE";
+    case TCredProviderType::INSTANCE_PROFILE:
+        return "INSTANCE_PROFILE";
+    case TCredProviderType::ENV:
+        return "ENV";
+    case TCredProviderType::SYSTEM_PROPERTIES:
+        return "SYSTEM_PROPERTIES";
+    case TCredProviderType::WEB_IDENTITY:
+        return "WEB_IDENTITY";
+    case TCredProviderType::CONTAINER:
+        return "CONTAINER";
+    case TCredProviderType::ANONYMOUS:
+        return "ANONYMOUS";
+    default:
+        return std::nullopt;
+    }
+}
+
+std::optional<std::string_view> gcp_credential_provider_name(
+        TGcpCredentialProviderType::type provider) {
+    switch (provider) {
+    case TGcpCredentialProviderType::DEFAULT:
+        return "DEFAULT";
+    case TGcpCredentialProviderType::COMPUTE_ENGINE:
+        return "COMPUTE_ENGINE";
+    default:
+        return std::nullopt;
+    }
+}
+
+std::string to_upper(std::string text) {
+    std::transform(text.begin(), text.end(), text.begin(),
+                   [](unsigned char c) { return 
static_cast<char>(std::toupper(c)); });
+    return text;
+}
+
+std::string normalize_provider(std::string text) {
+    const auto non_space = [](unsigned char c) { return !std::isspace(c); };
+    text.erase(text.begin(), std::find_if(text.begin(), text.end(), 
non_space));
+    text.erase(std::find_if(text.rbegin(), text.rend(), non_space).base(), 
text.end());
+    return to_upper(std::move(text));
+}
+
+std::string_view pop_dns_label(std::string_view& name) {
+    const auto dot = name.rfind('.');
+    if (dot == std::string_view::npos) {
+        return std::exchange(name, std::string_view {});
+    }
+    auto label = name.substr(dot + 1);
+    name = name.substr(0, dot);
+    return label;
+}
+
+} // namespace
+
+cloud::ObjectStoreInfoPB::Provider vault_storage_provider(const 
cloud::ObjectStoreInfoPB& info) {
+    if (info.has_provider() || info.endpoint().empty()) {
+        return info.provider();
+    }
+    return storage_provider_from_endpoint(info.endpoint());
+}
+
+cloud::ObjectStoreInfoPB::Provider 
storage_provider_from_endpoint(std::string_view endpoint) {
+    std::string url(endpoint);
+    std::transform(url.begin(), url.end(), url.begin(),
+                   [](unsigned char c) { return std::tolower(c); });
+    brpc::URI uri;
+    if (url.find('\0') != std::string::npos || uri.SetHttpURL(url.c_str()) != 
0 ||
+        (!uri.scheme().empty() && uri.scheme() != "http" && uri.scheme() != 
"https")) {
+        return Vault::S3;
+    }
+    std::string_view host = uri.host();
+    if (host.ends_with('.')) {
+        host.remove_suffix(1);
+    }
+    if (host.empty() || 
host.find_first_not_of("abcdefghijklmnopqrstuvwxyz0123456789-.") !=
+                                std::string_view::npos) {
+        return Vault::S3;
+    }
+
+    // Match from the DNS root so virtual-hosted buckets cannot change the 
service or cloud.
+    auto labels = host;
+    auto tld = pop_dns_label(labels);
+    auto domain = pop_dns_label(labels);
+    auto last = pop_dns_label(labels);
+    auto previous = pop_dns_label(labels);
+    auto is_region = [](std::string_view name) {
+        return name.size() > 3 && name[2] == '-' && name[0] >= 'a' && name[0] 
<= 'z' &&
+               name[1] >= 'a' && name[1] <= 'z';
+    };
+    if (domain == "aliyuncs" && tld == "com") {
+        if ((last.starts_with("oss-") && is_region(last.substr(4))) || last == 
"oss-accelerate" ||
+            last == "oss-accelerate-overseas" || (previous == "s3" && 
is_region(last)) ||
+            (last == "oss" && is_region(previous))) {
+            return Vault::OSS;
+        }
+    } else if (domain == "googleapis" && tld == "com" && last == "storage") {
+        return Vault::GCP;
+    } else if (domain == "myqcloud" && tld == "com" && previous == "cos" && 
!last.empty()) {
+        return Vault::COS;
+    } else if (domain == "myhuaweicloud" && tld == "com" && previous == "obs" 
&& !last.empty()) {
+        return Vault::OBS;
+    } else if (domain == "bcebos" && tld == "com" && !last.empty()) {
+        return Vault::BOS;
+    } else if ((domain == "volces" || domain == "ivolces") && tld == "com" &&
+               ((last.starts_with("tos-") && is_region(last.substr(4))) ||
+                (last.starts_with("tos-s3-") && is_region(last.substr(7))))) {
+        return Vault::TOS;
+    } else if (last == "core" && (previous == "blob" || previous == "dfs") && 
!labels.empty() &&
+               ((domain == "windows" && tld == "net") ||
+                (domain == "chinacloudapi" && tld == "cn") ||
+                (domain == "usgovcloudapi" && tld == "net") ||
+                (domain == "cloudapi" && tld == "de"))) {
+        return Vault::AZURE;
+    }
+    return Vault::S3;
+}
+
+AwsCredentialImplementation aws_credential_implementation(std::string_view 
version) {
+    return version == "v2" ? AwsCredentialImplementation::V2 : 
AwsCredentialImplementation::V1;
+}
+
+void normalize_obj_storage_endpoint(ObjStorageClientConfig& client, 
std::string_view scheme) {
+    auto& endpoint = obj_storage_connection(client).endpoint;
+    if (auto* aws = std::get_if<AwsObjStorageConfig>(&client); aws && 
aws->use_sdk_endpoint) {
+        endpoint.clear();
+        aws->sdk_endpoint_http = scheme == "http";
+        return;
+    }
+    if (!endpoint.empty() && endpoint.find("://") == std::string::npos) {
+        endpoint.insert(0,
+                        scheme == "http" && 
!std::holds_alternative<AzureObjStorageConfig>(client)
+                                ? "http://";
+                                : "https://";);
+    }
+}
+
+namespace {
+using Props = ObjStorageProperties;
+using Error = std::optional<std::string>;
+template <class T>
+using Parsed = ObjStorageConfigParseResult<T>;
+
+std::string value(const Props& properties, const char* key) {
+    auto it = properties.find(key);
+    return it == properties.end() ? std::string {} : it->second;
+}
+
+template <class T>
+Error read_number(const Props& props, const char* key, std::optional<T>& 
output) {
+    auto it = props.find(key);
+    if (it == props.end()) {
+        return {};
+    }
+    const auto& text = it->second;
+    int number;
+    auto [end, error] = std::from_chars(text.data(), text.data() + 
text.size(), number);
+    if (error != std::errc {} || end != text.data() + text.size() || number < 
-1) {
+        return fmt::format("invalid {} value \"{}\"", key, text);
+    }
+    // Legacy resources use -1 or 0 for host defaults. Passing a zero request 
timeout
+    // to the SDK would disable stalled-transfer detection instead of keeping 
the BE default.
+    if (number > 0) {
+        output = T(number);
+    }
+    return {};
+}
+Error read_timeouts(const Props& props, ObjStorageConnectionConfig& conn) {
+    if (auto error = read_number(props, "AWS_REQUEST_TIMEOUT_MS", 
conn.request_timeout)) {
+        return error;
+    }
+    return read_number(props, "AWS_CONNECTION_TIMEOUT_MS", 
conn.connect_timeout);
+}
+Error read_bool(const Props& props, const char* key, bool& output) {
+    auto it = props.find(key);
+    if (it == props.end()) {
+        return {};
+    }
+    auto text = it->second;
+    std::transform(text.begin(), text.end(), text.begin(),
+                   [](unsigned char c) { return 
static_cast<char>(std::tolower(c)); });
+    if (text == "true") {
+        output = true;
+    } else if (text == "false") {
+        output = false;
+    } else {
+        return fmt::format("invalid {} value \"{}\"", key, it->second);
+    }
+    return {};
+}
+template <class C>
+Error read_regional_options(const Props& props, C& client) {
+    client.region = value(props, "AWS_REGION");
+    client.connection.endpoint = value(props, "AWS_ENDPOINT");
+    if (auto error = read_bool(props, "use_path_style", 
client.use_path_style)) {
+        return error;
+    }
+    bool override_endpoint = true;
+    if (auto error = read_bool(props, "AWS_NEED_OVERRIDE_ENDPOINT", 
override_endpoint)) {
+        return error;
+    }
+    if (!override_endpoint) {
+        if constexpr (std::is_same_v<C, AwsObjStorageConfig>) {
+            client.use_sdk_endpoint = true;
+        } else {
+            return "AWS SDK automatic endpoint selection requires 
provider=AWS";
+        }
+    }
+    if (auto error = read_number(props, "AWS_MAX_CONNECTIONS", 
client.connection.max_connections)) {
+        return error;
+    }
+    return read_timeouts(props, client.connection);
+}
+
+AwsCredentialChainType chain_source(CredProviderType mode) {
+    switch (mode) {
+    case CredProviderType::Env:
+        return AwsCredentialChainType::Environment;
+    case CredProviderType::SystemProperties:
+        return AwsCredentialChainType::Profile;
+    case CredProviderType::WebIdentity:
+        return AwsCredentialChainType::WebIdentity;
+    case CredProviderType::Container:
+        return AwsCredentialChainType::Container;
+    case CredProviderType::InstanceProfile:
+        return AwsCredentialChainType::InstanceProfile;
+    default:
+        return AwsCredentialChainType::Default;
+    }
+}
+
+Parsed<AccessKeyOrAnonymousObjStorageAuth> parse_access_key_credentials(const 
Props& props) {
+    AccessKeyObjStorageAuth keys {value(props, "AWS_ACCESS_KEY"), value(props, 
"AWS_SECRET_KEY"),
+                                  value(props, "AWS_TOKEN")};
+    if (!keys.access_key.empty() && !keys.secret_key.empty()) {
+        // Some persisted resources also carry AWS role/provider fields. 
Static keys
+        // have always taken precedence; they include temporary credentials 
issued by FE.
+        return AccessKeyOrAnonymousObjStorageAuth {std::move(keys)};
+    }
+    if (keys.access_key.empty() != keys.secret_key.empty()) {
+        return std::string("Incomplete access key credentials");
+    }
+    if (!keys.session_token.empty()) {
+        return std::string("Session token requires access key credentials");
+    }
+    const auto mode = to_upper(value(props, "AWS_CREDENTIALS_PROVIDER_TYPE"));
+    if ((!mode.empty() && mode != "DEFAULT" && mode != "SIMPLE" && mode != 
"ANONYMOUS") ||
+        !value(props, "AWS_ROLE_ARN").empty() || !value(props, 
"AWS_EXTERNAL_ID").empty()) {
+        return std::string(
+                "AWS credential chains and AssumeRole are not supported for 
this provider");
+    }
+    return AccessKeyOrAnonymousObjStorageAuth {AnonymousObjStorageAuth {}};

Review Comment:
   [P2] Preserve the v2 credential chain for explicit OSS resources
   
   An existing S3 cold-storage resource with `provider=OSS`, 
`s3_validity_check=false`, no inline AK/SK, and BE 
`aws_credentials_provider_version=v2` can read a private OSS bucket using keys 
from the AWS environment/profile chain. `S3Resource` accepts this configuration 
and `S3ThriftAdapter` sends `OSS` explicitly. Previously the generic non-Azure 
factory used that chain; this path now returns `AnonymousObjStorageAuth`, so 
after an upgrade/restart or resource push its reads and cooldown uploads fail. 
The `UNKNOWN`-to-AWS correction does not cover this explicit-provider branch. 
Preserve legacy credential resolution in the adapters for these resources.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to