github-actions[bot] commented on code in PR #65420:
URL: https://github.com/apache/doris/pull/65420#discussion_r3597019317
##########
be/src/io/fs/azure_obj_storage_client.cpp:
##########
@@ -70,26 +76,98 @@ auto base64_encode_part_num(int part_num) {
return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)});
}
-template <typename Func>
-auto s3_rate_limit(doris::S3RateLimitType op, Func callback) ->
decltype(callback()) {
- if (!doris::config::enable_s3_rate_limiter) {
- return callback();
+const Azure::Core::Context::Key azure_bytes_rate_limit_context_key;
+
+size_t parse_azure_http_range_size(std::string_view value) {
+ constexpr std::string_view prefix = "bytes=";
+ DORIS_CHECK(value.starts_with(prefix));
+ value.remove_prefix(prefix.size());
+
+ const auto separator = value.find('-');
+ DORIS_CHECK_NE(separator, std::string_view::npos);
+ uint64_t first = 0;
+ uint64_t last = 0;
+ const auto [first_end, first_error] =
+ std::from_chars(value.data(), value.data() + separator, first);
+ const auto [last_end, last_error] =
+ std::from_chars(value.data() + separator + 1, value.data() +
value.size(), last);
+ DORIS_CHECK(first_error == std::errc {} && first_end == value.data() +
separator);
+ DORIS_CHECK(last_error == std::errc {} && last_end == value.data() +
value.size());
+ DORIS_CHECK_GE(last, first);
+ DORIS_CHECK_LE(last - first, std::numeric_limits<size_t>::max() - 1);
+ return static_cast<size_t>(last - first + 1);
+}
+
+size_t get_azure_request_bytes(Azure::Core::Http::Request& request,
doris::S3RateLimitType type) {
+ DORIS_CHECK(type == doris::S3RateLimitType::GET || type ==
doris::S3RateLimitType::PUT);
+ if (type == doris::S3RateLimitType::PUT) {
+ if (request.GetMethod() != Azure::Core::Http::HttpMethod::Put) {
+ return 0;
+ }
+ const auto* body = request.GetBodyStream();
+ DORIS_CHECK(body != nullptr);
+ DORIS_CHECK_GE(body->Length(), 0);
+ return static_cast<size_t>(body->Length());
+ }
+ if (request.GetMethod() != Azure::Core::Http::HttpMethod::Get) {
+ return 0;
+ }
+ auto range = request.GetHeader("x-ms-range");
+ DORIS_CHECK(range.HasValue());
+ return parse_azure_http_range_size(range.Value());
+}
+
+class AzureBytesRateLimitPolicy final : public
Azure::Core::Http::Policies::HttpPolicy {
+public:
+ explicit AzureBytesRateLimitPolicy(bool is_internal_bucket)
+ : _is_internal_bucket(is_internal_bucket) {}
+
+ std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy> Clone() const
override {
+ return std::make_unique<AzureBytesRateLimitPolicy>(*this);
+ }
+
+ std::unique_ptr<Azure::Core::Http::RawResponse> Send(
+ Azure::Core::Http::Request& request,
+ Azure::Core::Http::Policies::NextHttpPolicy next_policy,
+ const Azure::Core::Context& context) const override {
+ doris::S3RateLimitType type = doris::S3RateLimitType::UNKNOWN;
+ if (context.TryGetValue(azure_bytes_rate_limit_context_key, type)) {
+ const size_t bytes = get_azure_request_bytes(request, type);
+ if (!doris::acquire_s3_bytes_rate_limit(_is_internal_bucket, type,
bytes)) {
Review Comment:
[P1] Refresh Azure's timestamp after the throttle wait
This client policy runs after Azure Storage's service-specific
`StoragePerRetryPolicy`, which has already set `x-ms-date`, and before
`SharedKeyPolicy` computes `Authorization`. Azure rejects requests more than 15
minutes old. With the allowed 1 byte/s/core setting on four cores, a normal 4
MiB block can wait about 12 days here, so the later signature includes an
already-expired timestamp and every retry repeats the failure. The current S3
thread covers AWS's post-sign handler, but its retry-handler fix does not
affect this Azure pipeline. Please perform admission before `x-ms-date` is
created, or refresh the date after waiting and before SharedKey signs, and test
the actual policy order with a long simulated delay.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
