This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 73fea8a5895 branch-4.1: [fix](http) Clamp range reads to file size
#67634 (#67683)
73fea8a5895 is described below
commit 73fea8a58956d020231bdc053029107e8451f37c
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Sep 9 16:36:29 2026 +0800
branch-4.1: [fix](http) Clamp range reads to file size #67634 (#67683)
Cherry-picked from #67634
Co-authored-by: Gabriel <[email protected]>
---
be/src/io/fs/http_file_reader.cpp | 5 ++
be/test/io/fs/http_file_reader_test.cpp | 84 +++++++++++++++++++++++++++++++++
2 files changed, 89 insertions(+)
diff --git a/be/src/io/fs/http_file_reader.cpp
b/be/src/io/fs/http_file_reader.cpp
index dc676e4a9f4..7ae9fc82a3f 100644
--- a/be/src/io/fs/http_file_reader.cpp
+++ b/be/src/io/fs/http_file_reader.cpp
@@ -280,6 +280,11 @@ Status HttpFileReader::read_at_impl(size_t offset, Slice
result, size_t* bytes_r
remaining = std::min<uint64_t>(to_read, left);
}
size_t req_len = (remaining > READ_BUFFER_SIZE) ? remaining :
READ_BUFFER_SIZE;
+ if (_size_known) {
+ // Some servers return the entire object when an otherwise valid range
crosses EOF, so the
+ // advertised file size must remain a hard boundary for speculative
read-ahead.
+ req_len = std::min(req_len, _file_size - offset);
+ }
VLOG(2) << "Issuing HTTP GET request: offset=" << offset << " req_len=" <<
req_len
<< " with_range=" << _range_supported;
diff --git a/be/test/io/fs/http_file_reader_test.cpp
b/be/test/io/fs/http_file_reader_test.cpp
index 08241d5d5c5..1c63ce3fd21 100644
--- a/be/test/io/fs/http_file_reader_test.cpp
+++ b/be/test/io/fs/http_file_reader_test.cpp
@@ -19,9 +19,68 @@
#include <gtest/gtest.h>
+#include <cstdio>
+#include <mutex>
+#include <string>
+#include <utility>
+#include <vector>
+
#include "io/file_factory.h"
+#include "service/http/ev_http_server.h"
+#include "service/http/http_channel.h"
+#include "service/http/http_handler.h"
+#include "service/http/http_headers.h"
+#include "service/http/http_request.h"
namespace doris::io {
+namespace {
+
+class EofSensitiveRangeHandler final : public HttpHandler {
+public:
+ explicit EofSensitiveRangeHandler(std::string content) :
_content(std::move(content)) {}
+
+ void handle(HttpRequest* req) override {
+ if (req->method() == HttpMethod::HEAD) {
+ req->add_output_header(HttpHeaders::CONTENT_LENGTH,
+ std::to_string(_content.size()).c_str());
+ HttpChannel::send_reply(req);
+ return;
+ }
+
+ const std::string range = req->header(HttpHeaders::RANGE);
+ {
+ std::lock_guard<std::mutex> lock(_mutex);
+ _ranges.push_back(range);
+ }
+
+ unsigned long long begin = 0;
+ unsigned long long end = 0;
+ if (std::sscanf(range.c_str(), "bytes=%llu-%llu", &begin, &end) != 2
|| begin > end ||
+ end >= _content.size()) {
+ HttpChannel::send_reply(req, _content);
+ return;
+ }
+
+ const std::string content_range = "bytes " + std::to_string(begin) +
"-" +
+ std::to_string(end) + "/" +
+ std::to_string(_content.size());
+ req->add_output_header(HttpHeaders::CONTENT_RANGE,
content_range.c_str());
+ HttpChannel::send_reply(req, HttpStatus::PARTIAL_CONTENT,
+ _content.substr(begin, end - begin + 1));
+ }
+
+ std::vector<std::string> ranges() {
+ std::lock_guard<std::mutex> lock(_mutex);
+ return _ranges;
+ }
+
+private:
+ std::string _content;
+ std::mutex _mutex;
+ std::vector<std::string> _ranges;
+};
+
+} // namespace
TEST(HttpFileReaderTest, ChunkResponseDisablesFileCache) {
FileSystemProperties properties;
@@ -38,4 +97,29 @@ TEST(HttpFileReaderTest, ChunkResponseDisablesFileCache) {
EXPECT_NE(std::dynamic_pointer_cast<HttpFileReader>(reader.value()),
nullptr);
}
+TEST(HttpFileReaderTest, RangeReadDoesNotCrossKnownEof) {
+ const std::string content =
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+ EofSensitiveRangeHandler handler(content);
+ EvHttpServer server(0);
+ ASSERT_TRUE(server.register_handler(HEAD, "/file", &handler));
+ ASSERT_TRUE(server.register_handler(GET, "/file", &handler));
+ server.start();
+ ASSERT_NE(server.get_real_port(), 0);
+
+ const std::string url = "http://127.0.0.1:"; +
std::to_string(server.get_real_port()) + "/file";
+ auto reader = HttpFileReader::create(url, {}, FileReaderOptions::DEFAULT,
nullptr);
+ ASSERT_TRUE(reader.has_value()) << reader.error();
+
+ constexpr size_t tail_size = 16;
+ char output[tail_size];
+ size_t bytes_read = 0;
+ Status status = reader.value()->read_at(content.size() - tail_size,
Slice(output, tail_size),
+ &bytes_read);
+
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(bytes_read, tail_size);
+ EXPECT_EQ(std::string(output, bytes_read), content.substr(content.size() -
tail_size));
+ EXPECT_EQ(handler.ranges(), (std::vector<std::string> {"bytes=0-0",
"bytes=48-63"}));
+}
+
} // namespace doris::io
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
