This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bf58c2c9ac5 [fix](packed-file) Honor write_file_cache option for
packed small files (#69693)
bf58c2c9ac5 is described below
commit bf58c2c9ac5484edb9deac2cd509b1217299c70d
Author: Xin Liao <[email protected]>
AuthorDate: Wed Oct 28 11:56:06 2026 +0800
[fix](packed-file) Honor write_file_cache option for packed small files
(#69693)
### What problem does this PR solve?
Issue Number: None
Related PR: #59892, #60411, #64995
Problem Summary:
When packed file is enabled, the first segment file (`.dat`) and the
index file (`.idx`) of a rowset are written through `PackedFileSystem`.
The packed path decides whether to cache each small file from
`PackedAppendContext::write_file_cache`, which defaults to `true`.
`RowsetWriterContext` never sets this field.
---
be/src/io/fs/packed_file_system.cpp | 5 +++--
be/src/io/fs/packed_file_writer.h | 4 ----
be/test/io/fs/packed_file_system_test.cpp | 28 ++++++++++++++++++++++++++++
be/test/io/fs/packed_file_writer_test.cpp | 4 ++--
4 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/be/src/io/fs/packed_file_system.cpp
b/be/src/io/fs/packed_file_system.cpp
index ae235705f4c..fe45970e882 100644
--- a/be/src/io/fs/packed_file_system.cpp
+++ b/be/src/io/fs/packed_file_system.cpp
@@ -21,7 +21,6 @@
#include <string_view>
#include <utility>
-#include "cloud/config.h"
#include "common/status.h"
#include "io/fs/file_reader.h"
#include "io/fs/packed_file_reader.h"
@@ -99,8 +98,10 @@ Status PackedFileSystem::create_file_impl(const Path& file,
FileWriterPtr* write
return Status::OK();
}
+ // Follow the caller's file cache decision so that packed small files
behave the same as
+ // files written directly, e.g. a load with disable_file_cache=true must
not populate cache.
auto append_info = _append_info;
- if (config::enable_file_cache_write_index_file_only && opts != nullptr) {
+ if (opts != nullptr) {
append_info.write_file_cache = opts->write_file_cache;
}
diff --git a/be/src/io/fs/packed_file_writer.h
b/be/src/io/fs/packed_file_writer.h
index f571b188acb..9f03777c59b 100644
--- a/be/src/io/fs/packed_file_writer.h
+++ b/be/src/io/fs/packed_file_writer.h
@@ -57,10 +57,6 @@ public:
// Returns empty index if file is not in merge file
Status get_packed_slice_location(PackedSliceLocation* location) const
override;
-#ifdef BE_TEST
- size_t buffer_capacity_for_test() const { return _buffer.capacity(); }
-#endif
-
private:
void _release_buffer();
diff --git a/be/test/io/fs/packed_file_system_test.cpp
b/be/test/io/fs/packed_file_system_test.cpp
index b2007941ab7..f4333a1bcab 100644
--- a/be/test/io/fs/packed_file_system_test.cpp
+++ b/be/test/io/fs/packed_file_system_test.cpp
@@ -324,6 +324,34 @@ TEST_F(PackedFileSystemTest,
NonzeroFirstSegmentUsesPackedWriter) {
EXPECT_NE(dynamic_cast<PackedFileWriter*>(first_segment_index_writer.get()),
nullptr);
}
+TEST_F(PackedFileSystemTest, CreateFileFollowsWriteFileCacheOption) {
+ ASSERT_FALSE(config::enable_file_cache_write_index_file_only);
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ // A load with disable_file_cache=true must not write packed small files
to file cache.
+ FileWriterOptions no_cache_opts {.write_file_cache = false};
+ FileWriterPtr no_cache_writer;
+ ASSERT_TRUE(
+ merge_fs.create_file(Path("rowset_1_0.dat"), &no_cache_writer,
&no_cache_opts).ok());
+ auto* packed_writer =
dynamic_cast<PackedFileWriter*>(no_cache_writer.get());
+ ASSERT_NE(packed_writer, nullptr);
+ EXPECT_FALSE(packed_writer->_append_info.write_file_cache);
+
+ FileWriterOptions cache_opts {.write_file_cache = true};
+ FileWriterPtr cache_writer;
+ ASSERT_TRUE(merge_fs.create_file(Path("rowset_1_0.idx"), &cache_writer,
&cache_opts).ok());
+ packed_writer = dynamic_cast<PackedFileWriter*>(cache_writer.get());
+ ASSERT_NE(packed_writer, nullptr);
+ EXPECT_TRUE(packed_writer->_append_info.write_file_cache);
+
+ // Without options, keep the write_file_cache from append context.
+ FileWriterPtr default_writer;
+ ASSERT_TRUE(merge_fs.create_file(Path("rowset_1_0.dat"), &default_writer,
nullptr).ok());
+ packed_writer = dynamic_cast<PackedFileWriter*>(default_writer.get());
+ ASSERT_NE(packed_writer, nullptr);
+ EXPECT_TRUE(packed_writer->_append_info.write_file_cache);
+}
+
TEST_F(PackedFileSystemTest, OpenFileNotInMergeFile) {
PackedFileSystem merge_fs(_inner_fs, _append_info);
diff --git a/be/test/io/fs/packed_file_writer_test.cpp
b/be/test/io/fs/packed_file_writer_test.cpp
index d5b7dc2da79..bf0ec042f3e 100644
--- a/be/test/io/fs/packed_file_writer_test.cpp
+++ b/be/test/io/fs/packed_file_writer_test.cpp
@@ -181,14 +181,14 @@ TEST_F(PackedFileWriterTest,
SwitchToDirectWriteReleasesBufferedMemory) {
std::string small_data(80, 'a');
Slice small_slice(small_data);
ASSERT_TRUE(writer.appendv(&small_slice, 1).ok());
- EXPECT_GT(writer.buffer_capacity_for_test(), default_capacity);
+ EXPECT_GT(writer._buffer.capacity(), default_capacity);
std::string large_data(30, 'b');
Slice large_slice(large_data);
ASSERT_TRUE(writer.appendv(&large_slice, 1).ok());
EXPECT_GT(inner_writer_ptr->append_calls(), 0);
- EXPECT_LE(writer.buffer_capacity_for_test(), default_capacity);
+ EXPECT_LE(writer._buffer.capacity(), default_capacity);
}
TEST_F(PackedFileWriterTest, CloseAsync) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
