This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e88dc45b2b6 [fix](be) Fix FileScannerV2 runtime filter profile and
page cache (#65580)
e88dc45b2b6 is described below
commit e88dc45b2b64e8d0ac39e67dfc06951a71accfba
Author: Gabriel <[email protected]>
AuthorDate: Tue Jul 14 19:48:28 2026 +0800
[fix](be) Fix FileScannerV2 runtime filter profile and page cache (#65580)
FileScannerV2 had two issues exposed by an Iceberg web_sales profile:
FileScannerV2 deep-clones runtime-filter wrappers while remapping scan
expressions. RuntimeFilterExpr::clone_node created fresh profile counters, so
filtering performed by the clone was not reflected in the consumer profile and
FilterRows could remain zero. The clone now retains the counters attached to
the original wrapper.
V2 disabled Parquet page cache whenever neither the scan descriptor nor the
file reader supplied an mtime. The cache key now accepts mtime=0 only when the
table format explicitly guarantees that the file path is immutable; mutable
Hive, TVF, local, and generic external files remain uncached to prevent stale
reads after a same-path overwrite.
The immutable-file contract covers all applicable native V2 lakehouse
readers: Iceberg data/delete files, Paimon Parquet/ORC data files, and Hudi
Parquet/ORC base files. Hudi Merge-on-Read log files stay on the JNI merge path
and are not marked immutable because they may be appended.
---
be/src/exprs/runtime_filter_expr.cpp | 9 ++-
be/src/format_v2/parquet/parquet_file_context.cpp | 10 +--
be/src/format_v2/table/hudi_reader.cpp | 12 +++-
be/src/format_v2/table/iceberg_reader.cpp | 8 +++
be/src/format_v2/table/paimon_reader.cpp | 12 +++-
be/src/format_v2/table_reader.cpp | 1 +
be/src/format_v2/table_reader.h | 21 +++++++
be/src/io/file_factory.h | 5 ++
.../runtime_filter_expr_sampling_test.cpp | 35 +++++++++++
be/test/format_v2/parquet/parquet_reader_test.cpp | 72 ++++++++++++++++------
be/test/format_v2/table/hudi_reader_test.cpp | 14 +++++
be/test/format_v2/table/iceberg_reader_test.cpp | 16 +++++
be/test/format_v2/table/paimon_reader_test.cpp | 15 +++++
13 files changed, 203 insertions(+), 27 deletions(-)
diff --git a/be/src/exprs/runtime_filter_expr.cpp
b/be/src/exprs/runtime_filter_expr.cpp
index 584707fddc6..e0f6e7acc1b 100644
--- a/be/src/exprs/runtime_filter_expr.cpp
+++ b/be/src/exprs/runtime_filter_expr.cpp
@@ -73,9 +73,12 @@ Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr)
const {
DORIS_CHECK(_impl != nullptr);
VExprSPtr cloned_impl;
RETURN_IF_ERROR(_impl->deep_clone(&cloned_impl));
- *cloned_expr = RuntimeFilterExpr::create_shared(clone_texpr_node(),
std::move(cloned_impl),
- _ignore_thredhold,
_null_aware, _filter_id,
- _sampling_frequency);
+ auto cloned_runtime_filter = RuntimeFilterExpr::create_shared(
+ clone_texpr_node(), std::move(cloned_impl), _ignore_thredhold,
_null_aware, _filter_id,
+ _sampling_frequency);
+ cloned_runtime_filter->attach_profile_counter(_rf_input_rows,
_rf_filter_rows,
+ _always_true_filter_rows);
+ *cloned_expr = std::move(cloned_runtime_filter);
return Status::OK();
}
diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp
b/be/src/format_v2/parquet/parquet_file_context.cpp
index d80dc58181d..52151c7942a 100644
--- a/be/src/format_v2/parquet/parquet_file_context.cpp
+++ b/be/src/format_v2/parquet/parquet_file_context.cpp
@@ -192,10 +192,12 @@ std::string build_page_cache_file_key(const
io::FileReader& file_reader,
const io::FileDescription&
file_description) {
const int64_t mtime =
file_description.mtime != 0 ? file_description.mtime :
file_reader.mtime();
- if (mtime == 0) {
- // StoragePageCache is process-global. A key with only path + unknown
mtime can outlive a
- // rewritten local test file, or any external file whose version was
not propagated. Disable
- // v2 parquet page cache until the scan descriptor carries a stable
object version.
+ if (mtime == 0 && !file_description.is_immutable) {
+ // mtime == 0 means "unknown version", not the Unix epoch. V1
historically caches such a
+ // file under path::0, but copying that behavior for every V2 file is
unsafe: a mutable file
+ // can be overwritten with different bytes while retaining both its
path and size, causing
+ // process-global page cache entries to return stale data. Only
callers that explicitly
+ // guarantee path immutability may use the mtime=0 cache key below.
return {};
}
const int64_t file_size = file_description.file_size >= 0
diff --git a/be/src/format_v2/table/hudi_reader.cpp
b/be/src/format_v2/table/hudi_reader.cpp
index f37d24bd888..4294d51d043 100644
--- a/be/src/format_v2/table/hudi_reader.cpp
+++ b/be/src/format_v2/table/hudi_reader.cpp
@@ -34,7 +34,17 @@ Status HudiReader::prepare_split(const
format::SplitReadOptions& options) {
options.current_range.table_format_params.hudi_params.__isset.schema_id) {
_split_schema_id =
options.current_range.table_format_params.hudi_params.schema_id;
}
- return format::TableReader::prepare_split(options);
+ RETURN_IF_ERROR(format::TableReader::prepare_split(options));
+ if (current_split_pruned()) {
+ return Status::OK();
+ }
+ // This native reader only receives Hudi base-file splits that do not
require merging delta
+ // logs. Hudi creates a new versioned base file for updates and compaction
instead of modifying
+ // an existing Parquet/ORC base file in place, so missing mtime does not
make its page-cache key
+ // ambiguous. Merge-on-read log files remain on the JNI path because they
may be appended and
+ // must never be marked immutable here.
+ mark_current_data_file_immutable();
+ return Status::OK();
}
format::TableColumnMappingMode HudiReader::mapping_mode() const {
diff --git a/be/src/format_v2/table/iceberg_reader.cpp
b/be/src/format_v2/table/iceberg_reader.cpp
index 42f78d38c46..b817a6d1ead 100644
--- a/be/src/format_v2/table/iceberg_reader.cpp
+++ b/be/src/format_v2/table/iceberg_reader.cpp
@@ -247,6 +247,11 @@ Status IcebergTableReader::prepare_split(const
format::SplitReadOptions& options
if (current_split_pruned()) {
return Status::OK();
}
+ // Iceberg data files are immutable once referenced by a snapshot; updates
create new data files
+ // at new paths instead of overwriting existing files. This lets the
Parquet V2 reader use page
+ // cache when the scan range does not carry an mtime, without extending
V1's path::0 behavior to
+ // mutable Hive/local files.
+ mark_current_data_file_immutable();
if (_is_table_level_count_active()) {
return Status::OK();
}
@@ -467,6 +472,9 @@ std::unique_ptr<io::FileDescription>
IcebergTableReader::_delete_file_descriptio
file_description->file_size = range.__isset.file_size ? range.file_size :
-1;
file_description->range_start_offset = range.__isset.start_offset ?
range.start_offset : 0;
file_description->range_size = range.__isset.size ? range.size : -1;
+ // Iceberg delete files follow the same immutable-file contract as data
files: a snapshot
+ // references a fixed object and later changes publish a new file rather
than replacing it.
+ file_description->is_immutable = true;
if (range.__isset.fs_name) {
file_description->fs_name = range.fs_name;
}
diff --git a/be/src/format_v2/table/paimon_reader.cpp
b/be/src/format_v2/table/paimon_reader.cpp
index 236daac118a..258bbb5c021 100644
--- a/be/src/format_v2/table/paimon_reader.cpp
+++ b/be/src/format_v2/table/paimon_reader.cpp
@@ -38,7 +38,17 @@ Status PaimonReader::prepare_split(const
format::SplitReadOptions& options) {
if (paimon_params.__isset.schema_id) {
_split_schema_id = paimon_params.schema_id;
}
- return format::TableReader::prepare_split(options);
+ RETURN_IF_ERROR(format::TableReader::prepare_split(options));
+ if (current_split_pruned()) {
+ return Status::OK();
+ }
+ // Paimon commits data-file changes by adding and logically deleting files
in snapshots.
+ // Compaction also writes replacement files and commits them in a new
snapshot instead of
+ // modifying an existing Parquet/ORC file in place. Native Paimon data
files are therefore
+ // safe to cache by path and size when the split does not provide mtime.
Serialized JNI splits
+ // do not reach this reader.
+ mark_current_data_file_immutable();
+ return Status::OK();
}
format::TableColumnMappingMode PaimonReader::mapping_mode() const {
diff --git a/be/src/format_v2/table_reader.cpp
b/be/src/format_v2/table_reader.cpp
index 5fb32b84d0a..f30a1d567cf 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -117,6 +117,7 @@ std::string current_file_debug_string(const
std::unique_ptr<ScanTask>& task) {
out << "FileDescription{path=" << file.path << ", file_size=" <<
file.file_size
<< ", range_start_offset=" << file.range_start_offset << ",
range_size=" << file.range_size
<< ", mtime=" << file.mtime << ", fs_name=" << file.fs_name
+ << ", is_immutable=" << file.is_immutable
<< ", file_cache_admission=" << file.file_cache_admission << "}";
return out.str();
}
diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h
index a875058103c..8d99195c21d 100644
--- a/be/src/format_v2/table_reader.h
+++ b/be/src/format_v2/table_reader.h
@@ -185,6 +185,14 @@ public:
#ifdef BE_TEST
size_t TEST_batch_size() const { return _batch_size; }
+ bool TEST_current_data_file_is_immutable() const {
+ DORIS_CHECK(_current_task != nullptr);
+ DORIS_CHECK(_current_task->data_file != nullptr);
+ DORIS_CHECK(_current_file_description.has_value());
+ DORIS_CHECK(_current_task->data_file->is_immutable ==
+ _current_file_description->is_immutable);
+ return _current_task->data_file->is_immutable;
+ }
#endif
// Prepare for reading a new split/task.
@@ -318,6 +326,19 @@ public:
}
protected:
+ // TableReader keeps the active file description both in the scan task and
separately for
+ // creating the physical reader. Table-format readers must update both
copies when their
+ // snapshot protocol guarantees that a file path is never overwritten with
different bytes.
+ // This guarantee lets readers safely build cache keys without mtime; it
must not be used for
+ // ordinary Hive/TVF files whose paths may be overwritten in place.
+ void mark_current_data_file_immutable() {
+ DORIS_CHECK(_current_task != nullptr);
+ DORIS_CHECK(_current_task->data_file != nullptr);
+ DORIS_CHECK(_current_file_description.has_value());
+ _current_task->data_file->is_immutable = true;
+ _current_file_description->is_immutable = true;
+ }
+
std::optional<ColumnDefinition>
_find_current_table_column_by_field_id(int32_t field_id,
DataTypePtr type) const;
diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h
index 33595313b92..3ff7f3decaf 100644
--- a/be/src/io/file_factory.h
+++ b/be/src/io/file_factory.h
@@ -70,6 +70,11 @@ struct FileDescription {
// modification time of this file.
// 0 means unset.
int64_t mtime = 0;
+ // A correctness promise that the bytes at this path never change. This is
only used when
+ // mtime is unavailable: immutable table formats may still key
process-global caches by
+ // fs/path/size, while mutable files must not do so because the same path
and size can refer to
+ // different contents after an overwrite. Generic Hive/local files must
keep the default false.
+ bool is_immutable = false;
// for hdfs, eg: hdfs://nameservices1/
// because for a hive table, differenet partitions may have different
// locations(or fs), so different files may have different fs.
diff --git a/be/test/exec/runtime_filter/runtime_filter_expr_sampling_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_expr_sampling_test.cpp
index b3e512734c6..8af2575ff15 100644
--- a/be/test/exec/runtime_filter/runtime_filter_expr_sampling_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_expr_sampling_test.cpp
@@ -224,4 +224,39 @@ TEST_F(RuntimeFilterExprSamplingTest,
deep_clone_clones_impl_tree) {
EXPECT_EQ(cloned_slot->column_name(), "c0");
}
+// FileScannerV2 deep-clones runtime filters when rewriting scanner slot ids
to table-global
+// indices. Both the scanner wrapper and the localized clone contribute to one
runtime-filter
+// profile, so the clone must retain the counters attached by
RuntimeFilterConsumer.
+TEST_F(RuntimeFilterExprSamplingTest, deep_clone_shares_profile_counters) {
+ auto bool_type = TTypeDescBuilder()
+ .set_types(TTypeNodeBuilder()
+
.set_type(TTypeNodeType::SCALAR)
+
.set_scalar_type(TPrimitiveType::BOOLEAN)
+ .build())
+ .build();
+ TExprNode node = TExprNodeBuilder(TExprNodeType::IN_PRED, bool_type,
0).build();
+ node.in_predicate.__set_is_not_in(false);
+ node.__set_opcode(TExprOpcode::FILTER_IN);
+ node.__set_is_nullable(false);
+
+ auto slot = VSlotRef::create_shared(/*slot_id=*/0, /*column_id=*/0,
/*column_uniq_id=*/10,
+ std::make_shared<DataTypeInt32>(),
"c0");
+ auto impl = VDirectInPredicate::create_shared(node, nullptr);
+ impl->add_child(slot);
+ auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0.4, false,
/*filter_id=*/1);
+ auto input_rows = std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT,
0);
+ auto filter_rows = std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT,
0);
+ auto always_true_rows =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
+ wrapper->attach_profile_counter(input_rows, filter_rows, always_true_rows);
+
+ VExprSPtr cloned_expr;
+ ASSERT_TRUE(wrapper->deep_clone(&cloned_expr).ok());
+ auto* cloned_wrapper = dynamic_cast<RuntimeFilterExpr*>(cloned_expr.get());
+ ASSERT_NE(cloned_wrapper, nullptr);
+
+ EXPECT_EQ(cloned_wrapper->predicate_input_rows_counter(), input_rows);
+ EXPECT_EQ(cloned_wrapper->predicate_filtered_rows_counter(), filter_rows);
+ EXPECT_EQ(cloned_wrapper->predicate_always_true_rows_counter(),
always_true_rows);
+}
+
} // namespace doris
diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp
b/be/test/format_v2/parquet/parquet_reader_test.cpp
index 71d1cc29175..796123832c6 100644
--- a/be/test/format_v2/parquet/parquet_reader_test.cpp
+++ b/be/test/format_v2/parquet/parquet_reader_test.cpp
@@ -1135,7 +1135,8 @@ protected:
int64_t range_start_offset = 0, int64_t range_size = -1,
RuntimeProfile* profile = nullptr, bool
enable_mapping_timestamp_tz = false,
std::shared_ptr<io::IOContext> io_ctx = nullptr,
- std::optional<format::GlobalRowIdContext> global_rowid_context =
std::nullopt) const {
+ std::optional<format::GlobalRowIdContext> global_rowid_context =
std::nullopt,
+ bool is_immutable = false) const {
auto system_properties = std::make_shared<io::FileSystemProperties>();
system_properties->system_type = TFileType::FILE_LOCAL;
auto file_description = std::make_unique<io::FileDescription>();
@@ -1143,6 +1144,7 @@ protected:
file_description->file_size =
static_cast<int64_t>(std::filesystem::file_size(_file_path));
file_description->range_start_offset = range_start_offset;
file_description->range_size = range_size;
+ file_description->is_immutable = is_immutable;
return std::make_unique<format::parquet::ParquetReader>(
system_properties, file_description, std::move(io_ctx),
profile,
global_rowid_context, enable_mapping_timestamp_tz);
@@ -1504,11 +1506,46 @@ TEST_F(NewParquetReaderTest, ReadMultipleRowGroups) {
EXPECT_EQ(values, std::vector<std::string>({"one", "two", "three", "four",
"five"}));
}
-TEST_F(NewParquetReaderTest,
RewriteSameLocalPathDoesNotReuseUnknownMtimePageCache) {
+TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) {
+ _file_path = (_test_dir / "mutable_unknown_mtime.parquet").string();
+ write_parquet_file(_file_path);
+
+ RuntimeProfile profile("new_parquet_reader_mutable_unknown_mtime");
+ auto reader = create_reader(0, -1, &profile);
+ TQueryOptions query_options;
+ query_options.__set_enable_parquet_file_page_cache(true);
+ RuntimeState state {query_options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(0),
field_projection(1)};
+ ASSERT_TRUE(reader->open(request).ok());
+
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block(schema);
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ }
+
+ ASSERT_NE(profile.get_counter("PageReadCount"), nullptr);
+ ASSERT_NE(profile.get_counter("PageCacheWriteCount"), nullptr);
+ EXPECT_EQ(profile.get_counter("PageReadCount")->value(), 0);
+ EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0);
+}
+
+TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) {
+ _file_path = (_test_dir / "unknown_mtime_page_cache.parquet").string();
+ write_parquet_file(_file_path);
+
RuntimeProfile first_profile("new_parquet_reader_first_unknown_mtime");
{
- auto reader = create_reader(0, -1, &first_profile);
- RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto reader = create_reader(0, -1, &first_profile, false, nullptr,
std::nullopt, true);
+ TQueryOptions query_options;
+ query_options.__set_enable_parquet_file_page_cache(true);
+ RuntimeState state {query_options, TQueryGlobals()};
ASSERT_TRUE(reader->init(&state).ok());
std::vector<format::ColumnDefinition> schema;
@@ -1527,15 +1564,14 @@ TEST_F(NewParquetReaderTest,
RewriteSameLocalPathDoesNotReuseUnknownMtimePageCac
ASSERT_NE(first_profile.get_counter("PageReadCount"), nullptr);
ASSERT_NE(first_profile.get_counter("PageCacheWriteCount"), nullptr);
- EXPECT_EQ(first_profile.get_counter("PageReadCount")->value(), 0);
- EXPECT_EQ(first_profile.get_counter("PageCacheWriteCount")->value(), 0);
+ EXPECT_GT(first_profile.get_counter("PageReadCount")->value(), 0);
+ EXPECT_GT(first_profile.get_counter("PageCacheWriteCount")->value(), 0);
- // LocalFileReader reports mtime as 0. Rewriting the same path must not
reuse page-cache bytes
- // from the previous physical file, even when the query option enables
parquet file page cache.
- write_int_pair_parquet_file(_file_path);
RuntimeProfile second_profile("new_parquet_reader_second_unknown_mtime");
- auto reader = create_reader(0, -1, &second_profile);
- RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto reader = create_reader(0, -1, &second_profile, false, nullptr,
std::nullopt, true);
+ TQueryOptions query_options;
+ query_options.__set_enable_parquet_file_page_cache(true);
+ RuntimeState state {query_options, TQueryGlobals()};
ASSERT_TRUE(reader->init(&state).ok());
std::vector<format::ColumnDefinition> schema;
@@ -1545,7 +1581,7 @@ TEST_F(NewParquetReaderTest,
RewriteSameLocalPathDoesNotReuseUnknownMtimePageCac
ASSERT_TRUE(reader->open(request).ok());
std::vector<int32_t> ids;
- std::vector<int32_t> scores;
+ std::vector<std::string> values;
bool eof = false;
while (!eof) {
Block block = build_file_block(schema);
@@ -1555,19 +1591,19 @@ TEST_F(NewParquetReaderTest,
RewriteSameLocalPathDoesNotReuseUnknownMtimePageCac
continue;
}
const auto& id_column = nullable_nested_column<ColumnInt32>(block, 0);
- const auto& score_column = nullable_nested_column<ColumnInt32>(block,
1);
+ const auto& value_column = nullable_nested_column<ColumnString>(block,
1);
for (size_t row = 0; row < rows; ++row) {
ids.push_back(id_column.get_element(row));
- scores.push_back(score_column.get_element(row));
+ values.push_back(value_column.get_data_at(row).to_string());
}
}
EXPECT_EQ(ids, std::vector<int32_t>({1, 2, 3, 4, 5}));
- EXPECT_EQ(scores, std::vector<int32_t>({1, 2, 3, 4, 5}));
+ EXPECT_EQ(values, std::vector<std::string>({"one", "two", "three", "four",
"five"}));
ASSERT_NE(second_profile.get_counter("PageReadCount"), nullptr);
- ASSERT_NE(second_profile.get_counter("PageCacheWriteCount"), nullptr);
- EXPECT_EQ(second_profile.get_counter("PageReadCount")->value(), 0);
- EXPECT_EQ(second_profile.get_counter("PageCacheWriteCount")->value(), 0);
+ ASSERT_NE(second_profile.get_counter("PageCacheHitCount"), nullptr);
+ EXPECT_GT(second_profile.get_counter("PageReadCount")->value(), 0);
+ EXPECT_GT(second_profile.get_counter("PageCacheHitCount")->value(), 0);
}
TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection)
{
diff --git a/be/test/format_v2/table/hudi_reader_test.cpp
b/be/test/format_v2/table/hudi_reader_test.cpp
index 77cd75bc948..28c396371e4 100644
--- a/be/test/format_v2/table/hudi_reader_test.cpp
+++ b/be/test/format_v2/table/hudi_reader_test.cpp
@@ -178,6 +178,20 @@ TEST(HudiReaderTest,
ResetsSplitSchemaIdBeforePreparingNextSplit) {
EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_NAME);
}
+TEST(HudiReaderTest, NativeBaseFilesAreMarkedImmutableForPageCache) {
+ hudi::HudiReader reader;
+
+ for (const auto format : {FileFormat::PARQUET, FileFormat::ORC}) {
+ SplitReadOptions split_options;
+ split_options.current_split_format = format;
+ split_options.current_range.__set_path("hudi-base-file");
+
split_options.current_range.__set_table_format_params(hudi_table_format_desc(100));
+
+ ASSERT_TRUE(reader.prepare_split(split_options).ok());
+ EXPECT_TRUE(reader.TEST_current_data_file_is_immutable());
+ }
+}
+
TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) {
hudi::HudiHybridReader reader;
reader.TEST_install_batch_size_children();
diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp
b/be/test/format_v2/table/iceberg_reader_test.cpp
index 597816cab51..0831c98750e 100644
--- a/be/test/format_v2/table/iceberg_reader_test.cpp
+++ b/be/test/format_v2/table/iceberg_reader_test.cpp
@@ -213,6 +213,12 @@ public:
return customize_file_scan_request(request);
}
+ bool current_data_file_is_immutable() const {
+ DORIS_CHECK(_current_task != nullptr);
+ DORIS_CHECK(_current_task->data_file != nullptr);
+ return _current_task->data_file->is_immutable;
+ }
+
private:
std::unique_ptr<TQueryOptions> _query_options;
std::unique_ptr<TQueryGlobals> _query_globals;
@@ -3111,5 +3117,15 @@ TEST(IcebergV2ReaderTest,
RowPositionDeletePredicateColumnIsNotRepeatedAsOutputC
EXPECT_NE(request.delete_conjuncts[0], nullptr);
}
+TEST(IcebergV2ReaderTest, DataFileIsMarkedImmutableForPageCache) {
+ std::vector<ColumnDefinition> projected_columns;
+ projected_columns.push_back(make_table_column(0, "id",
std::make_shared<DataTypeInt32>()));
+
+ IcebergTableReaderScanRequestTestHelper reader;
+
ASSERT_TRUE(reader.init_for_scan_request_test(std::move(projected_columns)).ok());
+
+ EXPECT_TRUE(reader.current_data_file_is_immutable());
+}
+
} // namespace
} // namespace doris::format
diff --git a/be/test/format_v2/table/paimon_reader_test.cpp
b/be/test/format_v2/table/paimon_reader_test.cpp
index a56ebcc5e87..6a3f20fac52 100644
--- a/be/test/format_v2/table/paimon_reader_test.cpp
+++ b/be/test/format_v2/table/paimon_reader_test.cpp
@@ -549,6 +549,21 @@ TEST(PaimonReaderTest,
ResetsSplitSchemaIdBeforePreparingNextSplit) {
EXPECT_EQ(reader.TEST_mapping_mode(), TableColumnMappingMode::BY_NAME);
}
+TEST(PaimonReaderTest, NativeDataFilesAreMarkedImmutableForPageCache) {
+ paimon::PaimonReader reader;
+
+ for (const auto format : {FileFormat::PARQUET, FileFormat::ORC}) {
+ SplitReadOptions split_options;
+ split_options.current_split_format = format;
+ split_options.current_range.__set_path("paimon-data-file");
+ split_options.current_range.__set_table_format_params(
+ make_paimon_schema_table_format_desc(100));
+
+ ASSERT_TRUE(reader.prepare_split(split_options).ok());
+ EXPECT_TRUE(reader.TEST_current_data_file_is_immutable());
+ }
+}
+
// Scenario: Paimon reader should parse its bitmap deletion vector and let
TableReader apply the
// generated row-position delete predicate before returning table rows.
TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
