This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d6b7b8363f1 [Improvement](file scan) Keep File Scanner V2 predicate
pruning enabled (#66357)
d6b7b8363f1 is described below
commit d6b7b8363f107cbbf50349584c5aa5fab6d45d68
Author: Gabriel <[email protected]>
AuthorDate: Mon Aug 3 09:32:27 2026 +0800
[Improvement](file scan) Keep File Scanner V2 predicate pruning enabled
(#66357)
File Scanner V2 inherited the legacy session gates for runtime-filter
partition pruning and Parquet expression ZoneMap pruning. Disabling either
switch therefore also disabled the corresponding V2 pruning path, even though
V2 owns an independent pruning pipeline.
This change keeps both pruning paths enabled for File Scanner V2 regardless
of the legacy session switch values. File Scanner V1 behavior is unchanged.
---
be/src/exec/scan/file_scanner_v2.cpp | 5 +-
be/src/format_v2/parquet/parquet_statistics.cpp | 10 +--
be/test/exec/scan/file_scanner_v2_test.cpp | 60 ++++++++++++++++++
.../format_v2/parquet/parquet_statistics_test.cpp | 74 ++++++++++++++++++++++
.../java/org/apache/doris/qe/SessionVariable.java | 16 ++++-
gensrc/thrift/PaloInternalService.thrift | 5 +-
6 files changed, 160 insertions(+), 10 deletions(-)
diff --git a/be/src/exec/scan/file_scanner_v2.cpp
b/be/src/exec/scan/file_scanner_v2.cpp
index 6859fe8d543..4d513acbb9d 100644
--- a/be/src/exec/scan/file_scanner_v2.cpp
+++ b/be/src/exec/scan/file_scanner_v2.cpp
@@ -632,7 +632,10 @@ Status FileScannerV2::_prepare_table_reader_split(const
TFileRangeDesc& range,
VExprContextSPtrs conjuncts;
RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts));
VExprContextSPtrs partition_prune_conjuncts;
- if (_state->query_options().enable_runtime_filter_partition_prune) {
+ if (!partition_values.empty()) {
+ // A split without partition constants cannot be pruned here, so avoid
cloning every
+ // conjunct solely for a consumer that must return immediately.
FileScannerV2 otherwise
+ // keeps safe partition pruning enabled independently of the legacy
session gate.
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
}
RETURN_IF_ERROR(_table_reader->prepare_split({
diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp
b/be/src/format_v2/parquet/parquet_statistics.cpp
index 7d1afb39240..7f35b087910 100644
--- a/be/src/format_v2/parquet/parquet_statistics.cpp
+++ b/be/src/format_v2/parquet/parquet_statistics.cpp
@@ -452,11 +452,11 @@ std::optional<format::LocalColumnId>
file_column_id_by_block_position(
return std::nullopt;
}
-bool has_expr_zonemap_filter(const format::FileScanRequest& request,
- const RuntimeState* runtime_state) {
- if (!expr_zonemap::is_expr_zonemap_filter_enabled(runtime_state)) {
- return false;
- }
+bool has_expr_zonemap_filter(const format::FileScanRequest& request, const
RuntimeState*) {
+ // FileScannerV2 metadata pruning is a fixed part of its scan pipeline and
must not inherit
+ // the legacy scanner's expression ZoneMap session gate.
+ // TODO: Fence metadata pruning at the first unsafe/error-preserving
conjunct so a later
+ // ZoneMap predicate cannot bypass its row-level evaluation.
for (const auto& conjunct : request.conjuncts) {
if (conjunct != nullptr && conjunct->root() != nullptr &&
conjunct->root()->can_evaluate_zonemap_filter()) {
diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp
b/be/test/exec/scan/file_scanner_v2_test.cpp
index 0ca3213170d..353c08043ad 100644
--- a/be/test/exec/scan/file_scanner_v2_test.cpp
+++ b/be/test/exec/scan/file_scanner_v2_test.cpp
@@ -144,6 +144,18 @@ private:
std::shared_ptr<RetryableCloseState> _state;
};
+class CapturingSplitTableReader final : public format::TableReader {
+public:
+ Status prepare_split(const format::SplitReadOptions& options) override {
+ conjunct_count = options.conjuncts.has_value() ?
options.conjuncts->size() : 0;
+ partition_prune_conjunct_count =
options.partition_prune_conjuncts.size();
+ return Status::OK();
+ }
+
+ size_t conjunct_count = 0;
+ size_t partition_prune_conjunct_count = 0;
+};
+
VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const
std::string& name) {
return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type),
name);
}
@@ -478,6 +490,54 @@ TEST(FileScannerV2Test,
FailedTableReaderCloseCanBeRetriedThroughScanner) {
EXPECT_EQ(close_state->close_calls, 2);
}
+TEST(FileScannerV2Test,
PartitionPruningRemainsEnabledWhenSessionSwitchIsFalse) {
+ TQueryOptions query_options;
+ query_options.__set_enable_runtime_filter_partition_prune(false);
+ RuntimeState state {query_options, TQueryGlobals()};
+ ObjectPool pool;
+ TDescriptorTable thrift_descriptors;
+ TTupleDescriptor tuple_descriptor;
+ tuple_descriptor.id = 0;
+ tuple_descriptor.byteSize = 0;
+ tuple_descriptor.numNullBytes = 0;
+ thrift_descriptors.tupleDescriptors.push_back(tuple_descriptor);
+ DescriptorTbl* descriptors = nullptr;
+ ASSERT_TRUE(DescriptorTbl::create(&pool, thrift_descriptors,
&descriptors).ok());
+ TPlanNode plan_node;
+ plan_node.node_id = 0;
+ plan_node.node_type = TPlanNodeType::FILE_SCAN_NODE;
+ plan_node.num_children = 0;
+ plan_node.limit = -1;
+ plan_node.row_tuples.push_back(0);
+ plan_node.file_scan_node.tuple_id = 0;
+ plan_node.__isset.file_scan_node = true;
+ FileScanOperatorX parent(&pool, plan_node, 0, *descriptors, 1);
+ FileScanLocalState local_state(&state, &parent);
+ RuntimeProfile profile("file_scanner_v2_partition_prune");
+ auto table_reader = std::make_unique<CapturingSplitTableReader>();
+ auto* captured = table_reader.get();
+ FileScannerV2 scanner(&state, &profile, std::move(table_reader));
+ scanner._local_state = &local_state;
+
+ TFileScanRangeParams params;
+ params.__set_format_type(TFileFormatType::FORMAT_PARQUET);
+ scanner._params = ¶ms;
+ scanner._slot_id_to_global_index.emplace(7, format::GlobalIndex(0));
+ scanner._conjuncts = {VExprContext::create_shared(
+ slot_ref(7, 7, std::make_shared<DataTypeInt32>(),
"partition_col"))};
+
+ const auto range = range_with_format("hive",
TFileFormatType::FORMAT_PARQUET);
+ ASSERT_TRUE(scanner._prepare_table_reader_split(range, {}).ok());
+ EXPECT_EQ(captured->conjunct_count, 1);
+ EXPECT_EQ(captured->partition_prune_conjunct_count, 0);
+
+ ASSERT_TRUE(scanner._prepare_table_reader_split(
+ range, {{"partition_col",
Field::create_field<TYPE_INT>(1)}})
+ .ok());
+ EXPECT_EQ(captured->conjunct_count, 1);
+ EXPECT_EQ(captured->partition_prune_conjunct_count, 1);
+}
+
// Scenario: Once FileScannerV2 is selected, an unsupported range must fail
instead of falling back
// to FileScanner.
TEST(FileScannerV2Test, ValidateScanRangeRejectsUnsupportedRange) {
diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp
b/be/test/format_v2/parquet/parquet_statistics_test.cpp
index e69f1311e4a..740892367bb 100644
--- a/be/test/format_v2/parquet/parquet_statistics_test.cpp
+++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp
@@ -47,6 +47,7 @@
#include "format_v2/parquet/parquet_file_context.h"
#include "format_v2/parquet/reader/native/block_split_bloom_filter.h"
#include "io/fs/file_reader.h"
+#include "runtime/runtime_state.h"
#include "util/thrift_util.h"
namespace doris {
namespace {
@@ -669,6 +670,79 @@ TEST(NativeParquetStatisticsTest,
TypeDefinedBoundsRequireSupportedColumnOrder)
EXPECT_TRUE(selected_ranges.empty());
}
+TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
+ auto encode_int32 = [](int32_t value) {
+ std::string bytes(sizeof(value), '\0');
+ memcpy(bytes.data(), &value, sizeof(value));
+ return bytes;
+ };
+
+ auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ column_schema->local_id = 0;
+ column_schema->leaf_column_id = 0;
+ column_schema->type = std::make_shared<DataTypeInt32>();
+ column_schema->type_descriptor.doris_type = column_schema->type;
+ column_schema->type_descriptor.physical_type = tparquet::Type::INT32;
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
+ schema.push_back(std::move(column_schema));
+
+ tparquet::Statistics statistics;
+ statistics.__set_min_value(encode_int32(1));
+ statistics.__set_max_value(encode_int32(2));
+ statistics.__set_null_count(0);
+ tparquet::ColumnMetaData column_metadata;
+ column_metadata.__set_type(tparquet::Type::INT32);
+ column_metadata.__set_num_values(1);
+ column_metadata.__set_statistics(statistics);
+ tparquet::ColumnChunk chunk;
+ chunk.__set_meta_data(column_metadata);
+ tparquet::RowGroup row_group;
+ row_group.__set_columns({chunk});
+ row_group.__set_num_rows(1);
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order});
+ metadata.__set_row_groups({row_group});
+
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ request.predicate_columns =
{format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ request.conjuncts = {
+
VExprContext::create_shared(std::make_shared<MetadataInt32GreaterThanExpr>(100))};
+
+ TQueryOptions query_options;
+ query_options.__set_enable_expr_zonemap_filter(false);
+ RuntimeState state {query_options, TQueryGlobals()};
+ std::vector<int> selected_row_groups;
+ ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata,
schema, request, nullptr,
+
&selected_row_groups, false, nullptr,
+ nullptr, &state)
+ .ok());
+ EXPECT_TRUE(selected_row_groups.empty());
+
+ format::parquet::NativeParquetPageIndex page_index;
+ page_index.column_index.__set_min_values({encode_int32(1)});
+ page_index.column_index.__set_max_values({encode_int32(2)});
+ page_index.column_index.__set_null_pages({false});
+ page_index.column_index.__set_null_counts({0});
+ tparquet::PageLocation location;
+ location.__set_offset(0);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(0);
+ page_index.offset_index.__set_page_locations({location});
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex>
page_indexes;
+ page_indexes.emplace(0, std::move(page_index));
+ std::vector<format::parquet::RowRange> selected_ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
+ ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, page_indexes, schema, request, 1,
&selected_ranges, &skip_plans,
+ nullptr, nullptr, &state)
+ .ok());
+ EXPECT_TRUE(selected_ranges.empty());
+}
+
TEST(NativeParquetStatisticsTest,
ContradictoryAllNullPageCountsDisablePruning) {
auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 1938d19670b..312f8cb9fe6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -2198,7 +2198,15 @@ public class SessionVariable implements Serializable,
Writable {
@VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_PRUNE, needForward =
true, fuzzy = true)
public boolean enableRuntimeFilterPrune = true;
- @VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE,
needForward = true, fuzzy = true)
+ @VarAttrDef.VarAttr(
+ name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE,
+ description = {"控制支持该变量的 scanner 是否启用运行时过滤器分区裁剪。"
+ + "File Scanner V2 始终启用安全的分区裁剪。默认为 true。",
+ "Controls runtime-filter partition pruning in scanners
that honor this variable. "
+ + "File Scanner V2 always enables safe partition
pruning. "
+ + "The default value is true."},
+ needForward = true,
+ fuzzy = true)
public boolean enableRuntimeFilterPartitionPrune = true;
/**
@@ -2719,8 +2727,10 @@ public class SessionVariable implements Serializable,
Writable {
@VarAttrDef.VarAttr(
name = ENABLE_EXPR_ZONEMAP_FILTER,
fuzzy = true,
- description = {"控制 scanner 是否启用表达式 ZoneMap 过滤。默认为 true。",
- "Controls whether to enable expression ZoneMap filtering
in scanners. "
+ description = {"控制支持该变量的 scanner 是否启用表达式 ZoneMap 过滤。"
+ + "File Scanner V2 始终启用安全的表达式 ZoneMap 过滤。默认为 true。",
+ "Controls expression ZoneMap filtering in scanners that
honor this variable. "
+ + "File Scanner V2 always enables safe expression
ZoneMap filtering. "
+ "The default value is true."},
needForward = true)
public boolean enableExprZonemapFilter = true;
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index e4bde8cea41..549118e26a1 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -374,6 +374,8 @@ struct TQueryOptions {
148: optional i32 min_scanners_concurrency = 1;
149: optional i32 min_scan_scheduler_concurrency = 0; //deprecated
+ // Controls runtime-filter partition pruning for readers that honor this
option.
+ // FileScannerV2 always enables safe partition pruning.
150: optional bool enable_runtime_filter_partition_prune = true;
// The minimum memory that an operator required to run.
@@ -501,7 +503,8 @@ struct TQueryOptions {
// enable plan local exchange node in fe
223: optional bool enable_local_shuffle_planner;
- // To control whether BE scan readers may apply expression-based ZoneMap
pruning.
+ // Controls expression-based ZoneMap pruning for readers that honor this
option.
+ // FileScannerV2 always enables safe expression ZoneMap pruning.
224: optional bool enable_expr_zonemap_filter = true
225: optional i64 runtime_filter_tree_publish_max_send_bytes = 268435456
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
