This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new de203c53329 [refactor](be) Remove nested file column predicate filters
(#65130)
de203c53329 is described below
commit de203c533294c01134714795aa7da3b167a24a2f
Author: Gabriel <[email protected]>
AuthorDate: Thu Jul 2 19:15:15 2026 +0800
[refactor](be) Remove nested file column predicate filters (#65130)
---
be/src/format_v2/column_mapper.cpp | 19 +-
be/src/format_v2/column_mapper.h | 2 +-
be/src/format_v2/column_mapper_nested.cpp | 491 +--------------------
be/src/format_v2/column_mapper_nested.h | 7 -
be/src/format_v2/file_reader.cpp | 106 +----
be/src/format_v2/file_reader.h | 50 +--
be/src/format_v2/parquet/parquet_reader.cpp | 2 +-
be/src/format_v2/parquet/parquet_statistics.cpp | 17 +-
be/test/format_v2/column_mapper_test.cpp | 177 +-------
be/test/format_v2/parquet/parquet_reader_test.cpp | 49 +-
.../format_v2/parquet/parquet_statistics_test.cpp | 1 -
11 files changed, 45 insertions(+), 876 deletions(-)
diff --git a/be/src/format_v2/column_mapper.cpp
b/be/src/format_v2/column_mapper.cpp
index e6a0e1a28e7..4a76b112456 100644
--- a/be/src/format_v2/column_mapper.cpp
+++ b/be/src/format_v2/column_mapper.cpp
@@ -1387,8 +1387,8 @@ static Status
apply_scan_projection_to_mapping_file_type(const FileScanRequest&
// Example: for `SELECT s.a FROM t WHERE s.b.c > 1`, the output projection may
only contain `s.a`,
// but the file reader must also read `s.b.c` to evaluate the predicate. This
function collects the
// table-side filter path, resolves it through ColumnMapping first, and
records the corresponding
-// file-side projection in filter_projections. This keeps renamed fields
consistent across the scan
-// projection, row-level conjunct rewrite, and nested predicate pruning.
Example:
+// file-side projection in filter_projections. This keeps renamed fields
consistent between the scan
+// projection and row-level conjunct rewrite. Example:
// table filter path: s -> renamed_b -> c
// old file path: s -> b -> c
// recorded path: s -> b -> c
@@ -1875,8 +1875,6 @@ Status TableColumnMapper::localize_filters(const
std::vector<TableFilter>& table
}
FileColumnPredicateFilter column_predicate_filter;
column_predicate_filter.file_column_id =
LocalColumnId(*mapping->file_local_id);
- column_predicate_filter.target =
-
FileNestedPredicateTarget(column_predicate_filter.file_column_id);
const auto file_primitive_type =
remove_nullable(mapping->file_type)->get_primitive_type();
for (const auto& predicate : predicates) {
@@ -1890,19 +1888,6 @@ Status TableColumnMapper::localize_filters(const
std::vector<TableFilter>& table
}
file_request->column_predicate_filters.push_back(std::move(column_predicate_filter));
}
- for (const auto& table_filter : table_filters) {
- if (table_filter.conjunct == nullptr ||
- !table_filter_has_only_local_entries(table_filter,
_filter_entries)) {
- continue;
- }
- std::vector<FileColumnPredicateFilter>
nested_column_predicate_filters;
-
collect_nested_column_predicate_filters(table_filter.conjunct->root(),
filter_mappings,
-
&nested_column_predicate_filters);
- for (auto& column_predicate_filter :
nested_column_predicate_filters) {
-
merge_column_predicate_filter(std::move(column_predicate_filter),
-
&file_request->column_predicate_filters);
- }
- }
}
return Status::OK();
}
diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h
index 2ffbbbb9414..77c444ba0f1 100644
--- a/be/src/format_v2/column_mapper.h
+++ b/be/src/format_v2/column_mapper.h
@@ -272,7 +272,7 @@ protected:
};
// Parquet consumes the full FileScanRequest shape: predicate columns for lazy
materialization and
-// column_predicate_filters for statistics/page-index pruning.
+// top-level column_predicate_filters for statistics/page-index pruning.
class ParquetColumnMapper final : public TableColumnMapper {
public:
using TableColumnMapper::TableColumnMapper;
diff --git a/be/src/format_v2/column_mapper_nested.cpp
b/be/src/format_v2/column_mapper_nested.cpp
index 0e3539242ff..b172c6ecc87 100644
--- a/be/src/format_v2/column_mapper_nested.cpp
+++ b/be/src/format_v2/column_mapper_nested.cpp
@@ -26,17 +26,12 @@
#include "common/cast_set.h"
#include "common/exception.h"
#include "core/assert_cast.h"
-#include "core/data_type/convert_field_to_type.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type/primitive_type.h"
-#include "exprs/create_predicate_function.h"
#include "exprs/vexpr.h"
-#include "exprs/vin_predicate.h"
#include "format_v2/expr/cast.h"
#include "gen_cpp/Exprs_types.h"
-#include "storage/predicate/null_predicate.h"
-#include "storage/predicate/predicate_creator.h"
namespace doris::format {
@@ -46,43 +41,6 @@ static bool is_cast_expr(const VExprSPtr& expr) {
return dynamic_cast<const Cast*>(expr.get()) != nullptr;
}
-static bool is_binary_comparison_predicate(const VExprSPtr& expr) {
- if (expr == nullptr || expr->get_num_children() != 2 ||
- (expr->node_type() != TExprNodeType::BINARY_PRED &&
- expr->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED)) {
- return false;
- }
- switch (expr->op()) {
- case TExprOpcode::EQ:
- case TExprOpcode::EQ_FOR_NULL:
- case TExprOpcode::NE:
- case TExprOpcode::GE:
- case TExprOpcode::GT:
- case TExprOpcode::LE:
- case TExprOpcode::LT:
- return true;
- default:
- return false;
- }
-}
-
-static bool is_null_predicate_function(const VExprSPtr& expr, bool* is_null) {
- DORIS_CHECK(is_null != nullptr);
- if (expr == nullptr || expr->node_type() != TExprNodeType::FUNCTION_CALL ||
- expr->get_num_children() != 1) {
- return false;
- }
- if (expr->fn().name.function_name == "is_null_pred") {
- *is_null = true;
- return true;
- }
- if (expr->fn().name.function_name == "is_not_null_pred") {
- *is_null = false;
- return true;
- }
- return false;
-}
-
static bool is_signed_integer_type(PrimitiveType type) {
switch (type) {
case TYPE_TINYINT:
@@ -396,23 +354,6 @@ static const std::vector<ColumnDefinition>&
scan_file_children(const ColumnMappi
:
mapping.original_file_children;
}
-static const ColumnDefinition* resolve_file_leaf_from_projection(
- const std::vector<ColumnDefinition>& children, const LocalColumnIndex&
projection) {
- const auto child_it = std::ranges::find_if(children, [&](const
ColumnDefinition& child) {
- return child.file_local_id() == projection.local_id();
- });
- if (child_it == children.end()) {
- return nullptr;
- }
- if (projection.children.empty()) {
- return &*child_it;
- }
- if (projection.children.size() != 1) {
- return nullptr;
- }
- return resolve_file_leaf_from_projection(child_it->children,
projection.children[0]);
-}
-
static bool collect_file_child_names_from_projection(const
std::vector<ColumnDefinition>& children,
const LocalColumnIndex&
projection,
std::vector<std::string>*
file_child_names,
@@ -437,379 +378,6 @@ static bool
collect_file_child_names_from_projection(const std::vector<ColumnDef
file_child_names,
file_child_types);
}
-struct NestedPredicateTarget {
- LocalColumnIndex file_projection;
- FileNestedPredicateTarget file_target;
- std::string leaf_name;
- DataTypePtr leaf_type;
-};
-
-static std::unique_ptr<FileStructPredicateTarget>
build_struct_predicate_target_from_projection(
- const std::vector<ColumnDefinition>& children, const LocalColumnIndex&
projection) {
- const auto child_it = std::ranges::find_if(children, [&](const
ColumnDefinition& child) {
- return child.file_local_id() == projection.local_id();
- });
- if (child_it == children.end()) {
- return nullptr;
- }
- std::unique_ptr<FileStructPredicateTarget> nested_child;
- if (!projection.children.empty()) {
- if (projection.children.size() != 1) {
- return nullptr;
- }
- nested_child =
build_struct_predicate_target_from_projection(child_it->children,
-
projection.children[0]);
- if (nested_child == nullptr) {
- return nullptr;
- }
- }
- return
std::make_unique<FileStructPredicateTarget>(child_it->file_local_id(),
child_it->name,
-
std::move(nested_child));
-}
-
-static bool build_struct_predicate_target(const ColumnMapping& root_mapping,
- const LocalColumnIndex&
root_projection,
- FileNestedPredicateTarget*
file_target) {
- DORIS_CHECK(file_target != nullptr);
- if (!root_projection.column_id().is_valid() ||
root_projection.children.size() != 1) {
- return false;
- }
- auto struct_target = build_struct_predicate_target_from_projection(
- root_mapping.original_file_children, root_projection.children[0]);
- if (struct_target == nullptr) {
- return false;
- }
- *file_target = FileNestedPredicateTarget(root_projection.column_id(),
std::move(struct_target));
- return true;
-}
-
-static bool resolve_nested_predicate_target(const NestedStructPath& path,
- const std::vector<ColumnMapping>&
mappings,
- NestedPredicateTarget* target) {
- DORIS_CHECK(target != nullptr);
- ResolvedNestedStructPath resolved;
- if (!resolve_nested_struct_path_for_file(path, mappings, &resolved)) {
- return false;
- }
-
- const auto mapping_it = std::ranges::find_if(mappings, [&](const
ColumnMapping& mapping) {
- return mapping.global_index == path.root_global_index;
- });
- if (mapping_it == mappings.end() ||
resolved.file_projection.children.size() != 1) {
- return false;
- }
- const auto* file_leaf =
resolve_file_leaf_from_projection(mapping_it->original_file_children,
-
resolved.file_projection.children[0]);
- if (file_leaf == nullptr || file_leaf->type == nullptr) {
- return false;
- }
- target->leaf_type = remove_nullable(file_leaf->type);
- if (is_complex_type(target->leaf_type->get_primitive_type())) {
- return false;
- }
- target->leaf_name = file_leaf->name;
- target->file_projection = std::move(resolved.file_projection);
- if (!build_struct_predicate_target(*mapping_it, target->file_projection,
- &target->file_target)) {
- return false;
- }
- return true;
-}
-
-static VExprSPtr original_table_literal_for_nested_predicate(const VExprSPtr&
literal_expr) {
- DORIS_CHECK(literal_expr != nullptr);
- DORIS_CHECK(literal_expr->is_literal());
- const auto* rewritten_literal = dynamic_cast<const
SplitLocalFileLiteral*>(literal_expr.get());
- if (rewritten_literal == nullptr) {
- return literal_expr;
- }
- return VLiteral::create_shared(rewritten_literal->original_type(),
- rewritten_literal->original_field());
-}
-
-static std::optional<PredicateType> to_column_predicate_type(TExprOpcode::type
opcode) {
- switch (opcode) {
- case TExprOpcode::EQ:
- return PredicateType::EQ;
- case TExprOpcode::NE:
- return PredicateType::NE;
- case TExprOpcode::GT:
- return PredicateType::GT;
- case TExprOpcode::GE:
- return PredicateType::GE;
- case TExprOpcode::LT:
- return PredicateType::LT;
- case TExprOpcode::LE:
- return PredicateType::LE;
- default:
- return std::nullopt;
- }
-}
-
-static TExprOpcode::type reverse_comparison_opcode(TExprOpcode::type opcode) {
- switch (opcode) {
- case TExprOpcode::GT:
- return TExprOpcode::LT;
- case TExprOpcode::GE:
- return TExprOpcode::LE;
- case TExprOpcode::LT:
- return TExprOpcode::GT;
- case TExprOpcode::LE:
- return TExprOpcode::GE;
- default:
- return opcode;
- }
-}
-
-static std::shared_ptr<ColumnPredicate> create_comparison_column_predicate(
- PredicateType predicate_type, uint32_t column_id, const std::string&
column_name,
- const DataTypePtr& data_type, const Field& value) {
- switch (predicate_type) {
- case PredicateType::EQ:
- return create_comparison_predicate<PredicateType::EQ>(column_id,
column_name, data_type,
- value, false);
- case PredicateType::NE:
- return create_comparison_predicate<PredicateType::NE>(column_id,
column_name, data_type,
- value, false);
- case PredicateType::GT:
- return create_comparison_predicate<PredicateType::GT>(column_id,
column_name, data_type,
- value, false);
- case PredicateType::GE:
- return create_comparison_predicate<PredicateType::GE>(column_id,
column_name, data_type,
- value, false);
- case PredicateType::LT:
- return create_comparison_predicate<PredicateType::LT>(column_id,
column_name, data_type,
- value, false);
- case PredicateType::LE:
- return create_comparison_predicate<PredicateType::LE>(column_id,
column_name, data_type,
- value, false);
- default:
- return nullptr;
- }
-}
-
-static bool extract_child_id_path_from_projection(const LocalColumnIndex&
root_projection,
- std::vector<int32_t>*
file_child_id_path) {
- DORIS_CHECK(file_child_id_path != nullptr);
- file_child_id_path->clear();
- const auto* current_projection = &root_projection;
- while (!current_projection->children.empty()) {
- if (current_projection->children.size() != 1) {
- file_child_id_path->clear();
- return false;
- }
- current_projection = ¤t_projection->children[0];
- file_child_id_path->push_back(current_projection->local_id());
- }
- return !file_child_id_path->empty();
-}
-
-static std::shared_ptr<ColumnPredicate> build_nested_comparison_predicate(
- const VExprSPtr& literal_expr, TExprOpcode::type opcode, LocalColumnId
root_file_column_id,
- const std::string& leaf_name, const DataTypePtr& file_leaf_type) {
- if (literal_expr == nullptr || !literal_expr->is_literal() ||
file_leaf_type == nullptr) {
- return nullptr;
- }
- const auto predicate_type = to_column_predicate_type(opcode);
- if (!predicate_type.has_value()) {
- return nullptr;
- }
- const auto original_literal =
original_table_literal_for_nested_predicate(literal_expr);
- const Field original_field = literal_field(original_literal);
- Field file_field;
- try {
- convert_field_to_type(original_field, *file_leaf_type, &file_field,
- original_literal->data_type().get());
- } catch (const Exception&) {
- return nullptr;
- }
- if (file_field.is_null()) {
- return nullptr;
- }
- try {
- return create_comparison_column_predicate(*predicate_type,
-
cast_set<uint32_t>(root_file_column_id.value()),
- leaf_name, file_leaf_type,
file_field);
- } catch (const Exception&) {
- return nullptr;
- }
-}
-
-static std::shared_ptr<ColumnPredicate> build_nested_in_list_predicate(
- const VExprSPtrs& literal_exprs, LocalColumnId root_file_column_id,
- const std::string& leaf_name, const DataTypePtr& file_leaf_type) {
- if (literal_exprs.empty() || file_leaf_type == nullptr) {
- return nullptr;
- }
-
- auto value_column = file_leaf_type->create_column();
- for (const auto& literal_expr : literal_exprs) {
- if (literal_expr == nullptr || !literal_expr->is_literal()) {
- return nullptr;
- }
- const auto original_literal =
original_table_literal_for_nested_predicate(literal_expr);
- const Field original_field = literal_field(original_literal);
- Field file_field;
- try {
- convert_field_to_type(original_field, *file_leaf_type, &file_field,
- original_literal->data_type().get());
- } catch (const Exception&) {
- return nullptr;
- }
- if (file_field.is_null()) {
- return nullptr;
- }
- value_column->insert(file_field);
- }
-
- std::shared_ptr<HybridSetBase> values;
- try {
- values.reset(create_set(file_leaf_type->get_primitive_type(),
literal_exprs.size(), false));
- ColumnPtr value_column_ptr = std::move(value_column);
- values->insert_range_from(value_column_ptr, 0,
value_column_ptr->size());
- return create_in_list_predicate<PredicateType::IN_LIST>(
- cast_set<uint32_t>(root_file_column_id.value()), leaf_name,
file_leaf_type, values,
- false);
- } catch (const Exception&) {
- return nullptr;
- }
-}
-
-static std::shared_ptr<ColumnPredicate> build_nested_null_predicate(
- bool is_null, LocalColumnId root_file_column_id, const std::string&
leaf_name,
- const DataTypePtr& file_leaf_type) {
- if (file_leaf_type == nullptr) {
- return nullptr;
- }
- const auto leaf_primitive_type =
remove_nullable(file_leaf_type)->get_primitive_type();
- return
NullPredicate::create_shared(cast_set<uint32_t>(root_file_column_id.value()),
leaf_name,
- is_null, leaf_primitive_type);
-}
-
-static bool set_nested_column_filter_target(const NestedPredicateTarget&
target,
- FileColumnPredicateFilter*
column_filter) {
- DORIS_CHECK(column_filter != nullptr);
- std::vector<int32_t> file_child_id_path;
- if (!extract_child_id_path_from_projection(target.file_projection,
&file_child_id_path)) {
- return false;
- }
- column_filter->file_column_id = target.file_projection.column_id();
- column_filter->file_child_id_path = std::move(file_child_id_path);
- column_filter->target = target.file_target;
- return true;
-}
-
-static bool extract_nested_binary_comparison_filter(const VExprSPtr& expr,
- const
std::vector<ColumnMapping>& mappings,
- FileColumnPredicateFilter*
column_filter) {
- DORIS_CHECK(column_filter != nullptr);
- if (!is_binary_comparison_predicate(expr)) {
- return false;
- }
- NestedStructPath path;
- VExprSPtr literal_expr;
- TExprOpcode::type opcode = expr->op();
- if (extract_nested_struct_path_for_pruning(expr->children()[0], &path) &&
- expr->children()[1]->is_literal()) {
- literal_expr = expr->children()[1];
- } else if (extract_nested_struct_path_for_pruning(expr->children()[1],
&path) &&
- expr->children()[0]->is_literal()) {
- literal_expr = expr->children()[0];
- opcode = reverse_comparison_opcode(opcode);
- } else {
- return false;
- }
-
- NestedPredicateTarget target;
- if (!resolve_nested_predicate_target(path, mappings, &target)) {
- return false;
- }
- auto predicate = build_nested_comparison_predicate(literal_expr, opcode,
-
target.file_projection.column_id(),
- target.leaf_name,
target.leaf_type);
- if (predicate == nullptr) {
- return false;
- }
- if (!set_nested_column_filter_target(target, column_filter)) {
- return false;
- }
- column_filter->predicates.push_back(std::move(predicate));
- return true;
-}
-
-static bool extract_nested_in_list_filter(const VExprSPtr& expr,
- const std::vector<ColumnMapping>&
mappings,
- FileColumnPredicateFilter*
column_filter) {
- DORIS_CHECK(column_filter != nullptr);
- if (expr == nullptr || expr->node_type() != TExprNodeType::IN_PRED ||
- expr->get_num_children() < 2) {
- return false;
- }
- if (const auto* in_predicate = dynamic_cast<const
VInPredicate*>(expr.get());
- in_predicate != nullptr && in_predicate->is_not_in()) {
- return false;
- }
-
- NestedStructPath path;
- if (!extract_nested_struct_path_for_pruning(expr->children()[0], &path)) {
- return false;
- }
-
- VExprSPtrs literal_exprs;
- literal_exprs.reserve(expr->get_num_children() - 1);
- for (size_t child_idx = 1; child_idx < expr->children().size();
++child_idx) {
- if (!expr->children()[child_idx]->is_literal()) {
- return false;
- }
- literal_exprs.push_back(expr->children()[child_idx]);
- }
-
- NestedPredicateTarget target;
- if (!resolve_nested_predicate_target(path, mappings, &target)) {
- return false;
- }
- auto predicate = build_nested_in_list_predicate(
- literal_exprs, target.file_projection.column_id(),
target.leaf_name, target.leaf_type);
- if (predicate == nullptr) {
- return false;
- }
- if (!set_nested_column_filter_target(target, column_filter)) {
- return false;
- }
- column_filter->predicates.push_back(std::move(predicate));
- return true;
-}
-
-static bool extract_nested_null_filter(const VExprSPtr& expr,
- const std::vector<ColumnMapping>&
mappings,
- FileColumnPredicateFilter*
column_filter) {
- DORIS_CHECK(column_filter != nullptr);
- bool is_null = false;
- if (!is_null_predicate_function(expr, &is_null)) {
- return false;
- }
-
- NestedStructPath path;
- if (!extract_nested_struct_path_for_pruning(expr->children()[0], &path)) {
- return false;
- }
-
- NestedPredicateTarget target;
- if (!resolve_nested_predicate_target(path, mappings, &target)) {
- return false;
- }
- auto predicate = build_nested_null_predicate(is_null,
target.file_projection.column_id(),
- target.leaf_name,
target.leaf_type);
- if (predicate == nullptr) {
- return false;
- }
- if (!set_nested_column_filter_target(target, column_filter)) {
- return false;
- }
- column_filter->predicates.push_back(std::move(predicate));
- return true;
-}
-
} // namespace
SplitLocalFileLiteral::SplitLocalFileLiteral(const DataTypePtr& file_type,
const Field& file_field,
@@ -925,11 +493,11 @@ bool resolve_nested_struct_expr_for_file(const VExprSPtr&
expr,
return resolve_nested_struct_path_for_file(path, mappings, resolved, true);
}
-// Collect nested struct leaf references that can be turned into file-reader
projections and
-// primitive pruning predicates. For example, from `s.a > 1 AND element_at(s,
'b') = 2`, this
-// records two paths rooted at `s`: `s -> a` and `s -> b`. Non-struct
expressions are traversed
-// recursively, while a recognized struct path is emitted once so the caller
can merge it into the
-// scan projection for that top-level file column.
+// Collect nested struct leaf references that can be turned into file-reader
projections. For
+// example, from `s.a > 1 AND element_at(s, 'b') = 2`, this records two paths
rooted at `s`:
+// `s -> a` and `s -> b`. Non-struct expressions are traversed recursively,
while a recognized
+// struct path is emitted once so the caller can merge it into the scan
projection for that
+// top-level file column.
void collect_nested_struct_paths(const VExprSPtr& expr,
std::vector<NestedStructPath>* paths) {
DORIS_CHECK(paths != nullptr);
if (expr == nullptr) {
@@ -998,53 +566,4 @@ Status build_file_child_projection_from_schema(const
std::vector<ColumnDefinitio
return Status::OK();
}
-// Merge predicates that target the same physical file column or nested leaf.
For example,
-// `WHERE s.b > 1 AND s.b < 10` produces two predicates for the same target `s
-> b`; keeping them
-// in one FileColumnPredicateFilter lets the file reader apply both pruning
checks to the same leaf
-// instead of carrying duplicate target entries.
-void merge_column_predicate_filter(FileColumnPredicateFilter column_filter,
- std::vector<FileColumnPredicateFilter>*
filters) {
- DORIS_CHECK(filters != nullptr);
- auto existing_filter_it = std::ranges::find_if(*filters, [&](const auto&
existing_filter) {
- return existing_filter.same_target_as(column_filter);
- });
- if (existing_filter_it == filters->end()) {
- filters->push_back(std::move(column_filter));
- return;
- }
- existing_filter_it->predicates.insert(existing_filter_it->predicates.end(),
- column_filter.predicates.begin(),
- column_filter.predicates.end());
-}
-
-// Extract file-column pruning predicates from localized row-level conjuncts
that reference nested
-// struct leaves. This is separate from file_request->conjuncts: conjuncts do
row filtering, while
-// FileColumnPredicateFilter carries primitive leaf predicates for
file/page/statistics pruning.
-//
-// Example: for `WHERE s.b.c > 10 AND element_at(s, 'd') IS NOT NULL`, this
function emits pruning
-// filters for the nested targets `s -> b -> c` and `s -> d`. The caller only
invokes it after
-// table_filter_has_only_local_entries() succeeds, so each root slot already
has a file-local scan
-// source in _filter_entries.
-void collect_nested_column_predicate_filters(const VExprSPtr& expr,
- const std::vector<ColumnMapping>&
mappings,
-
std::vector<FileColumnPredicateFilter>* filters) {
- DORIS_CHECK(filters != nullptr);
- if (expr == nullptr) {
- return;
- }
- if (expr->node_type() == TExprNodeType::COMPOUND_PRED &&
- expr->op() == TExprOpcode::COMPOUND_AND) {
- for (const auto& child : expr->children()) {
- collect_nested_column_predicate_filters(child, mappings, filters);
- }
- return;
- }
- FileColumnPredicateFilter column_filter;
- if (extract_nested_binary_comparison_filter(expr, mappings,
&column_filter) ||
- extract_nested_in_list_filter(expr, mappings, &column_filter) ||
- extract_nested_null_filter(expr, mappings, &column_filter)) {
- merge_column_predicate_filter(std::move(column_filter), filters);
- }
-}
-
} // namespace doris::format
diff --git a/be/src/format_v2/column_mapper_nested.h
b/be/src/format_v2/column_mapper_nested.h
index b8b3f1f3334..7b2e8cb1513 100644
--- a/be/src/format_v2/column_mapper_nested.h
+++ b/be/src/format_v2/column_mapper_nested.h
@@ -95,11 +95,4 @@ Status build_file_child_projection_from_schema(const
std::vector<ColumnDefinitio
std::span<const
StructChildSelector> selectors,
LocalColumnIndex* projection);
-void merge_column_predicate_filter(FileColumnPredicateFilter column_filter,
- std::vector<FileColumnPredicateFilter>*
filters);
-
-void collect_nested_column_predicate_filters(const VExprSPtr& expr,
- const std::vector<ColumnMapping>&
mappings,
-
std::vector<FileColumnPredicateFilter>* filters);
-
} // namespace doris::format
diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp
index 31b3f27c697..9b28d6faaae 100644
--- a/be/src/format_v2/file_reader.cpp
+++ b/be/src/format_v2/file_reader.cpp
@@ -27,11 +27,6 @@
namespace doris::format {
namespace {
-std::unique_ptr<FileStructPredicateTarget> clone_struct_predicate_target(
- const std::unique_ptr<FileStructPredicateTarget>& target) {
- return target == nullptr ? nullptr :
std::make_unique<FileStructPredicateTarget>(*target);
-}
-
template <typename T, typename Formatter>
std::string join_debug_strings(const std::vector<T>& values, Formatter
formatter) {
std::ostringstream out;
@@ -46,110 +41,11 @@ std::string join_debug_strings(const std::vector<T>&
values, Formatter formatter
return out.str();
}
-std::string int_vector_debug_string(const std::vector<int32_t>& values) {
- std::ostringstream out;
- out << "[";
- for (size_t i = 0; i < values.size(); ++i) {
- if (i > 0) {
- out << ", ";
- }
- out << values[i];
- }
- out << "]";
- return out.str();
-}
-
-void append_struct_predicate_path(const FileStructPredicateTarget* target,
- std::vector<int32_t>* path) {
- DORIS_CHECK(path != nullptr);
- for (const auto* current = target; current != nullptr; current =
current->child.get()) {
- path->push_back(current->file_local_id);
- }
-}
-
-std::string struct_predicate_target_debug_string(const
FileStructPredicateTarget* target) {
- if (target == nullptr) {
- return "null";
- }
- std::ostringstream out;
- out << "{file_local_id=" << target->file_local_id
- << ", file_child_name=" << target->file_child_name
- << ", child=" <<
struct_predicate_target_debug_string(target->child.get()) << "}";
- return out.str();
-}
-
-bool struct_predicate_targets_equal(const FileStructPredicateTarget* lhs,
- const FileStructPredicateTarget* rhs) {
- while (lhs != nullptr && rhs != nullptr) {
- if (lhs->file_local_id != rhs->file_local_id) {
- return false;
- }
- lhs = lhs->child.get();
- rhs = rhs->child.get();
- }
- return lhs == nullptr && rhs == nullptr;
-}
-
} // namespace
-FileStructPredicateTarget::FileStructPredicateTarget(const
FileStructPredicateTarget& other)
- : file_local_id(other.file_local_id),
- file_child_name(other.file_child_name),
- child(clone_struct_predicate_target(other.child)) {}
-
-FileStructPredicateTarget& FileStructPredicateTarget::operator=(
- const FileStructPredicateTarget& other) {
- if (this == &other) {
- return *this;
- }
- file_local_id = other.file_local_id;
- file_child_name = other.file_child_name;
- child = clone_struct_predicate_target(other.child);
- return *this;
-}
-
-FileNestedPredicateTarget::FileNestedPredicateTarget(const
FileNestedPredicateTarget& other)
- : file_column_id(other.file_column_id),
- struct_target(clone_struct_predicate_target(other.struct_target)) {}
-
-FileNestedPredicateTarget& FileNestedPredicateTarget::operator=(
- const FileNestedPredicateTarget& other) {
- if (this == &other) {
- return *this;
- }
- file_column_id = other.file_column_id;
- struct_target = clone_struct_predicate_target(other.struct_target);
- return *this;
-}
-
-LocalColumnId FileColumnPredicateFilter::effective_file_column_id() const {
- return target.is_valid() ? target.file_column_id : file_column_id;
-}
-
-std::vector<int32_t> FileColumnPredicateFilter::effective_file_child_id_path()
const {
- if (!target.is_valid()) {
- return file_child_id_path;
- }
- std::vector<int32_t> path;
- append_struct_predicate_path(target.struct_target.get(), &path);
- return path;
-}
-
-bool FileColumnPredicateFilter::same_target_as(const
FileColumnPredicateFilter& other) const {
- if (target.is_valid() && other.target.is_valid()) {
- return target.file_column_id == other.target.file_column_id &&
- struct_predicate_targets_equal(target.struct_target.get(),
-
other.target.struct_target.get());
- }
- return effective_file_column_id() == other.effective_file_column_id() &&
- effective_file_child_id_path() ==
other.effective_file_child_id_path();
-}
-
std::string FileColumnPredicateFilter::debug_string() const {
std::ostringstream out;
- out << "FileColumnPredicateFilter{target={file_column_id=" <<
effective_file_column_id()
- << ", struct_target=" <<
struct_predicate_target_debug_string(target.struct_target.get())
- << "}, file_child_id_path=" <<
int_vector_debug_string(effective_file_child_id_path())
+ out << "FileColumnPredicateFilter{file_column_id=" << file_column_id
<< ", predicate_count=" << predicates.size() << "}";
return out.str();
}
diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h
index 76b8bea6097..e4c55bd9a6b 100644
--- a/be/src/format_v2/file_reader.h
+++ b/be/src/format_v2/file_reader.h
@@ -48,61 +48,13 @@ namespace doris::format {
class TableColumnMapper;
struct TableColumnMapperOptions;
-// Struct-only nested predicate target used by file-layer pruning.
-// This intentionally models only a STRUCT field chain. LIST/MAP/repeated
predicates need explicit
-// quantified semantics, so they must not be encoded here.
-struct FileStructPredicateTarget {
- int32_t file_local_id = -1;
- std::string file_child_name;
- std::unique_ptr<FileStructPredicateTarget> child;
-
- FileStructPredicateTarget() = default;
- FileStructPredicateTarget(int32_t local_id, std::string child_name,
- std::unique_ptr<FileStructPredicateTarget>
nested_child = nullptr)
- : file_local_id(local_id),
- file_child_name(std::move(child_name)),
- child(std::move(nested_child)) {}
- FileStructPredicateTarget(const FileStructPredicateTarget& other);
- FileStructPredicateTarget& operator=(const FileStructPredicateTarget&
other);
- FileStructPredicateTarget(FileStructPredicateTarget&& other) noexcept =
default;
- FileStructPredicateTarget& operator=(FileStructPredicateTarget&& other)
noexcept = default;
-};
-
-struct FileNestedPredicateTarget {
- LocalColumnId file_column_id = LocalColumnId::invalid();
- // Null means the predicate targets the top-level primitive column itself.
- std::unique_ptr<FileStructPredicateTarget> struct_target;
-
- FileNestedPredicateTarget() = default;
- explicit FileNestedPredicateTarget(LocalColumnId column_id) :
file_column_id(column_id) {}
- FileNestedPredicateTarget(LocalColumnId column_id,
- std::unique_ptr<FileStructPredicateTarget>
target)
- : file_column_id(column_id), struct_target(std::move(target)) {}
- FileNestedPredicateTarget(const FileNestedPredicateTarget& other);
- FileNestedPredicateTarget& operator=(const FileNestedPredicateTarget&
other);
- FileNestedPredicateTarget(FileNestedPredicateTarget&& other) noexcept =
default;
- FileNestedPredicateTarget& operator=(FileNestedPredicateTarget&& other)
noexcept = default;
-
- bool is_valid() const { return file_column_id.is_valid(); }
-};
-
// File-local single-column predicates for file-layer pruning, such as
min/max, page index,
-// dictionary and bloom filter.
-// Predicates must all belong to target.file_column_id. target.struct_target
points to the nested
-// primitive leaf under that root; null means the top-level column itself is
the primitive leaf.
+// dictionary and bloom filter. Predicates must all belong to file_column_id.
// These predicates are pruning hints only and are not row-level conjuncts.
struct FileColumnPredicateFilter {
- FileNestedPredicateTarget target;
- // Compatibility fields for call sites and tests that still construct
pruning filters directly.
- // New mapper code should fill target; file readers consume target first
and only fall back to
- // these fields while the API migration is in progress.
LocalColumnId file_column_id = LocalColumnId::invalid();
- std::vector<int32_t> file_child_id_path;
std::vector<std::shared_ptr<ColumnPredicate>> predicates;
- LocalColumnId effective_file_column_id() const;
- std::vector<int32_t> effective_file_child_id_path() const;
- bool same_target_as(const FileColumnPredicateFilter& other) const;
std::string debug_string() const;
};
diff --git a/be/src/format_v2/parquet/parquet_reader.cpp
b/be/src/format_v2/parquet/parquet_reader.cpp
index 24797200693..3e13333e7d4 100644
--- a/be/src/format_v2/parquet/parquet_reader.cpp
+++ b/be/src/format_v2/parquet/parquet_reader.cpp
@@ -395,7 +395,7 @@ Status
ParquetReader::open(std::shared_ptr<format::FileScanRequest> request) {
const int num_fields = static_cast<int>(_state->file_schema.size());
for (const auto& column_filter :
request_snapshot->column_predicate_filters) {
- const auto file_column_id = column_filter.effective_file_column_id();
+ const auto file_column_id = column_filter.file_column_id;
if (!file_column_id.is_valid() || file_column_id.value() >=
num_fields) {
return Status::InvalidArgument("Invalid parquet filter top-level
local id {}",
file_column_id.value());
diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp
b/be/src/format_v2/parquet/parquet_statistics.cpp
index 1207aecd8e4..66af8515aba 100644
--- a/be/src/format_v2/parquet/parquet_statistics.cpp
+++ b/be/src/format_v2/parquet/parquet_statistics.cpp
@@ -595,19 +595,10 @@ segment_v2::ZoneMap to_column_predicate_statistics(const
ParquetColumnStatistics
return predicate_statistics;
}
-const ParquetColumnSchema* find_child_schema_by_local_id(const
ParquetColumnSchema& column_schema,
- int32_t local_id) {
- const auto child_it = std::ranges::find_if(
- column_schema.children, [&](const
std::unique_ptr<ParquetColumnSchema>& child) {
- return child != nullptr && child->local_id == local_id;
- });
- return child_it == column_schema.children.end() ? nullptr :
child_it->get();
-}
-
const ParquetColumnSchema* resolve_predicate_leaf_schema(
const std::vector<std::unique_ptr<ParquetColumnSchema>>& schema,
const format::FileColumnPredicateFilter& column_filter) {
- const auto file_column_id = column_filter.effective_file_column_id();
+ const auto file_column_id = column_filter.file_column_id;
if (!file_column_id.is_valid() || file_column_id.value() >=
static_cast<int>(schema.size())) {
return nullptr;
}
@@ -615,12 +606,6 @@ const ParquetColumnSchema* resolve_predicate_leaf_schema(
if (column_schema == nullptr) {
return nullptr;
}
- for (const auto child_local_id :
column_filter.effective_file_child_id_path()) {
- column_schema = find_child_schema_by_local_id(*column_schema,
child_local_id);
- if (column_schema == nullptr) {
- return nullptr;
- }
- }
if (column_schema->kind != ParquetColumnSchemaKind::PRIMITIVE ||
column_schema->leaf_column_id < 0 ||
column_schema->max_repetition_level > 0) {
return nullptr;
diff --git a/be/test/format_v2/column_mapper_test.cpp
b/be/test/format_v2/column_mapper_test.cpp
index d870f85dd7d..f666c4a1523 100644
--- a/be/test/format_v2/column_mapper_test.cpp
+++ b/be/test/format_v2/column_mapper_test.cpp
@@ -175,14 +175,6 @@ std::vector<int32_t> projection_ids(const
std::vector<LocalColumnIndex>& project
return ids;
}
-std::vector<std::string> target_names(const FileStructPredicateTarget* target)
{
- std::vector<std::string> names;
- for (const auto* current = target; current != nullptr; current =
current->child.get()) {
- names.push_back(current->file_child_name);
- }
- return names;
-}
-
TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) {
ColumnDefinition child = field_id_col("child", 2, str(), 3);
child.name_mapping = {"legacy_child"};
@@ -436,23 +428,6 @@ VExprSPtr compound_predicate(TExprOpcode::type opcode,
const VExprSPtr& left,
return expr;
}
-ColumnMapping mapped_struct_column(int32_t root_file_local_id, const
std::string& child_name,
- int32_t child_file_local_id, DataTypePtr
child_type) {
- ColumnDefinition file_child = name_col(child_name, child_type,
child_file_local_id);
- ColumnMapping root;
- root.global_index = GlobalIndex(0);
- root.table_column_name = "s";
- root.file_local_id = root_file_local_id;
- root.file_column_name = "s";
- root.table_type =
- std::make_shared<DataTypeStruct>(DataTypes {child_type}, Strings
{child_name});
- root.file_type = root.table_type;
- root.original_file_type = root.table_type;
- root.original_file_children = {file_child};
- root.projected_file_children = {file_child};
- return root;
-}
-
std::vector<NestedStructPath> collect_paths(const VExprSPtr& expr) {
std::vector<NestedStructPath> paths;
collect_nested_struct_paths(expr, &paths);
@@ -768,82 +743,10 @@ TEST(ColumnMapperNestedHelperTest,
BuildsProjectionByNameAndOrdinalSelectors) {
EXPECT_EQ(ordinal_projection.children[0].local_id(), 0);
}
-TEST(ColumnMapperNestedHelperTest, MergesPredicateFiltersForSameNestedTarget) {
- FileColumnPredicateFilter gt_filter;
- gt_filter.target = FileNestedPredicateTarget(
- LocalColumnId(7), std::make_unique<FileStructPredicateTarget>(2,
"score"));
- gt_filter.file_column_id = LocalColumnId(7);
- gt_filter.file_child_id_path = {2};
-
gt_filter.predicates.push_back(create_comparison_predicate<PredicateType::GT>(
- 7, "score", i32(), Field::create_field<TYPE_INT>(10), false));
-
- FileColumnPredicateFilter lt_filter;
- lt_filter.target = FileNestedPredicateTarget(
- LocalColumnId(7), std::make_unique<FileStructPredicateTarget>(2,
"score"));
- lt_filter.file_column_id = LocalColumnId(7);
- lt_filter.file_child_id_path = {2};
-
lt_filter.predicates.push_back(create_comparison_predicate<PredicateType::LT>(
- 7, "score", i32(), Field::create_field<TYPE_INT>(100), false));
-
- std::vector<FileColumnPredicateFilter> filters;
- merge_column_predicate_filter(std::move(gt_filter), &filters);
- merge_column_predicate_filter(std::move(lt_filter), &filters);
-
- ASSERT_EQ(filters.size(), 1);
- EXPECT_EQ(filters[0].effective_file_column_id(), LocalColumnId(7));
- EXPECT_EQ(filters[0].effective_file_child_id_path(),
std::vector<int32_t>({2}));
- ASSERT_EQ(filters[0].predicates.size(), 2);
- EXPECT_EQ(target_names(filters[0].target.struct_target.get()),
- std::vector<std::string>({"score"}));
-}
-
-TEST(ColumnMapperNestedHelperTest, DoesNotExtractPredicateFiltersFromOr) {
- const auto int_type = i32();
- const auto struct_type = std::make_shared<DataTypeStruct>(DataTypes
{int_type}, Strings {"a"});
- const auto slot = table_slot(0, 0, struct_type, "s");
- const auto left = int_gt(struct_element(slot, int_type, "a"), 10);
- const auto right = int_gt(struct_element(slot, int_type, "a"), 20);
- const auto or_expr = compound_predicate(TExprOpcode::COMPOUND_OR, left,
right);
-
- std::vector<FileColumnPredicateFilter> filters;
- collect_nested_column_predicate_filters(or_expr, {mapped_struct_column(5,
"a", 0, int_type)},
- &filters);
-
- EXPECT_TRUE(filters.empty());
-}
-
-TEST(ColumnMapperNestedHelperTest,
DoesNotExtractPredicateFiltersFromUnsupportedExpression) {
- const auto int_type = i32();
- const auto struct_type = std::make_shared<DataTypeStruct>(DataTypes
{int_type}, Strings {"a"});
- auto add_expr = std::make_shared<TestFunctionExpr>("add", int_type);
- add_expr->add_child(struct_element(table_slot(0, 0, struct_type, "s"),
int_type, "a"));
- add_expr->add_child(literal(int_type, Field::create_field<TYPE_INT>(1)));
-
- std::vector<FileColumnPredicateFilter> filters;
- collect_nested_column_predicate_filters(add_expr, {mapped_struct_column(5,
"a", 0, int_type)},
- &filters);
-
- EXPECT_TRUE(filters.empty());
-}
-
-TEST(ColumnMapperNestedHelperTest,
DoesNotExtractPredicateFiltersThroughUnsafeCast) {
- const auto file_type = i64();
- const auto table_type = i32();
- const auto struct_type = std::make_shared<DataTypeStruct>(DataTypes
{file_type}, Strings {"a"});
- const auto nested_leaf = struct_element(table_slot(0, 0, struct_type,
"s"), file_type, "a");
- const auto filter_expr = int_gt(cast_expr(nested_leaf, table_type), 10);
-
- std::vector<FileColumnPredicateFilter> filters;
- collect_nested_column_predicate_filters(filter_expr,
- {mapped_struct_column(5, "a", 0,
file_type)}, &filters);
-
- EXPECT_TRUE(filters.empty());
-}
-
// ----------------------------------------------------------------------
// collect_nested_struct_paths() helper tests.
// These tests assert the entry helper for nested scan projection: it only
discovers
-// table-side struct paths. Later localization decides whether to build
pruning predicates.
+// table-side struct paths. Later localization decides how to add scan
projections.
// ----------------------------------------------------------------------
TEST(ColumnMapperCollectNestedStructPathsTest,
CollectsNameOrdinalAndBooleanSelectors) {
@@ -1057,10 +960,7 @@ TEST(ColumnMapperCollectNestedStructPathsTest,
ProjectionMergeKeepsFilterOnlyPat
EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(5));
ASSERT_FALSE(request.predicate_columns[0].project_all_children);
EXPECT_EQ(projection_ids(request.predicate_columns[0].children),
std::vector<int32_t>({0, 1}));
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({1}));
- ASSERT_EQ(request.column_predicate_filters[0].predicates.size(), 2);
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
// Scenario: row-oriented readers such as CSV/Text cannot lazy-read predicate
columns separately.
@@ -2318,7 +2218,7 @@ TEST(ColumnMapperLocalizeFiltersTest,
ColumnPredicatesUseOnlyExistingLocalPositi
EXPECT_EQ(request_with_local_position.non_predicate_columns[0].column_id(),
LocalColumnId(3));
EXPECT_TRUE(request_with_local_position.predicate_columns.empty());
ASSERT_EQ(request_with_local_position.column_predicate_filters.size(), 1);
-
EXPECT_EQ(request_with_local_position.column_predicate_filters[0].effective_file_column_id(),
+
EXPECT_EQ(request_with_local_position.column_predicate_filters[0].file_column_id,
LocalColumnId(3));
ASSERT_EQ(request_with_local_position.column_predicate_filters[0].predicates.size(),
1);
EXPECT_EQ(request_with_local_position.column_predicate_filters[0].predicates[0]->type(),
@@ -2360,12 +2260,7 @@ TEST(ColumnMapperLocalizeFiltersTest,
NestedFilterOnlyChildMergesIntoPredicatePr
EXPECT_EQ(request.local_positions.at(LocalColumnId(5)), LocalIndex(0));
ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(0)).is_local());
EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(0)).local_index(),
LocalIndex(0));
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(5));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({0}));
-
EXPECT_EQ(target_names(request.column_predicate_filters[0].target.struct_target.get()),
- std::vector<std::string>({"a"}));
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
TEST(ColumnMapperLocalizeFiltersTest,
PreservesExistingScanStateWhenAddingPredicateColumn) {
@@ -2435,7 +2330,7 @@ TEST(ColumnMapperScanRequestTest,
ColumnPredicatesDoNotForceRowPredicateMaterial
EXPECT_EQ(request.local_positions.at(LocalColumnId(0)), LocalIndex(0));
EXPECT_EQ(request.local_positions.at(LocalColumnId(1)), LocalIndex(1));
ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(0));
+ EXPECT_EQ(request.column_predicate_filters[0].file_column_id,
LocalColumnId(0));
}
TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback)
{
@@ -2495,11 +2390,7 @@ TEST(ColumnMapperScanRequestTest,
StructOutputAndFilterOnlyChildAreMerged) {
EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(5));
ASSERT_FALSE(request.predicate_columns[0].project_all_children);
EXPECT_EQ(projection_ids(request.predicate_columns[0].children),
std::vector<int32_t>({0, 1}));
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({0}));
-
EXPECT_EQ(target_names(request.column_predicate_filters[0].target.struct_target.get()),
- std::vector<std::string>({"a"}));
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
TEST(ColumnMapperScanRequestTest,
RenamedNestedPredicateTargetsMappedFileChild) {
@@ -2523,12 +2414,7 @@ TEST(ColumnMapperScanRequestTest,
RenamedNestedPredicateTargetsMappedFileChild)
FileScanRequest request;
ASSERT_TRUE(mapper.create_scan_request({filter}, {}, {table_struct},
&request).ok());
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(5));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({1}));
-
EXPECT_EQ(target_names(request.column_predicate_filters[0].target.struct_target.get()),
- std::vector<std::string>({"b"}));
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
TEST(ColumnMapperScanRequestTest,
NestedInNullAndReverseComparisonFiltersAreMerged) {
@@ -2566,18 +2452,7 @@ TEST(ColumnMapperScanRequestTest,
NestedInNullAndReverseComparisonFiltersAreMerg
FileScanRequest request;
ASSERT_TRUE(mapper.create_scan_request({filter}, {}, {table_struct},
&request).ok());
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(5));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({0}));
-
EXPECT_EQ(target_names(request.column_predicate_filters[0].target.struct_target.get()),
- std::vector<std::string>({"a"}));
- ASSERT_EQ(request.column_predicate_filters[0].predicates.size(), 4);
- EXPECT_EQ(request.column_predicate_filters[0].predicates[0]->type(),
PredicateType::IN_LIST);
- EXPECT_EQ(request.column_predicate_filters[0].predicates[1]->type(),
PredicateType::GT);
- EXPECT_EQ(request.column_predicate_filters[0].predicates[2]->type(),
PredicateType::IS_NULL);
- EXPECT_EQ(request.column_predicate_filters[0].predicates[3]->type(),
- PredicateType::IS_NOT_NULL);
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
TEST(ColumnMapperScanRequestTest, NestedPredicateFilterThroughSafeCast) {
@@ -2608,12 +2483,7 @@ TEST(ColumnMapperScanRequestTest,
NestedPredicateFilterThroughSafeCast) {
FileScanRequest request;
ASSERT_TRUE(mapper.create_scan_request({filter}, {}, {table_struct},
&request).ok());
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(5));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({0}));
- ASSERT_EQ(request.column_predicate_filters[0].predicates.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].predicates[0]->type(),
PredicateType::GT);
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
TEST(ColumnMapperScanRequestTest, UnsafeCastDoesNotBuildNestedPredicateFilter)
{
@@ -2683,14 +2553,7 @@ TEST(ColumnMapperScanRequestTest,
DeepNestedPredicateTargetsLeafPath) {
FileScanRequest request;
ASSERT_TRUE(mapper.create_scan_request({filter}, {}, {table_struct},
&request).ok());
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(5));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({0, 0}));
-
EXPECT_EQ(target_names(request.column_predicate_filters[0].target.struct_target.get()),
- std::vector<std::string>({"a", "id"}));
- ASSERT_EQ(request.column_predicate_filters[0].predicates.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].predicates[0]->type(),
PredicateType::IN_LIST);
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
TEST(ColumnMapperScanRequestTest, ArrayStructProjectionPrunesElementChildren) {
@@ -2811,8 +2674,8 @@ TEST(ColumnMapperScanRequestTest,
StructProjectionPrunesChildrenByName) {
}
// Scenario: a row filter reaches a struct child through an array wrapper
-// (`items.item.a > 5`). The nested predicate filter path only supports direct
struct paths, so
-// the mapper keeps this as a row predicate and reads the full array root for
predicate evaluation.
+// (`items.item.a > 5`). The mapper keeps this as a row predicate and reads
the full array root for
+// predicate evaluation.
TEST(ColumnMapperScanRequestTest,
ArrayWrapperDoesNotBuildNestedPredicateFilter) {
const auto int_type = i32();
const auto string_type = str();
@@ -3035,14 +2898,11 @@ TEST(ColumnMapperScanRequestTest,
PredicateOnlyTopLevelColumnUsesHiddenMapping)
EXPECT_TRUE(request.predicate_columns[0].children.empty());
ASSERT_EQ(request.conjuncts.size(), 1);
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(10));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({0}));
+ EXPECT_TRUE(request.column_predicate_filters.empty());
}
-// Scenario: a nested predicate targets a table-side renamed struct field;
both predicate pruning and
-// scan projection must resolve that field to the old physical file child.
+// Scenario: a nested predicate targets a table-side renamed struct field;
scan projection must
+// resolve that field to the old physical file child.
TEST(ColumnMapperScanRequestTest,
NestedPredicateProjectionUsesMappedRenamedChild) {
const auto int_type = i32();
@@ -3065,17 +2925,14 @@ TEST(ColumnMapperScanRequestTest,
NestedPredicateProjectionUsesMappedRenamedChil
FileScanRequest request;
ASSERT_TRUE(mapper.create_scan_request({filter}, {}, {table_struct},
&request).ok());
- ASSERT_EQ(request.column_predicate_filters.size(), 1);
- EXPECT_EQ(request.column_predicate_filters[0].effective_file_column_id(),
LocalColumnId(10));
-
EXPECT_EQ(request.column_predicate_filters[0].effective_file_child_id_path(),
- std::vector<int32_t>({1}));
+ EXPECT_TRUE(request.column_predicate_filters.empty());
ASSERT_EQ(request.predicate_columns.size(), 1);
EXPECT_TRUE(request.predicate_columns[0].project_all_children);
EXPECT_TRUE(request.predicate_columns[0].children.empty());
}
// Scenario: element_at(struct, 'table_name') in a row filter is localized to
the physical file
-// child name, matching the struct_element rewrite and nested predicate filter
resolution paths.
+// child name, matching the struct_element rewrite path.
TEST(ColumnMapperScanRequestTest,
FileLocalElementAtConjunctUsesFileChildNameForRenamedStructField) {
const auto int_type = i32();
diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp
b/be/test/format_v2/parquet/parquet_reader_test.cpp
index ec71ebc614d..7fbfacd5d5d 100644
--- a/be/test/format_v2/parquet/parquet_reader_test.cpp
+++ b/be/test/format_v2/parquet/parquet_reader_test.cpp
@@ -1650,7 +1650,7 @@ TEST_F(NewParquetReaderTest,
ScanRangeFiltersRowGroupsBeforeDictionaryPruning) {
EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0);
}
-TEST_F(NewParquetReaderTest,
NestedStructPredicateFiltersRowGroupsByStatistics) {
+TEST_F(NewParquetReaderTest,
NestedStructPredicateDoesNotFilterRowGroupsByStatistics) {
write_struct_filter_parquet_file(_file_path);
auto parquet_file_reader =
::parquet::ParquetFileReader::OpenFile(_file_path, false);
ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2);
@@ -1667,7 +1667,6 @@ TEST_F(NewParquetReaderTest,
NestedStructPredicateFiltersRowGroupsByStatistics)
format::FileScanRequest request;
format::FileColumnPredicateFilter column_filter;
column_filter.file_column_id = format::LocalColumnId(0);
- column_filter.file_child_id_path = {0};
auto id_type = std::make_shared<DataTypeInt32>();
column_filter.predicates.push_back(create_comparison_predicate<PredicateType::GT>(
0, "id", id_type, Field::create_field<TYPE_INT>(5), false));
@@ -1679,15 +1678,14 @@ TEST_F(NewParquetReaderTest,
NestedStructPredicateFiltersRowGroupsByStatistics)
parquet_file_reader.get(), file_schema,
request, scan_range,
false, &plan)
.ok());
- ASSERT_EQ(plan.row_groups.size(), 1);
- EXPECT_EQ(plan.row_groups[0].row_group_id, 1);
+ ASSERT_EQ(plan.row_groups.size(), 2);
EXPECT_EQ(plan.pruning_stats.total_row_groups, 2);
- EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1);
- EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1);
- EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2);
+ EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2);
+ EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0);
+ EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0);
}
-TEST_F(NewParquetReaderTest,
NestedStructPredicateFiltersRowGroupsByDictionary) {
+TEST_F(NewParquetReaderTest,
NestedStructPredicateDoesNotFilterRowGroupsByDictionary) {
write_nested_dictionary_filter_parquet_file(_file_path);
auto parquet_file_reader =
::parquet::ParquetFileReader::OpenFile(_file_path, false);
ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6);
@@ -1712,7 +1710,6 @@ TEST_F(NewParquetReaderTest,
NestedStructPredicateFiltersRowGroupsByDictionary)
format::FileScanRequest request;
format::FileColumnPredicateFilter column_filter;
column_filter.file_column_id = format::LocalColumnId(0);
- column_filter.file_child_id_path = {1};
auto name_type = std::make_shared<DataTypeString>();
column_filter.predicates.push_back(create_comparison_predicate<PredicateType::EQ>(
0, "name", name_type, Field::create_field<TYPE_STRING>("lm"),
false));
@@ -1724,12 +1721,11 @@ TEST_F(NewParquetReaderTest,
NestedStructPredicateFiltersRowGroupsByDictionary)
parquet_file_reader.get(), file_schema,
request, scan_range,
false, &plan)
.ok());
- ASSERT_EQ(plan.row_groups.size(), 1);
- EXPECT_EQ(plan.row_groups[0].row_group_id, 2);
+ ASSERT_EQ(plan.row_groups.size(), 6);
EXPECT_EQ(plan.pruning_stats.total_row_groups, 6);
- EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1);
- EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5);
- EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5);
+ EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6);
+ EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0);
+ EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0);
}
TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) {
@@ -1790,7 +1786,7 @@ TEST_F(NewParquetReaderTest,
PlannerNarrowsRowRangesByPageIndex) {
EXPECT_EQ(plan.pruning_stats.selected_row_ranges,
plan.row_groups[0].selected_ranges.size());
}
-TEST_F(NewParquetReaderTest, NestedStructPredicateNarrowsRowRangesByPageIndex)
{
+TEST_F(NewParquetReaderTest,
NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) {
write_nested_page_index_filter_parquet_file(_file_path);
auto parquet_file_reader =
::parquet::ParquetFileReader::OpenFile(_file_path, false);
ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1);
@@ -1814,7 +1810,6 @@ TEST_F(NewParquetReaderTest,
NestedStructPredicateNarrowsRowRangesByPageIndex) {
format::FileScanRequest request;
format::FileColumnPredicateFilter column_filter;
column_filter.file_column_id = format::LocalColumnId(0);
- column_filter.file_child_id_path = {0};
auto id_type = std::make_shared<DataTypeInt32>();
column_filter.predicates.push_back(create_comparison_predicate<PredicateType::GT>(
0, "id", id_type, Field::create_field<TYPE_INT>(63), false));
@@ -1828,26 +1823,14 @@ TEST_F(NewParquetReaderTest,
NestedStructPredicateNarrowsRowRangesByPageIndex) {
.ok());
ASSERT_EQ(plan.row_groups.size(), 1);
ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty());
- EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0);
- EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128);
- auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0);
- ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end());
- EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0);
- EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0);
- EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1);
- ASSERT_EQ(skip_plan_it->second.skipped_pages.size(),
- skip_plan_it->second.skipped_page_compressed_sizes.size());
- int64_t skipped_compressed_bytes = 0;
- for (size_t page_idx = 0; page_idx <
skip_plan_it->second.skipped_pages.size(); ++page_idx) {
- if (skip_plan_it->second.should_skip_page(page_idx)) {
- skipped_compressed_bytes +=
skip_plan_it->second.skipped_page_compressed_size(page_idx);
- }
- }
- EXPECT_GT(skipped_compressed_bytes, 0);
+ EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0);
+ EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length,
+ parquet_file_reader->metadata()->RowGroup(0)->num_rows());
+ EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty());
EXPECT_EQ(plan.pruning_stats.total_row_groups, 1);
EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1);
EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0);
- EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0);
+ EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0);
EXPECT_EQ(plan.pruning_stats.selected_row_ranges,
plan.row_groups[0].selected_ranges.size());
}
diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp
b/be/test/format_v2/parquet/parquet_statistics_test.cpp
index f2ae2448013..ae9ee4bf1e3 100644
--- a/be/test/format_v2/parquet/parquet_statistics_test.cpp
+++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp
@@ -57,7 +57,6 @@ format::FileColumnPredicateFilter bloom_filter_with_predicate(
const std::shared_ptr<ColumnPredicate>& predicate) {
format::FileColumnPredicateFilter filter;
filter.file_column_id = format::LocalColumnId(0);
- filter.target = format::FileNestedPredicateTarget(filter.file_column_id);
filter.predicates.push_back(predicate);
return filter;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
