This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f75e66f8810 [improvement](parquet) Prune array_contains with row group
statistics (#67441)
f75e66f8810 is described below
commit f75e66f8810b1fa369c2f4d4003593be67fdd887
Author: Gabriel <[email protected]>
AuthorDate: Mon Sep 7 10:59:19 2026 +0800
[improvement](parquet) Prune array_contains with row group statistics
(#67441)
## Summary
- Enable `array_contains(array_column, constant)` to participate in
expression-level ZoneMap evaluation for supported primitive array
elements.
- Resolve the logical ARRAY slot to its physical Parquet LIST element
leaf in File Scanner V2 and use row-group footer min/max statistics to
reject impossible matches.
- Keep missing, incompatible, or unsafe statistics conservative, and
deliberately exclude repeated leaves from page-index pruning because
pages do not preserve parent-row boundaries.
- Generalize the nested metadata probe path so Parquet Bloom and ZoneMap
pruning share the same validated physical leaf resolution.
- Preserve scalar Page Index pruning when the predicate is wrapped by
`RuntimeFilterExpr`.
- Keep Page Index scan test doubles structurally equivalent to
production scalar comparisons so the physical probe classifier sees
their slot and literal children.
## Fix boundary
- The metadata-safe `array_contains` opt-in is limited to signatures
accepted by the current BE execution dispatch. Complex ARRAY, MAP, and
STRUCT element signatures remain unsupported and are kept outside the
safe prefix so metadata pruning cannot suppress their residual runtime
error.
- `RuntimeFilterExpr` is unwrapped only for Page Index capability and
path classification. The original wrapper remains in the scan request
and is used for actual evaluation.
- This change does not add Page Index pruning for repeated ARRAY leaves;
`array_contains` continues to use row-group footer statistics only.
- The TeamCity follow-up changes only test fixtures. The production Page
Index admission gate remains conservative.
## Testing
- Added RED/GREEN coverage for complex-element `array_contains`
safe-prefix rejection.
- Added RED/GREEN coverage for runtime-filter-only Page Index admission
and range selection.
- Reproduced all three BE UT failures from TeamCity build 1037684
locally before the test-fixture fix; all three pass afterward.
- `TableReaderTest.*`, `NativeParquetStatisticsTest.*`,
`ExprZonemapFilterTest.*`, and `ParquetBloomFilterPruningTest.*` (194
related tests passed).
- Page Index coverage from `NewParquetReaderTest` and `ParquetScanTest`
(130 related tests passed).
- clang-format 16 dry run on all changed C/C++ files.
- `build-support/check-build-hygiene.sh`.
- `git diff --check`.
## Links
None
---
be/src/exprs/expr_zonemap_filter.cpp | 123 +++++++++++--
be/src/exprs/expr_zonemap_filter.h | 27 ++-
be/src/exprs/function/array/function_array_index.h | 17 ++
be/src/exprs/vectorized_fn_call.cpp | 34 +++-
be/src/exprs/vin_predicate.cpp | 4 +-
be/src/format_v2/parquet/parquet_statistics.cpp | 112 ++++++++---
be/test/exprs/expr_zonemap_filter_test.cpp | 56 +++++-
be/test/format_v2/parquet/parquet_reader_test.cpp | 9 +-
be/test/format_v2/parquet/parquet_scan_test.cpp | 9 +-
.../format_v2/parquet/parquet_statistics_test.cpp | 205 ++++++++++++++++++++-
be/test/format_v2/table_reader_test.cpp | 78 ++++++++
11 files changed, 610 insertions(+), 64 deletions(-)
diff --git a/be/src/exprs/expr_zonemap_filter.cpp
b/be/src/exprs/expr_zonemap_filter.cpp
index 878f4c8908c..339123c7aad 100644
--- a/be/src/exprs/expr_zonemap_filter.cpp
+++ b/be/src/exprs/expr_zonemap_filter.cpp
@@ -28,6 +28,7 @@
#include "common/config.h"
#include "common/logging.h"
#include "core/column/column.h"
+#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_nullable.h"
#include "core/string_ref.h"
#include "exprs/hybrid_set.h"
@@ -104,7 +105,7 @@ bool floating_point_bloom_filter_may_contain(const
segment_v2::BloomFilter& bloo
return value == T {0} && test_value(-value);
}
-bool bloom_filter_probes_equal(const BloomFilterProbe& lhs, const
BloomFilterProbe& rhs) {
+bool metadata_probes_equal(const MetadataProbe& lhs, const MetadataProbe& rhs)
{
return lhs.slot_index == rhs.slot_index && lhs.path == rhs.path &&
data_types_compatible(lhs.value_type, rhs.value_type);
}
@@ -249,12 +250,71 @@ std::optional<SlotLiteral> extract_slot_and_literal(const
VExprSPtrs& args) {
return std::nullopt;
}
-std::optional<BloomFilterProbe> extract_bloom_filter_probe(const VExprSPtr&
expr) {
+std::optional<SlotLiteral> extract_array_contains_slot_and_literal(const
VExprSPtrs& args) {
+ if (args.size() != 2) {
+ return std::nullopt;
+ }
+ auto probe = extract_metadata_probe(args[0]);
+ auto literal = field_from_literal_expr(args[1]);
+ if (!probe.has_value() || !literal.has_value()) {
+ return std::nullopt;
+ }
+ const auto array_type = remove_nullable(probe->value_type);
+ if (array_type == nullptr || array_type->get_primitive_type() !=
TYPE_ARRAY) {
+ return std::nullopt;
+ }
+ const auto& element_type = assert_cast<const
DataTypeArray&>(*array_type).get_nested_type();
+ if (element_type == nullptr ||
+ is_complex_type(remove_nullable(element_type)->get_primitive_type())) {
+ return std::nullopt;
+ }
+ auto [literal_value, literal_type] = std::move(*literal);
+ return SlotLiteral {.slot_index = probe->slot_index,
+ .slot_type = element_type,
+ .literal = std::move(literal_value),
+ .literal_type = std::move(literal_type),
+ .literal_on_left = false};
+}
+
+ZoneMapFilterResult evaluate_array_contains_zonemap(const ZoneMapEvalContext&
ctx,
+ const VExprSPtrs& args) {
+ auto slot_literal = extract_array_contains_slot_and_literal(args);
+ DORIS_CHECK(slot_literal.has_value());
+ auto slot_type = ctx.data_type(slot_literal->slot_index);
+ if (slot_type == nullptr || !data_types_compatible(slot_type,
slot_literal->slot_type)) {
+ // Shared Segment and legacy Parquet callers bind the logical ARRAY,
while Format V2 binds
+ // its repeated element; only the latter context can safely evaluate
element statistics.
+ return unsupported_zonemap_filter(ctx);
+ }
+ auto zone_map_ref = ctx.zone_map(slot_literal->slot_index);
+ if (zone_map_ref == nullptr) {
+ return unsupported_zonemap_filter(ctx);
+ }
+ const auto& zone_map = *zone_map_ref;
+ if (!zone_map.has_not_null) {
+ return ZoneMapFilterResult::kNoMatch;
+ }
+ if (!range_stats_usable_for_zonemap(zone_map, slot_type)) {
+ return unsupported_zonemap_filter(ctx);
+ }
+ return slot_literal->literal < zone_map.min_value || zone_map.max_value <
slot_literal->literal
+ ? ZoneMapFilterResult::kNoMatch
+ : ZoneMapFilterResult::kMayMatch;
+}
+
+bool can_evaluate_array_contains_zonemap(const VExprSPtrs& args) {
+ auto slot_literal = extract_array_contains_slot_and_literal(args);
+ return slot_literal.has_value() && !slot_literal->literal.is_null() &&
+ !slot_literal->literal.is_nan() &&
+ data_types_compatible(slot_literal->slot_type,
slot_literal->literal_type);
+}
+
+std::optional<MetadataProbe> extract_metadata_probe(const VExprSPtr& expr) {
if (expr == nullptr || expr->data_type() == nullptr) {
return std::nullopt;
}
if (auto slot = std::dynamic_pointer_cast<VSlotRef>(expr); slot) {
- return BloomFilterProbe {
+ return MetadataProbe {
.slot_index = slot->column_id(), .value_type =
slot->data_type(), .path = {}};
}
if ((expr->fn().name.function_name != "element_at" &&
@@ -263,7 +323,7 @@ std::optional<BloomFilterProbe>
extract_bloom_filter_probe(const VExprSPtr& expr
return std::nullopt;
}
- auto probe = extract_bloom_filter_probe(expr->get_child(0));
+ auto probe = extract_metadata_probe(expr->get_child(0));
auto selector = field_from_literal_expr(expr->get_child(1));
if (!probe.has_value() || !selector.has_value() ||
selector->first.is_null()) {
return std::nullopt;
@@ -273,10 +333,10 @@ std::optional<BloomFilterProbe>
extract_bloom_filter_probe(const VExprSPtr& expr
return std::nullopt;
}
- BloomFilterPathElement path_element;
+ MetadataPathElement path_element;
switch (parent_type->get_primitive_type()) {
case TYPE_STRUCT: {
- path_element.kind = BloomFilterPathKind::STRUCT_FIELD;
+ path_element.kind = MetadataPathKind::STRUCT_FIELD;
const auto selector_type = remove_nullable(selector->second);
if (selector_type == nullptr) {
return std::nullopt;
@@ -295,7 +355,7 @@ std::optional<BloomFilterProbe>
extract_bloom_filter_probe(const VExprSPtr& expr
case TYPE_ARRAY:
// Array element positions share one repeated Parquet leaf; membership
in that leaf is a
// necessary condition for any element_at(array, constant) equality to
match.
- path_element.kind = BloomFilterPathKind::LIST_ELEMENT;
+ path_element.kind = MetadataPathKind::LIST_ELEMENT;
break;
default:
return std::nullopt;
@@ -305,11 +365,28 @@ std::optional<BloomFilterProbe>
extract_bloom_filter_probe(const VExprSPtr& expr
return probe;
}
-bool collect_unique_bloom_filter_probe(const VExprSPtr& expr,
- std::optional<BloomFilterProbe>*
result) {
+bool collect_unique_metadata_probe(const VExprSPtr& expr, bool zonemap,
+ std::optional<MetadataProbe>* result) {
DORIS_CHECK(result != nullptr);
- if (auto probe = extract_bloom_filter_probe(expr); probe.has_value()) {
- if (result->has_value() && !bloom_filter_probes_equal(**result,
*probe)) {
+ std::optional<MetadataProbe> probe;
+ if (zonemap && expr != nullptr && expr->fn().name.function_name ==
"array_contains" &&
+ expr->get_num_children() == 2) {
+ probe = extract_metadata_probe(expr->get_child(0));
+ if (probe.has_value()) {
+ const auto array_type = remove_nullable(probe->value_type);
+ if (array_type == nullptr || array_type->get_primitive_type() !=
TYPE_ARRAY) {
+ return false;
+ }
+ probe->value_type = assert_cast<const
DataTypeArray&>(*array_type).get_nested_type();
+ MetadataPathElement path_element;
+ path_element.kind = MetadataPathKind::LIST_ELEMENT;
+ probe->path.push_back(std::move(path_element));
+ }
+ } else {
+ probe = extract_metadata_probe(expr);
+ }
+ if (probe.has_value()) {
+ if (result->has_value() && !metadata_probes_equal(**result, *probe)) {
return false;
}
*result = std::move(probe);
@@ -323,18 +400,26 @@ bool collect_unique_bloom_filter_probe(const VExprSPtr&
expr,
if (child == nullptr || child->is_literal()) {
continue;
}
- // Every Bloom-capable branch must bind to the same leaf; a
conflicting subtree cannot be
- // treated like a branch without a probe because the compound
evaluator would use it.
- if (!collect_unique_bloom_filter_probe(child, result)) {
+ // Every metadata-capable branch must bind to the same leaf; a
conflicting subtree cannot
+ // be treated like a branch without a probe because the compound
evaluator would use it.
+ if (!collect_unique_metadata_probe(child, zonemap, result)) {
return false;
}
}
return true;
}
-std::optional<BloomFilterProbe> extract_bloom_filter_predicate_probe(const
VExprSPtr& expr) {
- std::optional<BloomFilterProbe> result;
- if (!collect_unique_bloom_filter_probe(expr, &result)) {
+std::optional<MetadataProbe> extract_bloom_filter_predicate_probe(const
VExprSPtr& expr) {
+ std::optional<MetadataProbe> result;
+ if (!collect_unique_metadata_probe(expr, false, &result)) {
+ return std::nullopt;
+ }
+ return result;
+}
+
+std::optional<MetadataProbe> extract_zonemap_filter_predicate_probe(const
VExprSPtr& expr) {
+ std::optional<MetadataProbe> result;
+ if (!collect_unique_metadata_probe(expr, true, &result)) {
return std::nullopt;
}
return result;
@@ -345,7 +430,7 @@ std::optional<SlotLiteral>
extract_bloom_filter_slot_and_literal(const VExprSPtr
return std::nullopt;
}
for (size_t probe_idx = 0; probe_idx < args.size(); ++probe_idx) {
- auto probe = extract_bloom_filter_probe(args[probe_idx]);
+ auto probe = extract_metadata_probe(args[probe_idx]);
auto literal = field_from_literal_expr(args[1 - probe_idx]);
if (!probe.has_value() || !literal.has_value()) {
continue;
@@ -551,7 +636,7 @@ ZoneMapFilterResult eval_in_bloom_filter(const
BloomFilterEvalContext& ctx,
if (is_not_in) {
return ZoneMapFilterResult::kUnsupported;
}
- auto probe = extract_bloom_filter_probe(slot_expr);
+ auto probe = extract_metadata_probe(slot_expr);
DORIS_CHECK(probe.has_value());
const auto* slot_filter = ctx.slot(probe->slot_index);
if (slot_filter == nullptr || slot_filter->data_type == nullptr ||
diff --git a/be/src/exprs/expr_zonemap_filter.h
b/be/src/exprs/expr_zonemap_filter.h
index fb0a92f7c35..311d0fb8c65 100644
--- a/be/src/exprs/expr_zonemap_filter.h
+++ b/be/src/exprs/expr_zonemap_filter.h
@@ -97,32 +97,41 @@ struct SlotLiteral {
bool literal_on_left;
};
-enum class BloomFilterPathKind {
+enum class MetadataPathKind {
STRUCT_FIELD,
LIST_ELEMENT,
};
-struct BloomFilterPathElement {
- BloomFilterPathKind kind;
+struct MetadataPathElement {
+ MetadataPathKind kind;
std::string field_name;
int32_t field_ordinal = -1;
- bool operator==(const BloomFilterPathElement&) const = default;
+ bool operator==(const MetadataPathElement&) const = default;
};
-struct BloomFilterProbe {
+struct MetadataProbe {
int slot_index;
DataTypePtr value_type;
- std::vector<BloomFilterPathElement> path;
+ std::vector<MetadataPathElement> path;
- bool operator==(const BloomFilterProbe&) const = default;
+ bool operator==(const MetadataProbe&) const = default;
};
std::optional<SlotLiteral> extract_slot_and_literal(const VExprSPtrs& args);
-std::optional<BloomFilterProbe> extract_bloom_filter_probe(const VExprSPtr&
expr);
+std::optional<SlotLiteral> extract_array_contains_slot_and_literal(const
VExprSPtrs& args);
-std::optional<BloomFilterProbe> extract_bloom_filter_predicate_probe(const
VExprSPtr& expr);
+ZoneMapFilterResult evaluate_array_contains_zonemap(const ZoneMapEvalContext&
ctx,
+ const VExprSPtrs& args);
+
+bool can_evaluate_array_contains_zonemap(const VExprSPtrs& args);
+
+std::optional<MetadataProbe> extract_metadata_probe(const VExprSPtr& expr);
+
+std::optional<MetadataProbe> extract_bloom_filter_predicate_probe(const
VExprSPtr& expr);
+
+std::optional<MetadataProbe> extract_zonemap_filter_predicate_probe(const
VExprSPtr& expr);
std::optional<SlotLiteral> extract_bloom_filter_slot_and_literal(const
VExprSPtrs& args);
diff --git a/be/src/exprs/function/array/function_array_index.h
b/be/src/exprs/function/array/function_array_index.h
index 0984e803792..f456b3b1629 100644
--- a/be/src/exprs/function/array/function_array_index.h
+++ b/be/src/exprs/function/array/function_array_index.h
@@ -22,6 +22,7 @@
#include <stddef.h>
#include <memory>
+#include <type_traits>
#include <utility>
#include "common/status.h"
@@ -45,6 +46,7 @@
#include "core/field.h"
#include "core/string_ref.h"
#include "core/types.h"
+#include "exprs/expr_zonemap_filter.h"
#include "exprs/function/function.h"
#include "storage/index/index_reader_helper.h"
#include "storage/index/inverted/inverted_index_query_type.h"
@@ -218,6 +220,21 @@ public:
return _execute_dispatch(block, arguments, result, input_rows_count);
}
+ ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx,
+ const VExprSPtrs& arguments)
const override {
+ if constexpr (!std::is_same_v<ConcreteAction, ArrayContainsAction>) {
+ return unsupported_zonemap_filter(ctx);
+ }
+ return expr_zonemap::evaluate_array_contains_zonemap(ctx, arguments);
+ }
+
+ bool can_evaluate_zonemap_filter(const VExprSPtrs& arguments) const
override {
+ if constexpr (!std::is_same_v<ConcreteAction, ArrayContainsAction>) {
+ return false;
+ }
+ return expr_zonemap::can_evaluate_array_contains_zonemap(arguments);
+ }
+
private:
template <PrimitiveType PType>
ColumnPtr _execute_view(const ColumnArrayView<PType>& array_view,
diff --git a/be/src/exprs/vectorized_fn_call.cpp
b/be/src/exprs/vectorized_fn_call.cpp
index ec6881dbb84..0d1fbd5f77b 100644
--- a/be/src/exprs/vectorized_fn_call.cpp
+++ b/be/src/exprs/vectorized_fn_call.cpp
@@ -36,12 +36,14 @@
#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/block/column_numbers.h"
+#include "core/call_on_type_index.h"
#include "core/column/column.h"
#include "core/column/column_array.h"
#include "core/column/column_nullable.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_agg_state.h"
+#include "core/data_type/data_type_array.h"
#include "core/types.h"
#include "exec/common/util.hpp"
#include "exec/pipeline/pipeline_task.h"
@@ -684,13 +686,35 @@ bool
VectorizedFnCall::is_safe_to_execute_on_selected_rows() const {
"is_not_null_pred",
"element_at",
"struct_element"};
+ bool function_is_total =
TOTAL_PREDICATE_FUNCTIONS.contains(_function_name);
+ if (_function_name == "array_contains" && get_num_children() == 2) {
+ const auto& left_child_type = get_child(0)->data_type();
+ const auto& right_child_type = get_child(1)->data_type();
+ if (left_child_type != nullptr && right_child_type != nullptr) {
+ const auto array_type = remove_nullable(left_child_type);
+ const auto right_type = remove_nullable(right_child_type);
+ if (array_type->get_primitive_type() == TYPE_ARRAY) {
+ const auto element_type = remove_nullable(
+ assert_cast<const
DataTypeArray&>(*array_type).get_nested_type());
+ const auto element_primitive_type =
element_type->get_primitive_type();
+ const auto right_primitive_type =
right_type->get_primitive_type();
+ const bool types_match = element_primitive_type ==
right_primitive_type ||
+
(is_string_type(element_primitive_type) &&
+
is_string_type(right_primitive_type));
+ // Match the executable dispatch domain so metadata pruning
cannot hide an
+ // unsupported-signature error from a row rejected by a later
predicate.
+ function_is_total =
+ types_match &&
dispatch_switch_all(element_primitive_type,
+ [](const auto&) {
return true; });
+ }
+ }
+ }
// Selected-row execution may hide data-dependent errors in rows rejected
by an earlier
// predicate. Keep function calls unsafe by default and opt in only
operations that are total
- // for their input domain. Accessors return NULL for absent elements, so
admitting them keeps
- // nested metadata predicates reachable without crossing an
error-producing child such as
- // gt(mod(x, -1), 0).
- return TOTAL_PREDICATE_FUNCTIONS.contains(_function_name) &&
- VExpr::is_safe_to_execute_on_selected_rows();
+ // for their input domain. Accessors return NULL for absent elements, and
supported membership
+ // signatures are total, so admitting them keeps nested metadata
predicates reachable without
+ // crossing an error-producing child such as gt(mod(x, -1), 0).
+ return function_is_total && VExpr::is_safe_to_execute_on_selected_rows();
}
bool VectorizedFnCall::equals(const VExpr& other) {
diff --git a/be/src/exprs/vin_predicate.cpp b/be/src/exprs/vin_predicate.cpp
index 0a0c43b7eaf..b7466d959e4 100644
--- a/be/src/exprs/vin_predicate.cpp
+++ b/be/src/exprs/vin_predicate.cpp
@@ -167,7 +167,7 @@ void VInPredicate::_prepare_zonemap_min_max(VExprContext*
context) {
return;
}
- auto bloom_probe = expr_zonemap::extract_bloom_filter_probe(_children[0]);
+ auto bloom_probe = expr_zonemap::extract_metadata_probe(_children[0]);
if (!bloom_probe.has_value()) {
return;
}
@@ -225,7 +225,7 @@ ZoneMapFilterResult
VInPredicate::evaluate_bloom_filter(const BloomFilterEvalCon
bool VInPredicate::can_evaluate_bloom_filter() const {
return _zonemap_min_max != nullptr && !_is_not_in &&
- expr_zonemap::extract_bloom_filter_probe(get_child(0)).has_value();
+ expr_zonemap::extract_metadata_probe(get_child(0)).has_value();
}
bool VInPredicate::can_execute_on_raw_fixed_values(const DataTypePtr&
data_type,
diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp
b/be/src/format_v2/parquet/parquet_statistics.cpp
index a894601a160..79cd23c5bb0 100644
--- a/be/src/format_v2/parquet/parquet_statistics.cpp
+++ b/be/src/format_v2/parquet/parquet_statistics.cpp
@@ -456,9 +456,9 @@ const ParquetColumnSchema* resolve_local_leaf_schema(
return column_schema;
}
-const ParquetColumnSchema* resolve_bloom_filter_leaf_schema(
+const ParquetColumnSchema* resolve_metadata_leaf_schema(
const std::vector<std::unique_ptr<ParquetColumnSchema>>& schema,
- const format::LocalColumnId file_column_id, const
expr_zonemap::BloomFilterProbe& probe) {
+ const format::LocalColumnId file_column_id, const
expr_zonemap::MetadataProbe& probe) {
if (probe.path.empty()) {
return resolve_local_leaf_schema(schema, file_column_id);
}
@@ -472,7 +472,7 @@ const ParquetColumnSchema* resolve_bloom_filter_leaf_schema(
if (column_schema == nullptr) {
return nullptr;
}
- if (path_element.kind ==
expr_zonemap::BloomFilterPathKind::STRUCT_FIELD) {
+ if (path_element.kind == expr_zonemap::MetadataPathKind::STRUCT_FIELD)
{
if (column_schema->kind != ParquetColumnSchemaKind::STRUCT) {
return nullptr;
}
@@ -863,6 +863,36 @@ bool has_expr_zonemap_filter(const
format::FileScanRequest& request, const Runti
return has_variant_shredded_filter(request);
}
+bool can_evaluate_native_page_index(const VExprSPtr& expr) {
+ if (expr == nullptr || !expr->can_evaluate_zonemap_filter()) {
+ return false;
+ }
+ if (const auto impl = expr->get_impl(); impl != nullptr) {
+ // RuntimeFilterExpr keeps its predicate outside the inherited child
list. Unwrap only for
+ // shape classification; pruning still evaluates the original wrapper
and its semantics.
+ return can_evaluate_native_page_index(impl);
+ }
+ if (expr->op() == TExprOpcode::COMPOUND_AND) {
+ return std::ranges::any_of(expr->children(),
can_evaluate_native_page_index);
+ }
+ if (expr->op() == TExprOpcode::COMPOUND_OR) {
+ return !expr->children().empty() &&
+ std::ranges::all_of(expr->children(),
can_evaluate_native_page_index);
+ }
+ const auto probe =
expr_zonemap::extract_zonemap_filter_predicate_probe(expr);
+ return probe.has_value() && probe->path.empty();
+}
+
+bool has_native_page_index_filter(const format::FileScanRequest& request) {
+ const auto conjuncts = metadata_pruning_conjuncts(request);
+ return std::ranges::any_of(conjuncts,
+ [](const auto& conjunct) {
+ return conjunct != nullptr &&
+
can_evaluate_native_page_index(conjunct->root());
+ }) ||
+ has_variant_shredded_filter(request);
+}
+
std::set<int> collect_expr_zonemap_slot_indexes(const VExprContextSPtrs&
conjuncts) {
std::set<int> slot_indexes;
for (const auto& conjunct : conjuncts) {
@@ -933,9 +963,10 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext&
ctx, ParquetPruningStats
} // namespace
-bool can_use_parquet_page_index(const format::FileScanRequest& request,
- const RuntimeState* runtime_state) {
- return config::enable_parquet_page_index &&
has_expr_zonemap_filter(request, runtime_state);
+bool can_use_parquet_page_index(const format::FileScanRequest& request, const
RuntimeState*) {
+ // Footer-only repeated paths cannot map physical value pages back to
parent rows, so they must
+ // not trigger Page Index I/O unless another predicate can actually use
the loaded indexes.
+ return config::enable_parquet_page_index &&
has_native_page_index_filter(request);
}
std::shared_ptr<segment_v2::ZoneMap> ParquetStatisticsUtils::MakeZoneMap(
@@ -1052,21 +1083,13 @@ bool check_native_statistics(const
tparquet::FileMetaData& metadata,
const format::FileScanRequest& request,
ParquetPruningStats* pruning_stats, const
cctz::time_zone* timezone) {
const auto conjuncts = metadata_pruning_conjuncts(request);
- const auto slot_indexes = collect_expr_zonemap_slot_indexes(conjuncts);
- if (slot_indexes.empty()) {
- return false;
- }
- ZoneMapEvalContext ctx;
- for (const int slot_index : slot_indexes) {
- const auto file_column_id = file_column_id_by_block_position(request,
slot_index);
- if (!file_column_id.has_value()) {
- continue;
- }
- const auto* column_schema = resolve_local_leaf_schema(file_schema,
*file_column_id);
+ const auto add_column_zonemap = [&](ZoneMapEvalContext* ctx, int
slot_index,
+ const ParquetColumnSchema*
column_schema) {
+ DORIS_CHECK(ctx != nullptr);
if (column_schema == nullptr || column_schema->type == nullptr ||
!native_metadata_predicate_is_type_safe(*column_schema) ||
column_schema->leaf_column_id >=
static_cast<int>(row_group.columns.size())) {
- continue;
+ return;
}
const auto& chunk = row_group.columns[column_schema->leaf_column_id];
std::shared_ptr<segment_v2::ZoneMap> zone_map;
@@ -1085,9 +1108,54 @@ bool check_native_statistics(const
tparquet::FileMetaData& metadata,
safe_statistics.has_value() ? &*safe_statistics :
nullptr,
column_metadata.num_values, timezone));
}
- add_slot_zonemap(&ctx, slot_index, column_schema->type,
std::move(zone_map));
+ add_slot_zonemap(ctx, slot_index, column_schema->type,
std::move(zone_map));
+ };
+
+ VExprContextSPtrs direct_conjuncts;
+ for (const auto& conjunct : conjuncts) {
+ if (conjunct == nullptr || conjunct->root() == nullptr ||
+ !conjunct->root()->can_evaluate_zonemap_filter()) {
+ direct_conjuncts.push_back(conjunct);
+ continue;
+ }
+ const auto probe =
expr_zonemap::extract_zonemap_filter_predicate_probe(conjunct->root());
+ if (!probe.has_value() || probe->path.empty()) {
+ direct_conjuncts.push_back(conjunct);
+ continue;
+ }
+ const auto file_column_id = file_column_id_by_block_position(request,
probe->slot_index);
+ // Row-group statistics safely summarize every repeated LIST value.
Page-index pruning keeps
+ // using top-level leaves because repeated values do not preserve
page-to-parent-row bounds.
+ const auto* column_schema =
+ file_column_id.has_value()
+ ? resolve_metadata_leaf_schema(file_schema,
*file_column_id, *probe)
+ : nullptr;
+ ZoneMapEvalContext nested_ctx;
+ if (column_schema != nullptr &&
+ expr_zonemap::data_types_compatible(column_schema->type,
probe->value_type)) {
+ add_column_zonemap(&nested_ctx, probe->slot_index, column_schema);
+ }
+ const auto result = VExprContext::evaluate_zonemap_filter({conjunct},
nested_ctx);
+ accumulate_zonemap_stats(nested_ctx, pruning_stats);
+ if (result == ZoneMapFilterResult::kNoMatch) {
+ return true;
+ }
+ }
+
+ const auto slot_indexes =
collect_expr_zonemap_slot_indexes(direct_conjuncts);
+ if (slot_indexes.empty()) {
+ return false;
+ }
+ ZoneMapEvalContext ctx;
+ for (const int slot_index : slot_indexes) {
+ const auto file_column_id = file_column_id_by_block_position(request,
slot_index);
+ if (!file_column_id.has_value()) {
+ continue;
+ }
+ const auto* column_schema = resolve_local_leaf_schema(file_schema,
*file_column_id);
+ add_column_zonemap(&ctx, slot_index, column_schema);
}
- const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx);
+ const auto result =
VExprContext::evaluate_zonemap_filter(direct_conjuncts, ctx);
accumulate_zonemap_stats(ctx, pruning_stats);
return result == ZoneMapFilterResult::kNoMatch;
}
@@ -1316,9 +1384,7 @@ ParquetRowGroupPruneReason
native_bloom_filter_prune_reason(
continue;
}
const auto* column_schema =
- probe->path.empty()
- ? resolve_local_leaf_schema(file_schema,
*file_column_id)
- : resolve_bloom_filter_leaf_schema(file_schema,
*file_column_id, *probe);
+ resolve_metadata_leaf_schema(file_schema, *file_column_id,
*probe);
if (column_schema == nullptr ||
!expr_zonemap::data_types_compatible(column_schema->type,
probe->value_type)) {
continue;
diff --git a/be/test/exprs/expr_zonemap_filter_test.cpp
b/be/test/exprs/expr_zonemap_filter_test.cpp
index f75502e1390..88dee1cb974 100644
--- a/be/test/exprs/expr_zonemap_filter_test.cpp
+++ b/be/test/exprs/expr_zonemap_filter_test.cpp
@@ -832,11 +832,11 @@ TEST(ExprZonemapFilterTest,
EqualityBloomAcceptsStructAndListLeafAccessors) {
"element_at", list_type, make_slot(0, nested_type),
make_string_literal("items"));
auto nested_leaf = std::make_shared<MetadataAccessorExpr>(
"element_at", leaf_type, std::move(nested_list),
make_int_literal(1));
- auto nested_probe = expr_zonemap::extract_bloom_filter_probe(nested_leaf);
+ auto nested_probe = expr_zonemap::extract_metadata_probe(nested_leaf);
ASSERT_TRUE(nested_probe.has_value());
ASSERT_EQ(nested_probe->path.size(), 2);
- EXPECT_EQ(nested_probe->path[0].kind,
expr_zonemap::BloomFilterPathKind::STRUCT_FIELD);
- EXPECT_EQ(nested_probe->path[1].kind,
expr_zonemap::BloomFilterPathKind::LIST_ELEMENT);
+ EXPECT_EQ(nested_probe->path[0].kind,
expr_zonemap::MetadataPathKind::STRUCT_FIELD);
+ EXPECT_EQ(nested_probe->path[1].kind,
expr_zonemap::MetadataPathKind::LIST_ELEMENT);
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
equals.evaluate_bloom_filter(bloom_ctx, {nested_leaf,
make_int_literal(2)}));
}
@@ -1035,6 +1035,56 @@ TEST(ExprZonemapFilterTest,
FunctionStringStartsWithZonemapUsesPrefixRange) {
{slot,
make_string_literal(max_byte_prefix)}));
}
+TEST(ExprZonemapFilterTest, FunctionArrayContainsZonemapUsesElementRange) {
+ auto element_type = int_type();
+ auto array_type =
std::make_shared<DataTypeArray>(make_nullable(element_type));
+ auto slot = make_slot(0, array_type);
+ auto array_contains = SimpleFunctionFactory::instance().get_function(
+ "array_contains",
+ ColumnsWithTypeAndName {{nullptr, array_type, "slot"},
+ {nullptr, element_type, "needle"}},
+ std::make_shared<DataTypeUInt8>());
+ ASSERT_NE(array_contains, nullptr);
+ EXPECT_EQ("array_contains", array_contains->get_name());
+
+ EXPECT_TRUE(array_contains->can_evaluate_zonemap_filter({slot,
make_int_literal(7)}));
+ EXPECT_FALSE(array_contains->can_evaluate_zonemap_filter({slot,
make_null_int_literal()}));
+ EXPECT_FALSE(array_contains->can_evaluate_zonemap_filter(
+ {make_int_literal(7), make_int_literal(7)}));
+
+ auto below_range_ctx = make_context(make_int_zonemap(10, 20),
element_type);
+ EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
array_contains->evaluate_zonemap_filter(
+ below_range_ctx, {slot,
make_int_literal(7)}));
+
+ auto above_range_ctx = make_context(make_int_zonemap(10, 20),
element_type);
+ EXPECT_EQ(
+ ZoneMapFilterResult::kNoMatch,
+ array_contains->evaluate_zonemap_filter(above_range_ctx, {slot,
make_int_literal(21)}));
+
+ auto overlap_ctx = make_context(make_int_zonemap(10, 20), element_type);
+ EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
+ array_contains->evaluate_zonemap_filter(overlap_ctx, {slot,
make_int_literal(15)}));
+
+ segment_v2::ZoneMap only_null;
+ only_null.has_null = true;
+ auto null_ctx = make_context(only_null, element_type);
+ EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
+ array_contains->evaluate_zonemap_filter(null_ctx, {slot,
make_int_literal(15)}));
+
+ auto missing_ctx = ZoneMapEvalContext {};
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ array_contains->evaluate_zonemap_filter(missing_ctx, {slot,
make_int_literal(15)}));
+ EXPECT_EQ(1, missing_ctx.stats.unusable_zonemap_eval_count);
+
+ // Segment and legacy Parquet callers bind the logical ARRAY instead of
the repeated element.
+ // The shared capability must fall back instead of treating that binding
as an invariant breach.
+ auto logical_array_ctx = make_context(make_int_zonemap(10, 20),
array_type);
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ array_contains->evaluate_zonemap_filter(logical_array_ctx,
+ {slot,
make_int_literal(15)}));
+ EXPECT_EQ(1, logical_array_ctx.stats.unusable_zonemap_eval_count);
+}
+
TEST(ExprZonemapFilterTest, CharZonemapUsesTrimmedLogicalBounds) {
auto char_type = std::make_shared<DataTypeString>(10, TYPE_CHAR);
auto slot = make_slot(0, char_type);
diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp
b/be/test/format_v2/parquet/parquet_reader_test.cpp
index d20a3fe8e94..38b6761c89d 100644
--- a/be/test/format_v2/parquet/parquet_reader_test.cpp
+++ b/be/test/format_v2/parquet/parquet_reader_test.cpp
@@ -114,7 +114,14 @@ public:
Int32GreaterThanExpr(int column_id, int32_t value)
: VExpr(std::make_shared<DataTypeUInt8>(), false),
_column_id(column_id),
- _value(value) {}
+ _value(value) {
+ _fn.name.function_name = "gt";
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ // Keep the test double structurally equivalent to a scalar comparison
because Page Index
+ // admission resolves the physical probe from expression children.
+ add_child(VSlotRef::create_shared(column_id, column_id, -1, int_type,
"c0"));
+ add_child(VLiteral::create_shared(int_type,
Field::create_field<TYPE_INT>(value)));
+ }
Status execute_column_impl(VExprContext* context, const Block* block,
const Selector* selector,
size_t count, ColumnPtr& result_column) const
override {
diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp
b/be/test/format_v2/parquet/parquet_scan_test.cpp
index 635355826a0..d12f78c9b10 100644
--- a/be/test/format_v2/parquet/parquet_scan_test.cpp
+++ b/be/test/format_v2/parquet/parquet_scan_test.cpp
@@ -174,7 +174,14 @@ public:
: VExpr(std::make_shared<DataTypeUInt8>(), false),
_column_id(column_id),
_op(op),
- _value(value) {}
+ _value(value) {
+ _fn.name.function_name = op == Op::GE ? "ge" : op == Op::GT ? "gt" :
"lt";
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ // Keep the test double structurally equivalent to a scalar comparison
because Page Index
+ // admission resolves the physical probe from expression children.
+ add_child(VSlotRef::create_shared(column_id, column_id, -1, int_type,
"c0"));
+ add_child(VLiteral::create_shared(int_type,
Field::create_field<TYPE_INT>(value)));
+ }
const std::string& expr_name() const override { return _expr_name; }
diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp
b/be/test/format_v2/parquet/parquet_statistics_test.cpp
index 48e4a157b94..4aa996ab00c 100644
--- a/be/test/format_v2/parquet/parquet_statistics_test.cpp
+++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp
@@ -47,6 +47,7 @@
#include "exprs/function/functions_comparison.h"
#include "exprs/hybrid_set.h"
#include "exprs/hybrid_set_min_max.h"
+#include "exprs/runtime_filter_expr.h"
#include "exprs/vcompound_pred.h"
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
@@ -316,6 +317,39 @@ private:
const std::string _expr_name = "MetadataAccessorExpr";
};
+class MetadataArrayContainsExpr final : public VExpr {
+public:
+ MetadataArrayContainsExpr(DataTypePtr array_type, int32_t value)
+ : VExpr(std::make_shared<DataTypeUInt8>(), false) {
+ _fn.name.function_name = "array_contains";
+ add_child(VSlotRef::create_shared(0, 0, -1, std::move(array_type),
"items"));
+ add_child(VLiteral::create_shared(std::make_shared<DataTypeInt32>(),
+
Field::create_field<TYPE_INT>(value)));
+ }
+
+ const std::string& expr_name() const override { return _expr_name; }
+
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
+ ColumnPtr&) const override {
+ return Status::InternalError("MetadataArrayContainsExpr is
metadata-only");
+ }
+
+ bool can_evaluate_zonemap_filter() const override {
+ return expr_zonemap::can_evaluate_array_contains_zonemap(_children);
+ }
+
+ ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx)
const override {
+ return expr_zonemap::evaluate_array_contains_zonemap(ctx, _children);
+ }
+
+ void collect_slot_column_ids(std::set<int>& column_ids) const override {
+ get_child(0)->collect_slot_column_ids(column_ids);
+ }
+
+private:
+ const std::string _expr_name = "MetadataArrayContainsExpr";
+};
+
class DictionaryStringInExpr final : public VExpr {
public:
DictionaryStringInExpr() : VExpr(std::make_shared<DataTypeUInt8>(), false)
{}
@@ -342,7 +376,12 @@ private:
class MetadataInt32GreaterThanExpr final : public VExpr {
public:
explicit MetadataInt32GreaterThanExpr(int32_t value)
- : VExpr(std::make_shared<DataTypeUInt8>(), false), _value(value) {}
+ : VExpr(std::make_shared<DataTypeUInt8>(), false), _value(value) {
+ _fn.name.function_name = "gt";
+ add_child(VSlotRef::create_shared(0, 0, -1,
std::make_shared<DataTypeInt32>(), "c0"));
+ add_child(VLiteral::create_shared(std::make_shared<DataTypeInt32>(),
+
Field::create_field<TYPE_INT>(value)));
+ }
const std::string& expr_name() const override { return _expr_name; }
Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
@@ -1820,6 +1859,170 @@ TEST(NativeParquetStatisticsTest,
TypeDefinedBoundsRequireSupportedColumnOrder)
EXPECT_TRUE(selected_ranges.empty());
}
+TEST(NativeParquetStatisticsTest,
RuntimeFilterWrapperKeepsScalarPageIndexPruning) {
+ auto encode_int32 = [](int32_t value) {
+ std::string bytes(sizeof(value), '\0');
+ memcpy(bytes.data(), &value, sizeof(value));
+ return bytes;
+ };
+
+ auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ column_schema->local_id = 0;
+ column_schema->leaf_column_id = 0;
+ column_schema->type = std::make_shared<DataTypeInt32>();
+ column_schema->type_descriptor.doris_type = column_schema->type;
+ column_schema->type_descriptor.physical_type = tparquet::Type::INT32;
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
+ schema.push_back(std::move(column_schema));
+
+ tparquet::ColumnChunk chunk;
+ tparquet::ColumnMetaData column_metadata;
+ column_metadata.__set_type(tparquet::Type::INT32);
+ column_metadata.__set_num_values(1);
+ chunk.__set_meta_data(column_metadata);
+ tparquet::RowGroup row_group;
+ row_group.__set_columns({chunk});
+ row_group.__set_num_rows(1);
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order});
+ metadata.__set_row_groups({row_group});
+
+ TExprNode runtime_filter_node;
+
runtime_filter_node.__set_type(std::make_shared<DataTypeUInt8>()->to_thrift());
+ runtime_filter_node.__set_is_nullable(false);
+ auto runtime_filter = RuntimeFilterExpr::create_shared(
+ runtime_filter_node,
std::make_shared<MetadataInt32GreaterThanExpr>(100), 0.0, false,
+ 7);
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ request.predicate_columns =
{format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ request.conjuncts =
{VExprContext::create_shared(std::move(runtime_filter))};
+ EXPECT_TRUE(format::parquet::can_use_parquet_page_index(request, nullptr));
+
+ format::parquet::NativeParquetPageIndex page_index;
+ page_index.column_index.__set_min_values({encode_int32(1)});
+ page_index.column_index.__set_max_values({encode_int32(2)});
+ page_index.column_index.__set_null_pages({false});
+ page_index.column_index.__set_null_counts({0});
+ tparquet::PageLocation location;
+ location.__set_offset(0);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(0);
+ page_index.offset_index.__set_page_locations({location});
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex>
page_indexes;
+ page_indexes.emplace(0, std::move(page_index));
+ std::vector<format::parquet::RowRange> selected_ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
+ ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, metadata.row_groups[0], page_indexes,
schema, request, 1,
+ &selected_ranges, &skip_plans, nullptr)
+ .ok());
+ EXPECT_TRUE(selected_ranges.empty());
+}
+
+TEST(NativeParquetStatisticsTest,
ArrayContainsUsesRepeatedLeafFooterStatisticsOnly) {
+ const auto encode_int32 = [](int32_t value) {
+ std::string bytes(sizeof(value), '\0');
+ memcpy(bytes.data(), &value, sizeof(value));
+ return bytes;
+ };
+
+ const auto leaf_type = std::make_shared<DataTypeInt32>();
+ const auto array_type = std::make_shared<DataTypeArray>(leaf_type);
+ auto root_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ root_schema->kind = format::parquet::ParquetColumnSchemaKind::LIST;
+ root_schema->local_id = 0;
+ root_schema->name = "items";
+ root_schema->type = array_type;
+ root_schema->max_repetition_level = 1;
+ auto leaf_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ leaf_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ leaf_schema->local_id = 0;
+ leaf_schema->name = "element";
+ leaf_schema->leaf_column_id = 0;
+ leaf_schema->type = leaf_type;
+ leaf_schema->type_descriptor.doris_type = leaf_type;
+ leaf_schema->type_descriptor.physical_type = tparquet::Type::INT32;
+ leaf_schema->max_repetition_level = 1;
+ root_schema->children.push_back(std::move(leaf_schema));
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
+ schema.push_back(std::move(root_schema));
+
+ tparquet::Statistics statistics;
+ statistics.__set_min_value(encode_int32(10));
+ statistics.__set_max_value(encode_int32(20));
+ statistics.__set_null_count(0);
+ tparquet::ColumnMetaData column_metadata;
+ column_metadata.__set_type(tparquet::Type::INT32);
+ column_metadata.__set_num_values(2);
+ column_metadata.__set_statistics(statistics);
+ tparquet::ColumnChunk chunk;
+ chunk.__set_meta_data(column_metadata);
+ tparquet::RowGroup row_group;
+ row_group.__set_columns({chunk});
+ row_group.__set_num_rows(1);
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order});
+ metadata.__set_row_groups({row_group});
+
+ const auto select_for_value = [&](int32_t value,
+ format::parquet::ParquetPruningStats*
pruning_stats) {
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ request.predicate_columns =
{format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ request.conjuncts = {VExprContext::create_shared(
+ std::make_shared<MetadataArrayContainsExpr>(array_type,
value))};
+ std::vector<int> selected_row_groups;
+ EXPECT_TRUE(format::parquet::select_row_groups_by_metadata(metadata,
schema, request,
+ nullptr,
&selected_row_groups,
+ false,
pruning_stats)
+ .ok());
+ return selected_row_groups;
+ };
+
+ format::parquet::ParquetPruningStats disjoint_stats;
+ EXPECT_TRUE(select_for_value(7, &disjoint_stats).empty());
+ EXPECT_EQ(disjoint_stats.filtered_row_groups_by_statistics, 1);
+ format::parquet::ParquetPruningStats overlap_stats;
+ EXPECT_EQ(select_for_value(15, &overlap_stats), std::vector<int>({0}));
+ metadata.row_groups[0].columns[0].meta_data.__isset.statistics = false;
+ format::parquet::ParquetPruningStats missing_stats;
+ EXPECT_EQ(select_for_value(7, &missing_stats), std::vector<int>({0}));
+ EXPECT_EQ(missing_stats.expr_zonemap_unusable_evals, 1);
+
+ format::FileScanRequest page_request;
+ page_request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ page_request.conjuncts = {VExprContext::create_shared(
+ std::make_shared<MetadataArrayContainsExpr>(array_type, 7))};
+ EXPECT_FALSE(format::parquet::can_use_parquet_page_index(page_request,
nullptr));
+ format::parquet::NativeParquetPageIndex page_index;
+ page_index.column_index.__set_min_values({encode_int32(10)});
+ page_index.column_index.__set_max_values({encode_int32(20)});
+ page_index.column_index.__set_null_pages({false});
+ page_index.column_index.__set_null_counts({0});
+ tparquet::PageLocation location;
+ location.__set_offset(0);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(0);
+ page_index.offset_index.__set_page_locations({location});
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex>
page_indexes;
+ page_indexes.emplace(0, std::move(page_index));
+ std::vector<format::parquet::RowRange> selected_ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
+ ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, metadata.row_groups[0], page_indexes,
schema, page_request, 1,
+ &selected_ranges, &skip_plans, nullptr)
+ .ok());
+ ASSERT_EQ(selected_ranges.size(), 1);
+ EXPECT_EQ(selected_ranges[0].start, 0);
+ EXPECT_EQ(selected_ranges[0].length, 1);
+}
+
TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
auto encode_int32 = [](int32_t value) {
std::string bytes(sizeof(value), '\0');
diff --git a/be/test/format_v2/table_reader_test.cpp
b/be/test/format_v2/table_reader_test.cpp
index 800f0040dd6..074479d5718 100644
--- a/be/test/format_v2/table_reader_test.cpp
+++ b/be/test/format_v2/table_reader_test.cpp
@@ -263,6 +263,17 @@ VExprSPtr table_int32_greater_than_expr(int slot_id, int
column_id, int32_t valu
return expr;
}
+VExprSPtr table_array_contains_int32_expr(int slot_id, int column_id, const
DataTypePtr& array_type,
+ int32_t value) {
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ auto expr =
+ table_function_expr("array_contains",
make_nullable(std::make_shared<DataTypeUInt8>()),
+ {array_type, int_type});
+ expr->add_child(VSlotRef::create_shared(slot_id, column_id, slot_id,
array_type, "items"));
+ expr->add_child(table_int32_literal(value));
+ return expr;
+}
+
VExprSPtr table_struct_int32_child_greater_than_expr(int slot_id, int
column_id,
const DataTypePtr&
struct_type,
int32_t child_ordinal,
int32_t value) {
@@ -1344,6 +1355,18 @@ public:
file_block->replace_by_position(
block_position.value(),
make_not_null_nullable_column(std::move(struct_column)));
+ } else if (file_column_id == LocalColumnId(3)) {
+ auto values = ColumnInt32::create();
+ values->insert_value(1);
+ values->insert_value(2);
+ auto nullable_values =
make_not_null_nullable_column(std::move(values));
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->get_data().assign({1, 2});
+ auto array_column =
+ ColumnArray::create(std::move(nullable_values),
std::move(offsets));
+ file_block->replace_by_position(
+ block_position.value(),
+
make_not_null_nullable_column(std::move(array_column)));
} else {
return Status::InvalidArgument("Unexpected fake file column id
{}",
file_column_id.value());
@@ -1642,6 +1665,61 @@ TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath)
{
ASSERT_TRUE(reader.close().ok());
}
+TEST(TableReaderTest, ArrayContainsReachesMetadataPruningSafeRequestPrefix) {
+ const auto raw_array_type =
std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>());
+ std::vector<ColumnDefinition> file_schema;
+ file_schema.push_back(make_file_column(3, "items", raw_array_type));
+ std::vector<ColumnDefinition> projected_columns;
+ projected_columns.push_back(make_table_column(3, "items", raw_array_type));
+ set_name_identifiers(&projected_columns);
+ const auto array_type = projected_columns[0].type;
+
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto fake_state = std::make_shared<FakeFileReaderState>();
+ FakeTableReader reader(file_schema, fake_state);
+ ASSERT_TRUE(reader.init({
+ .projected_columns = projected_columns,
+ .conjuncts = {prepared_conjunct(
+ &state,
+ table_array_contains_int32_expr(0,
0, array_type, 1))},
+ .format = FileFormat::PARQUET,
+ .scan_params = nullptr,
+ .io_ctx = nullptr,
+ .runtime_state = &state,
+ .scanner_profile = nullptr,
+ })
+ .ok());
+
+ SplitReadOptions split;
+ split.current_range.__set_path("fake-table-reader-input");
+ ASSERT_TRUE(reader.prepare_split(split).ok());
+ Block block = build_table_block(projected_columns);
+ bool eos = false;
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ ASSERT_NE(fake_state->last_request, nullptr);
+ ASSERT_EQ(fake_state->last_request->conjuncts.size(), 1);
+ EXPECT_EQ(fake_state->last_request->metadata_pruning_safe_conjunct_count,
1);
+
EXPECT_TRUE(fake_state->last_request->conjuncts[0]->root()->can_evaluate_zonemap_filter());
+ ASSERT_TRUE(reader.close().ok());
+}
+
+TEST(TableReaderTest, ComplexArrayContainsIsUnsafeForMetadataPruning) {
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ const auto inner_array_type = std::make_shared<DataTypeArray>(int_type);
+ const auto outer_array_type =
std::make_shared<DataTypeArray>(inner_array_type);
+ auto expr = table_function_expr("array_contains",
std::make_shared<DataTypeUInt8>(),
+ {outer_array_type, inner_array_type});
+ expr->add_child(VSlotRef::create_shared(0, 0, 0, outer_array_type,
"nested_items"));
+ expr->add_child(VLiteral::create_shared(
+ inner_array_type,
+ Field::create_field<TYPE_ARRAY>(Array
{Field::create_field<TYPE_INT>(1)})));
+
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto conjunct = prepared_conjunct(&state, expr);
+ EXPECT_FALSE(conjunct->root()->is_safe_to_execute_on_selected_rows());
+ conjunct->close();
+}
+
TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) {
std::vector<ColumnDefinition> projected_columns;
auto partition_column = make_table_column(0, "part",
std::make_shared<DataTypeInt32>());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
