mohitgurav20 commented on code in PR #25811:
URL: https://github.com/apache/datafusion/pull/25811#discussion_r4345147242
##########
datafusion/datasource-parquet/src/opener/mod.rs:
##########
@@ -1551,53 +1544,62 @@ impl RowGroupsPrunedParquetOpen {
continue;
}
};
- row_group_filters.insert(
- column_name,
- bf,
- *physical_type,
- *type_length,
- );
+ any_filter_loaded = true;
+
+ // Evaluate the predicate against *only* this column's
Bloom filter.
+ //
+ // Accumulating all loaded filters into a single
statistics object
+ // and re-running `predicate.prune()` after every column
would make
+ // membership-check complexity O(N²) for N filtered
columns (e.g. 528
+ // checks instead of 32 for N=32 when all values survive).
Because
+ // `PruningStatistics::contained` returns `None` for
columns absent
+ // from the statistics object, evaluating a single-column
filter is
+ // semantically equivalent: only this column can
contribute a pruning
+ // decision in this round, so short-circuit correctness is
preserved
+ // for both conjunctive and disjunctive predicates.
+ let mut col_filter =
BloomFilterStatistics::with_capacity(1);
+ col_filter.insert(column_name, bf, *physical_type,
*type_length);
+
+ let _timer_guard = bloom_filter_eval_time.timer();
+ match predicate.prune(&col_filter) {
Review Comment:
thanks for ur patience sir will work on it and follow ur steps
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
