zhuqi-lucas commented on code in PR #23696:
URL: https://github.com/apache/datafusion/pull/23696#discussion_r3764788230
##########
datafusion/datasource-parquet/src/push_decoder.rs:
##########
@@ -356,15 +467,70 @@ impl PushDecoderStreamState {
}
self.rg_plan = kept;
}
- if pruned_count > 0 {
+
+ // Decide whether the per-row `RowFilter` needs to be
+ // toggled for the upcoming RG. `desired_filter` is
+ // `Some(true)` when the next RG needs a real filter,
+ // `Some(false)` when it's fully-matched (filter is a
+ // no-op, so we suppress it), and `None` when there is no
+ // pushdown predicate at all (toggling is meaningless).
+ let desired_filter: Option<bool> = self
+ .row_filter_context
+ .as_ref()
+ .and_then(|_| self.rg_plan.front().map(|e|
!e.fully_matched));
+ let filter_needs_toggle =
+ desired_filter.is_some_and(|want| want !=
self.filter_installed);
+
+ if pruned_count > 0 || filter_needs_toggle {
if self.rg_plan.is_empty() {
return None;
}
let decoder = self.decoder.take().expect("decoder
present");
let new_indices: Vec<usize> =
self.rg_plan.iter().map(|e| e.rg_index).collect();
let rebuilt = match decoder.into_builder() {
- Ok(b) => b.with_row_groups(new_indices).build(),
+ Ok(mut builder) => {
+ builder = builder.with_row_groups(new_indices);
Review Comment:
Pulled the advance/prune helpers out (e0fbd24).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
