leaves12138 commented on code in PR #732:
URL: https://github.com/apache/paimon-rust/pull/732#discussion_r3812004984
##########
crates/integrations/datafusion/src/filter_pushdown.rs:
##########
@@ -334,6 +347,55 @@ impl<'a> FilterTranslator<'a> {
self.exact(predicate)
}
+ fn translate_array_function(&self, func: &ScalarFunction) ->
Option<TranslatedPredicate> {
+ if func.args.len() != 2 {
+ return None;
+ }
+ let field = self.resolve_field(&func.args[0])?;
+ let DataType::Array(array_type) = field.data_type() else {
+ return None;
+ };
+ let predicate = match func.name() {
+ "array_has" | "list_has" => {
+ let scalar = extract_scalar_literal(&func.args[1])?;
+ let literal = scalar_to_datum(scalar,
array_type.element_type())?;
+ self.predicate_builder
+ .array_contains(field.name(), literal)
+ .ok()?
+ }
+ "array_has_any" | "list_has_any" | "arrays_overlap" => {
+ let literals = extract_array_literals(&func.args[1],
array_type.element_type())?;
+ self.predicate_builder
+ .arrays_overlap(field.name(), literals)
+ .ok()?
+ }
+ "array_has_all" | "list_has_all" => {
+ let literals = extract_array_literals(&func.args[1],
array_type.element_type())?;
+ // DataFusion 54's empty-needle fast path currently returns
true
+ // even for a NULL haystack, while Paimon/Java
ARRAY_CONTAINS_ALL
+ // returns false for NULL arrays. Pushing it would remove rows
+ // before DataFusion can apply its own semantics.
+ if literals.is_empty() {
+ return None;
+ }
+ self.predicate_builder
+ .array_contains_all(field.name(), literals)
+ .ok()?
+ }
+ _ => return None,
+ };
+ Some(TranslatedPredicate {
+ predicate,
+ // Paimon's core residual follows Java Float.compare /
Double.compare
+ // and canonicalizes all NaNs. DataFusion's Arrow equality keeps
NaN
+ // payloads distinct, so retain its residual for floating arrays.
+ requires_residual: matches!(
Review Comment:
Keeping a DataFusion residual is not sufficient once this inexact predicate
is negated. For `NOT array_has(float_array, NaN(payload A))`, a row containing
`NaN(payload B)` matches the Paimon inner predicate because all NaNs are
canonicalized, but does not match the DataFusion inner predicate because the
payloads differ. After `NOT`, the Paimon predicate rejects a row that
DataFusion should keep, and the residual cannot recover a row already removed
by pushdown. Please avoid translating `NOT` when the translated child already
has `requires_residual` (or otherwise disable floating-array pushdown under
negation), and add a regression test for distinct NaN payloads.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
