alamb commented on code in PR #14567:
URL: https://github.com/apache/datafusion/pull/14567#discussion_r1954515692
##########
datafusion/physical-optimizer/src/pruning.rs:
##########
@@ -1710,6 +1711,76 @@ fn build_like_match(
Some(combined)
}
+// For predicate `col NOT LIKE 'const_prefix%'`, we rewrite it as `(col_min
NOT LIKE 'const_prefix%' OR col_max NOT LIKE 'const_prefix%')`. If both col_min
and col_max have the prefix const_prefix, we skip the entire row group (as we
can be certain that all data in this row group has the prefix const_prefix).
Review Comment:
I have a suggestion to explain the intuition here
```suggestion
// For predicate `col NOT LIKE 'const_prefix%'`, we rewrite it as `(col_min
NOT LIKE 'const_prefix%' OR col_max NOT LIKE 'const_prefix%')`.
//
// The intuition is that if both `col_min` and `col_max` begin with
`const_prefix` that means
// **all** data in this row group begins with `const_prefix` as well (and
therefore the predicate
// looking for rows that don't begin with `const_prefix` can never be true)
```
##########
datafusion/physical-optimizer/src/pruning.rs:
##########
@@ -4061,6 +4132,106 @@ mod tests {
prune_with_expr(expr, &schema, &statistics, expected_ret);
}
+ #[test]
+ fn prune_utf8_not_like_one() {
+ let (schema, statistics) = utf8_setup();
+
+ let expr = col("s1").not_like(lit("A\u{10ffff}_"));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["A", "Z"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["A", "L"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["N", "Z"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["M", "M"] ==> some rows could pass (must keep)
+ true,
+ // s1 [NULL, NULL] ==> unknown (must keep)
+ true,
+ // s1 ["A", NULL] ==> some rows could pass (must keep)
+ true,
+ // s1 ["", "A"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["", ""] ==> some rows could pass (must keep)
+ true,
+ // s1 ["AB", "A\u{10ffff}\u{10ffff}\u{10ffff}"] ==> some rows
could pass (must keep)
+ true,
+ // s1 ["A\u{10ffff}\u{10ffff}", "A\u{10ffff}\u{10ffff}"] ==> no
row match. (min, max) maybe truncate
+ // orignal (min, max) maybe ("A\u{10ffff}\u{10ffff}\u{10ffff}",
"A\u{10ffff}\u{10ffff}\u{10ffff}\u{10ffff}")
+ true,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+ }
+
+ #[test]
+ fn prune_utf8_not_like_many() {
+ let (schema, statistics) = utf8_setup();
+
+ let expr = col("s1").not_like(lit("A\u{10ffff}%"));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["A", "Z"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["A", "L"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["N", "Z"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["M", "M"] ==> some rows could pass (must keep)
+ true,
+ // s1 [NULL, NULL] ==> unknown (must keep)
+ true,
+ // s1 ["A", NULL] ==> some rows could pass (must keep)
+ true,
+ // s1 ["", "A"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["", ""] ==> some rows could pass (must keep)
+ true,
+ // s1 ["AB", "A\u{10ffff}\u{10ffff}\u{10ffff}"] ==> some rows
could pass (must keep)
+ true,
+ // s1 ["A\u{10ffff}\u{10ffff}", "A\u{10ffff}\u{10ffff}"] ==> no
row match
+ false,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ let expr = col("s1").not_like(lit("A\u{10ffff}%\u{10ffff}"));
+ #[rustfmt::skip]
+ let expected_ret = &[
+ // s1 ["A", "Z"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["A", "L"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["N", "Z"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["M", "M"] ==> some rows could pass (must keep)
+ true,
+ // s1 [NULL, NULL] ==> unknown (must keep)
+ true,
+ // s1 ["A", NULL] ==> some rows could pass (must keep)
+ true,
+ // s1 ["", "A"] ==> some rows could pass (must keep)
+ true,
+ // s1 ["", ""] ==> some rows could pass (must keep)
+ true,
+ // s1 ["AB", "A\u{10ffff}\u{10ffff}\u{10ffff}"] ==> some rows
could pass (must keep)
+ true,
+ // s1 ["A\u{10ffff}\u{10ffff}", "A\u{10ffff}\u{10ffff}"] ==> some
rows could pass (must keep)
+ true,
+ ];
+ prune_with_expr(expr, &schema, &statistics, expected_ret);
+
+ let expr = col("s1").not_like(lit("A\\%%"));
Review Comment:
Can you please add some negative tests that verify predicates like `NOT LIKE
'foo%bar'` and `NOT LIKE 'foo%bar_` (aka non empty additional patterns) can not
be used to prune anything?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
