alamb commented on code in PR #8654:
URL: https://github.com/apache/arrow-datafusion/pull/8654#discussion_r1437604724
##########
datafusion/physical-expr/src/utils/guarantee.rs:
##########
@@ -645,9 +680,107 @@ mod test {
);
}
- // TODO https://github.com/apache/arrow-datafusion/issues/8436
- // a IN (...)
- // b NOT IN (...)
+ #[test]
+ fn test_single_inlist() {
+ // b IN (1, 2, 3)
+ test_analyze(
+ col("b").in_list(vec![lit(1), lit(2), lit(3)], false),
+ vec![in_guarantee("b", [1, 2, 3])],
+ );
+ // b NOT IN (1, 2, 3)
+ test_analyze(
+ col("b").in_list(vec![lit(1), lit(2), lit(3)], true),
+ vec![not_in_guarantee("b", [1, 2, 3])],
+ );
+ }
+
+ #[test]
+ fn test_inlist_conjunction() {
+ // b IN (1, 2, 3) AND b IN (2, 3, 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").in_list(vec![lit(2), lit(3), lit(4)], false)),
+ vec![in_guarantee("b", [2, 3])],
+ );
+ // b NOT IN (1, 2, 3) AND b IN (2, 3, 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").in_list(vec![lit(2), lit(3), lit(4)], false)),
+ vec![
+ not_in_guarantee("b", [1, 2, 3]),
+ in_guarantee("b", [2, 3, 4]),
+ ],
+ );
+ // b NOT IN (1, 2, 3) AND b NOT IN (2, 3, 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").in_list(vec![lit(2), lit(3), lit(4)], true)),
+ vec![not_in_guarantee("b", [1, 2, 3, 4])],
+ );
+ // b IN (1, 2, 3) AND b = 4
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(4))),
+ vec![],
+ );
+ // b IN (1, 2, 3) AND b = 2
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(2))),
+ vec![in_guarantee("b", [2])],
+ );
+ // b IN (1, 2, 3) AND b != 2
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").not_eq(lit(2))),
+ vec![in_guarantee("b", [1, 2, 3]), not_in_guarantee("b", [2])],
+ );
+ // b NOT IN (1, 2, 3) AND b != 4
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").not_eq(lit(4))),
+ vec![not_in_guarantee("b", [1, 2, 3, 4])],
+ );
+ // b NOT IN (1, 2, 3) AND b != 2
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").not_eq(lit(2))),
+ vec![not_in_guarantee("b", [1, 2, 3])],
+ );
+ }
+
+ #[test]
+ fn test_inlist_with_disjunction() {
+ // b IN (1, 2, 3) AND (b = 3 OR b = 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(3)).or(col("b").eq(lit(4)))),
+ vec![in_guarantee("b", [3])],
+ );
+ // b IN (1, 2, 3) AND (b = 4 OR b = 5)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(4)).or(col("b").eq(lit(5)))),
+ vec![],
Review Comment:
> This would be invalid since intersection between [1,2,3] and [4,5] is
empty.
Right -- I wonder if in general for cases where the intersection is empty,
can we infer that the expression can not be true ever 🤔 (there is no way to
represent this today in the guarantee framework, maybe something like
```
enum Guarantee {
/// This predicate can not be true
CanNotBeTrue,
/// This predicate can only be true if all the `LiteralGurantee`s are valie
List(Vec<LiteralGurantee>)
}
And then return `Gurantee::CanNotBeTrue` if we discover that the expression
can not be true 🤔
> BTW, I left a comment in
https://github.com/apache/arrow-datafusion/pull/8437. Please cc when you are
free.
I am not quite sure what you are referring to.
https://github.com/apache/arrow-datafusion/pull/8437 doesn't seem to have had
any recent activity that I can see.
##########
datafusion/physical-expr/src/utils/guarantee.rs:
##########
@@ -645,9 +680,107 @@ mod test {
);
}
- // TODO https://github.com/apache/arrow-datafusion/issues/8436
- // a IN (...)
- // b NOT IN (...)
+ #[test]
+ fn test_single_inlist() {
+ // b IN (1, 2, 3)
+ test_analyze(
+ col("b").in_list(vec![lit(1), lit(2), lit(3)], false),
+ vec![in_guarantee("b", [1, 2, 3])],
+ );
+ // b NOT IN (1, 2, 3)
+ test_analyze(
+ col("b").in_list(vec![lit(1), lit(2), lit(3)], true),
+ vec![not_in_guarantee("b", [1, 2, 3])],
+ );
+ }
+
+ #[test]
+ fn test_inlist_conjunction() {
+ // b IN (1, 2, 3) AND b IN (2, 3, 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").in_list(vec![lit(2), lit(3), lit(4)], false)),
+ vec![in_guarantee("b", [2, 3])],
+ );
+ // b NOT IN (1, 2, 3) AND b IN (2, 3, 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").in_list(vec![lit(2), lit(3), lit(4)], false)),
+ vec![
+ not_in_guarantee("b", [1, 2, 3]),
+ in_guarantee("b", [2, 3, 4]),
+ ],
+ );
+ // b NOT IN (1, 2, 3) AND b NOT IN (2, 3, 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").in_list(vec![lit(2), lit(3), lit(4)], true)),
+ vec![not_in_guarantee("b", [1, 2, 3, 4])],
+ );
+ // b IN (1, 2, 3) AND b = 4
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(4))),
+ vec![],
+ );
+ // b IN (1, 2, 3) AND b = 2
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(2))),
+ vec![in_guarantee("b", [2])],
+ );
+ // b IN (1, 2, 3) AND b != 2
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").not_eq(lit(2))),
+ vec![in_guarantee("b", [1, 2, 3]), not_in_guarantee("b", [2])],
+ );
+ // b NOT IN (1, 2, 3) AND b != 4
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").not_eq(lit(4))),
+ vec![not_in_guarantee("b", [1, 2, 3, 4])],
+ );
+ // b NOT IN (1, 2, 3) AND b != 2
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], true)
+ .and(col("b").not_eq(lit(2))),
+ vec![not_in_guarantee("b", [1, 2, 3])],
+ );
+ }
+
+ #[test]
+ fn test_inlist_with_disjunction() {
+ // b IN (1, 2, 3) AND (b = 3 OR b = 4)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(3)).or(col("b").eq(lit(4)))),
+ vec![in_guarantee("b", [3])],
+ );
+ // b IN (1, 2, 3) AND (b = 4 OR b = 5)
+ test_analyze(
+ col("b")
+ .in_list(vec![lit(1), lit(2), lit(3)], false)
+ .and(col("b").eq(lit(4)).or(col("b").eq(lit(5)))),
+ vec![],
Review Comment:
> This would be invalid since intersection between [1,2,3] and [4,5] is
empty.
Right -- I wonder if in general for cases where the intersection is empty,
can we infer that the expression can not be true ever 🤔 (there is no way to
represent this today in the guarantee framework, maybe something like
```
enum Guarantee {
/// This predicate can not be true
CanNotBeTrue,
/// This predicate can only be true if all the `LiteralGurantee`s are valie
List(Vec<LiteralGurantee>)
}
```
And then return `Gurantee::CanNotBeTrue` if we discover that the expression
can not be true 🤔
> BTW, I left a comment in
https://github.com/apache/arrow-datafusion/pull/8437. Please cc when you are
free.
I am not quite sure what you are referring to.
https://github.com/apache/arrow-datafusion/pull/8437 doesn't seem to have had
any recent activity that I can see.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
