This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9f379f0b8268 [SPARK-55868][SQL][TESTS] Fix Predicate Pushdown for
InMemoryTable for V2Filters
9f379f0b8268 is described below
commit 9f379f0b826818d53e2248310b014148106f1aa0
Author: Szehon Ho <[email protected]>
AuthorDate: Sat Mar 7 10:29:36 2026 -0800
[SPARK-55868][SQL][TESTS] Fix Predicate Pushdown for InMemoryTable for
V2Filters
### What changes were proposed in this pull request?
Fix some Predicate (ie V2 Filter) name match in the filter pushdown code
(for the test table supporting V2Filter of InMemoryDataSource for DSV2)
### Why are the changes needed?
These Predicate names are not matching, and no filters are accepted by this
test table. This was found in https://github.com/apache/spark/pull/54459
### Does this PR introduce _any_ user-facing change?
No, tests only
### How was this patch tested?
Running affect tests
### Was this patch authored or co-authored using generative AI tooling?
Yes, cursor
Closes #54666 from szehon-ho/partition_test_fix.
Authored-by: Szehon Ho <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/connector/catalog/InMemoryTableWithV2Filter.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2Filter.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2Filter.scala
index 7c962ca1678f..f3559c84080a 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2Filter.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableWithV2Filter.scala
@@ -155,10 +155,10 @@ object InMemoryTableWithV2Filter {
} else {
value == attrVal
}
- case p: Predicate if p.name().equals("IS NULL") =>
+ case p: Predicate if p.name().equals("IS_NULL") =>
val attr = p.children()(0).toString
null == InMemoryBaseTable.extractValue(attr, partitionNames,
partValues)
- case p: Predicate if p.name().equals("IS NOT NULL") =>
+ case p: Predicate if p.name().equals("IS_NOT_NULL") =>
val attr = p.children()(0).toString
null != InMemoryBaseTable.extractValue(attr, partitionNames,
partValues)
case p: Predicate if p.name().equals("ALWAYS_TRUE") => true
@@ -172,8 +172,8 @@ object InMemoryTableWithV2Filter {
predicates.flatMap(splitAnd).forall {
case p: Predicate if p.name().equals("=") => true
case p: Predicate if p.name().equals("<=>") => true
- case p: Predicate if p.name().equals("IS NULL") => true
- case p: Predicate if p.name().equals("IS NOT NULL") => true
+ case p: Predicate if p.name().equals("IS_NULL") => true
+ case p: Predicate if p.name().equals("IS_NOT_NULL") => true
case p: Predicate if p.name().equals("ALWAYS_TRUE") => true
case _ => false
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
