This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new efc3a2c5874b [SPARK-60575][SQL] Preserve compound operand grouping in
connector IN expressions
efc3a2c5874b is described below
commit efc3a2c5874b049e1b73e07c13285930ba389d9e
Author: Linhong Liu <[email protected]>
AuthorDate: Wed Oct 28 10:25:27 2026 -0700
[SPARK-60575][SQL] Preserve compound operand grouping in connector IN
expressions
### What changes were proposed in this pull request?
Reuse the existing operand-grouping helper when rendering the left operand
of IN, including the CASE expression generated for an empty list.
### Why are the changes needed?
Without grouping, `(NOT a) IN (true, false)` renders as `NOT (a) IN (true,
false)`. With `a = true`, the intended expression is true but the generated
expression is false. Other compound operands can also parse differently.
### Does this PR introduce _any_ user-facing change?
Yes, a bug fix. Generated connector SQL preserves the intended meaning of
compound IN operands. Simple column references retain their existing rendering.
### How was this patch tested?
Added exact SQL assertions and 108 evaluations covering OR, NOT and
comparison operands, TRUE/FALSE/NULL inputs, and empty or non-empty IN lists.
The new semantic test fails on unchanged Apache code because an empty IN with
an OR operand returns NULL instead of false. On JDK 17, all 22 V2PredicateSuite
tests and three JDBC filter/grouping tests passed. Catalyst Java Checkstyle,
SQL test Scalastyle and git diff --check passed.
```bash
build/sbt 'sql/testOnly
org.apache.spark.sql.execution.datasources.v2.V2PredicateSuite'
build/sbt 'sql/testOnly org.apache.spark.sql.jdbc.JDBCSuite -- -z "compile
filters" -z "IS [NOT] NULL"'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
Closes #59813 from linhongliu-db/fix-in-operand-precedence.
Authored-by: Linhong Liu <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 2bc5e240fadc94ae31924b4c5ab5ef43ca897efa)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../sql/connector/util/V2ExpressionSQLBuilder.java | 4 +-
.../datasources/v2/V2PredicateSuite.scala | 51 +++++++++++++++++++++-
2 files changed, 53 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
index 3ceb131d2cec..9d80bf730e48 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
@@ -125,7 +125,9 @@ public class V2ExpressionSQLBuilder {
case "IN" -> {
Expression[] expressions = e.children();
List<String> children = expressionsToStringList(expressions, 1,
expressions.length - 1);
- yield visitIn(build(expressions[0]), children);
+ // IN needs the same operand grouping as IS NULL: without it, NOT a
IN (...)
+ // means NOT (a IN (...)) rather than (NOT a) IN (...).
+ yield visitIn(visitIsNullOperand(expressions[0]), children);
}
case "IS_NULL" -> visitIsNull(visitIsNullOperand(e.children()[0]));
case "IS_NOT_NULL" ->
visitIsNotNull(visitIsNullOperand(e.children()[0]));
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
index 932a105dd392..96d9f998b64c 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2PredicateSuite.scala
@@ -18,12 +18,15 @@
package org.apache.spark.sql.execution.datasources.v2
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.expressions.{Literal => CatalystLiteral}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.connector.expressions.{Expression, FieldReference,
Literal, LiteralValue}
import org.apache.spark.sql.connector.expressions.filter._
import org.apache.spark.sql.execution.datasources.v2.V2PredicateSuite.ref
import org.apache.spark.sql.internal.connector.PredicateUtils
import org.apache.spark.sql.sources.{AlwaysFalse => V1AlwaysFalse, AlwaysTrue
=> V1AlwaysTrue, And => V1And, EqualNullSafe, EqualTo, GreaterThan,
GreaterThanOrEqual, In, IsNotNull, IsNull, LessThan, LessThanOrEqual, Not =>
V1Not, Or => V1Or, StringContains, StringEndsWith, StringStartsWith}
-import org.apache.spark.sql.types.{IntegerType, StringType}
+import org.apache.spark.sql.types.{BooleanType, IntegerType, StringType}
import org.apache.spark.unsafe.types.UTF8String
class V2PredicateSuite extends SparkFunSuite {
@@ -173,6 +176,52 @@ class V2PredicateSuite extends SparkFunSuite {
assert(PredicateUtils.toV1(predicate1).get.toV2 == predicate1)
}
+ for ((operand, expected) <- Seq(
+ (new Predicate("OR", Array[Expression](ref("a"), ref("b"))),
+ "((a) OR (b)) IN (true, false)"),
+ (new Predicate("NOT", Array[Expression](ref("a"))), "(NOT (a)) IN (true,
false)"),
+ (new Predicate("=", Array[Expression](ref("a"), ref("b"))), "(a = b) IN
(true, false)"))) {
+ test(s"IN preserves value operand precedence: ${operand.name()}") {
+ val predicate = new Predicate("IN", Array[Expression](
+ operand, LiteralValue(true, BooleanType), LiteralValue(false,
BooleanType)))
+ assert(predicate.describe() == expected)
+ val empty = new Predicate("IN", Array[Expression](operand))
+ val groupedOperand = expected.stripSuffix(" IN (true, false)")
+ assert(empty.describe() == s"CASE WHEN $groupedOperand IS NULL THEN NULL
ELSE FALSE END")
+ }
+ }
+
+ test("IN preserves three-valued logic for compound operands") {
+ def evaluate(sql: String, a: java.lang.Boolean, b: java.lang.Boolean): Any
= {
+ CatalystSqlParser.parseExpression(sql).transform {
+ case attr: UnresolvedAttribute =>
+ CatalystLiteral.create(if (attr.name == "a") a else b, BooleanType)
+ }.eval()
+ }
+
+ val values = Seq[java.lang.Boolean](true, false, null)
+ for {
+ (operand, expectedOperand) <- Seq(
+ (new Predicate("OR", Array[Expression](ref("a"), ref("b"))), "(a OR
b)"),
+ (new Predicate("NOT", Array[Expression](ref("a"))), "(NOT a)"),
+ (new Predicate("=", Array[Expression](ref("a"), ref("b"))), "(a = b)"))
+ a <- values
+ b <- values
+ list <- Seq(Seq.empty, Seq(true), Seq(false), Seq(true, false))
+ } {
+ val predicate = new Predicate("IN", Array[Expression](operand) ++
+ list.map(value => LiteralValue(value, BooleanType)))
+ val expected = if (list.isEmpty) {
+ s"CASE WHEN $expectedOperand IS NULL THEN NULL ELSE FALSE END"
+ } else {
+ s"$expectedOperand IN (${list.mkString(", ")})"
+ }
+ withClue(s"${predicate.describe()}, a=$a, b=$b: ") {
+ assert(evaluate(predicate.describe(), a, b) == evaluate(expected, a,
b))
+ }
+ }
+ }
+
test("In") {
val predicate1 = new Predicate("IN",
Array(ref("a"), LiteralValue(1, IntegerType), LiteralValue(2,
IntegerType),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
