cloud-fan commented on code in PR #58077:
URL: https://github.com/apache/spark/pull/58077#discussion_r3812018372
##########
sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala:
##########
@@ -2678,4 +2678,52 @@ class SubquerySuite extends SharedSparkSession
assert(exposedAttribute.exprId == outerReferenceAttribute.exprId)
}
+
+ test("SPARK-58481: InSubqueryExec nullable correctly accounts for subquery
output nullability") {
+ // 5 NOT IN (99, NULL) is UNKNOWN, not TRUE or FALSE. A join condition
that is not TRUE
+ // matches no rows, so a FULL OUTER JOIN must emit null-padded rows for
every row in each
+ // side -- 3 + 3 = 6 null-padded rows -- not the full cross product (9
rows).
+ withTable("t0", "t1", "t3") {
+ sql("CREATE TABLE t0(c0 INT) USING PARQUET")
+ sql("INSERT INTO t0 VALUES (1), (2), (3)")
+ sql("CREATE TABLE t1(c0 INT) USING PARQUET")
+ sql("INSERT INTO t1 VALUES (10), (20), (30)")
+ sql("CREATE TABLE t3(c0 INT) USING PARQUET")
+ sql("INSERT INTO t3 VALUES (99), (CAST(NULL AS INT))")
+
+ // t1 rows are null-padded (no match on left), t0 rows are null-padded
(no match on right).
+ val expected = Seq(
+ Row(null, 10), Row(null, 20), Row(null, 30), // t0 side: null-padded
+ Row(1, null), Row(2, null), Row(3, null)) // t1 side: null-padded
+ checkAnswer(
+ sql("SELECT t0.c0, t1.c0 FROM t1 FULL OUTER JOIN t0 ON (5 NOT IN
(SELECT t3.c0 FROM t3))"),
+ expected)
+ }
+ }
+
+ test("SPARK-58481: multi-column IN subquery with nullable non-head output is
nullable") {
+ // The logical InSubquery.nullable checks ALL query outputs, not just the
first.
+ // The physical InSubqueryExec must do the same: for multi-column IN,
results are
+ // stored as InternalRow objects and InSet.hasNull only detects top-level
null elements,
+ // not null fields inside a row. If only plan.output.head.nullable were
checked, a
+ // subquery with a non-nullable first column but a nullable second column
would be
+ // incorrectly declared non-nullable, causing codegen to skip the
null-result path and
+ // evaluate NOT IN as !false = true instead of UNKNOWN.
+ // Here: lhs has (a NOT NULL, b NOT NULL); rhs has (a NOT NULL, b
nullable).
+ // (1,1) NOT IN ((99,99),(NULL_a,NULL_b)): no exact match and the set
contains a row
+ // with null fields => UNKNOWN => no row satisfies the ON condition
+ // => FULL OUTER JOIN emits null-padded rows from both sides (2 + 2 = 4
rows).
+ withTable("lhs", "rhs") {
+ sql("CREATE TABLE lhs(a INT NOT NULL, b INT NOT NULL) USING PARQUET")
+ sql("INSERT INTO lhs VALUES (1, 1), (2, 2)")
+ sql("CREATE TABLE rhs(a INT NOT NULL, b INT) USING PARQUET")
+ sql("INSERT INTO rhs VALUES (99, 99), (CAST(NULL AS INT), CAST(NULL AS
INT))")
+
+ val result = sql(
+ """SELECT lhs.a, rhs.a FROM lhs FULL OUTER JOIN rhs
+ |ON ((lhs.a, lhs.b) NOT IN (SELECT a, b FROM rhs))""".stripMargin)
+ assert(result.count() === 4,
Review Comment:
**Non-blocking:**
Please assert the actual rows here. The current broken runtime path makes
this join condition TRUE and returns a 2 x 2 cross product, which also has
count 4; the intended UNKNOWN path returns four null-padded rows. A row-level
assertion will distinguish those outputs and fail until nested-NULL propagation
is fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
