HyukjinKwon commented on code in PR #59134:
URL: https://github.com/apache/spark/pull/59134#discussion_r4173425569
##########
sql/core/src/test/scala/org/apache/spark/sql/AsOfJoinSortMergeSQLSuite.scala:
##########
@@ -440,6 +448,136 @@ class AsOfJoinSortMergeSQLSuite extends QueryTest
Row(Timestamp.valueOf("2026-06-29 10:03:00"), 1, "v1.1")))
}
+ gridTest("NULL whole STRUCT column MATCH_CONDITION never matches")(Seq(
+ ("CAST(NULL AS STRUCT<a: INT>)", "named_struct('a', 1)",
"named_struct('a', 5)"),
+ ("CAST(NULL AS STRUCT<a: INT, b: INT>)",
+ "named_struct('a', 1, 'b', 1)", "named_struct('a', 5, 'b', 5)"),
+ ("CAST(NULL AS STRUCT<e: STRUCT<a: INT>>)",
+ "named_struct('e', named_struct('a', 1))", "named_struct('e',
named_struct('a', 5))"),
+ // Non-NULL values of this type are all equal.
+ ("CAST(NULL AS STRUCT<e: STRUCT<>>)",
+ "named_struct('e', named_struct())", "named_struct('e',
named_struct())"))) {
+ case (nullStruct, one, five) =>
+ // A NULL struct must never match, the same as a NULL scalar operand.
+ Seq(true, false).foreach { ansiEnabled =>
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+ def asOfTags(joinType: String, left: String, op: String, rights:
String): DataFrame =
+ sql(
+ s"""
+ |SELECT r.tag
+ |FROM VALUES ($left, 'l') AS t(k, tag)
+ |$joinType ASOF JOIN VALUES $rights AS r(k, tag)
+ | MATCH_CONDITION (t.k $op r.k)
+ |""".stripMargin)
+ checkSortMergeAsOf(
+ asOfTags("", nullStruct, ">=", s"($nullStruct, 'rnull'), ($one,
'r1')"), Nil)
+ checkSortMergeAsOf(asOfTags("", nullStruct, "<=", s"($one, 'r1')"),
Nil)
+ checkSortMergeAsOf(asOfTags("", five, ">=", s"($nullStruct,
'rnull')"), Nil)
+ // LEFT ASOF JOIN keeps the NULL left row with NULL right columns.
+ checkSortMergeAsOf(
+ asOfTags("LEFT", nullStruct, ">=", s"($nullStruct, 'rnull'),
($one, 'r1')"),
+ Row(null) :: Nil)
+ }
+ }
+ }
+
+ gridTest("whole STRUCT column MATCH_CONDITION matches a struct whose fields
are all NULL")(Seq(
+ ">=" -> Seq(Row("allnull")), "<=" -> Seq(Row("allnull")), ">" -> Nil,
"<" -> Nil)) {
+ case (op, expected) =>
+ // Two structs of NULL fields are equal, and a NULL struct never matches.
+ val allNull = "named_struct('a', CAST(NULL AS INT), 'b', CAST(NULL AS
INT))"
+ val nullStruct = "CAST(NULL AS STRUCT<a: INT, b: INT>)"
+ Seq(true, false).foreach { ansiEnabled =>
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+ checkSortMergeAsOf(
+ sql(
+ s"""
+ |SELECT r.tag
+ |FROM VALUES ($allNull, 'l') AS t(k, tag)
+ |ASOF JOIN VALUES ($nullStruct, 'rnull'), ($allNull,
'allnull') AS r(k, tag)
+ | MATCH_CONDITION (t.k $op r.k)
+ |""".stripMargin),
+ expected)
+ }
+ }
+ }
+
+ test("nested STRUCT column MATCH_CONDITION keeps a NULL inner struct apart
from NULL fields") {
+ // {e: NULL} sorts before {e: {a: NULL}}, as in a plain comparison.
+ val eNull = "named_struct('e', CAST(NULL AS STRUCT<a: INT>))"
+ val aNull = "named_struct('e', named_struct('a', CAST(NULL AS INT)))"
+ for {
+ (left, op, right) <- Seq((aNull, ">", eNull), (eNull, "<", aNull))
+ ansiEnabled <- Seq(true, false)
+ } {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
+ checkSortMergeAsOf(
+ sql(
+ s"""
+ |SELECT r.tag
+ |FROM VALUES ($left, 'l') AS t(k, tag)
+ |ASOF JOIN VALUES ($right, 'r') AS r(k, tag)
+ | MATCH_CONDITION (t.k $op r.k)
+ |""".stripMargin),
+ Row("r") :: Nil)
+ }
+ }
+ }
+
+ test("tuple vs nullable STRUCT column MATCH_CONDITION: a NULL struct does
not end the scan") {
+ // A NULL struct and {NULL, NULL} have the same fields, so a sort by
fields ties them. Both
Review Comment:
**Nit (P3):** This comment does not match what the test exercises. The right
operand `r.s` is a whole struct column, so `matchSortKeys` keeps it as a single
sort value (only a SQL tuple literal splits into per-field keys). Sorted
NullsFirst, the NULL struct (`rnull`) sorts strictly before the all-NULL-fields
struct (`allnull`) -- as the production comment puts it, "a NULL struct comes
before a struct of NULL fields" -- so they do not tie, and the NULL row sorts
first, not after a match. The "sort by fields ties them" reasoning only applies
to the decomposed left tuple, as the adjacent test at line 553 notes. Consider
rewording to describe the whole-struct ordering and the actual reason both
right-side input orders are exercised.
**Verification:**
- **Inspection:** Read the revised comment and confirm it describes
whole-struct ordering (NULL struct before all-NULL-fields struct, no tie)
consistent with matchSortKeys and the production comment in
basicLogicalOperators.scala.
- **Regression:** Run the 'tuple vs nullable STRUCT column MATCH_CONDITION'
test and confirm it still passes with the unchanged assertions, so the comment
edit did not alter behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
