cloud-fan commented on code in PR #58077:
URL: https://github.com/apache/spark/pull/58077#discussion_r3815458758
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala:
##########
@@ -165,14 +176,76 @@ case class InSubqueryExec(
}
}
+ // Three-valued IN semantics for multi-column subqueries (SPARK-58481).
+ // result rows are InternalRow objects; InSet uses TreeSet ordering which
treats null fields as
Review Comment:
**Nit:**
Please capitalize `Result` at the start of this sentence.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala:
##########
@@ -165,14 +176,76 @@ case class InSubqueryExec(
}
}
+ // Three-valued IN semantics for multi-column subqueries (SPARK-58481).
+ // result rows are InternalRow objects; InSet uses TreeSet ordering which
treats null fields as
+ // non-equal and therefore cannot distinguish a definitively-false candidate
(a non-null field
+ // differs) from an indeterminate one (all non-null fields match but some
fields are null).
+ // We replicate In.eval's per-candidate logic: TRUE if any candidate matches
exactly,
+ // UNKNOWN if no TRUE and at least one candidate is indeterminate, FALSE
otherwise.
+ private def evalMultiColumn(inputRow: InternalRow): Any = {
+ val value = child.eval(inputRow)
+ if (value == null) return null
+ val inputStruct = value.asInstanceOf[InternalRow]
+ val numFields = plan.output.length
+ var hasUnknown = false
+ var i = 0
+ while (i < result.length) {
+ val candidate = result(i).asInstanceOf[InternalRow]
+ var fieldIdx = 0
+ var candidateIsUnknown = false
+ var candidateIsFalse = false
+ while (fieldIdx < numFields && !candidateIsFalse) {
+ val inputField = inputStruct.get(fieldIdx,
plan.output(fieldIdx).dataType)
+ val candidateField = candidate.get(fieldIdx,
plan.output(fieldIdx).dataType)
+ if (candidateField == null || inputField == null) {
+ candidateIsUnknown = true
+ } else if (!inputField.equals(candidateField)) {
Review Comment:
**Blocking:**
Use Catalyst's interpreted ordering for each field here. `Object.equals` is
not SQL equality for supported values such as `BinaryType` byte arrays, so
equal binary fields from separately materialized rows compare unequal and a
multi-column `IN` can incorrectly return FALSE. Please use the field data
type's Catalyst ordering and add a binary-column regression case.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala:
##########
@@ -165,14 +176,76 @@ case class InSubqueryExec(
}
}
+ // Three-valued IN semantics for multi-column subqueries (SPARK-58481).
+ // result rows are InternalRow objects; InSet uses TreeSet ordering which
treats null fields as
+ // non-equal and therefore cannot distinguish a definitively-false candidate
(a non-null field
+ // differs) from an indeterminate one (all non-null fields match but some
fields are null).
+ // We replicate In.eval's per-candidate logic: TRUE if any candidate matches
exactly,
+ // UNKNOWN if no TRUE and at least one candidate is indeterminate, FALSE
otherwise.
+ private def evalMultiColumn(inputRow: InternalRow): Any = {
+ val value = child.eval(inputRow)
+ if (value == null) return null
+ val inputStruct = value.asInstanceOf[InternalRow]
+ val numFields = plan.output.length
+ var hasUnknown = false
+ var i = 0
+ while (i < result.length) {
+ val candidate = result(i).asInstanceOf[InternalRow]
+ var fieldIdx = 0
+ var candidateIsUnknown = false
+ var candidateIsFalse = false
+ while (fieldIdx < numFields && !candidateIsFalse) {
+ val inputField = inputStruct.get(fieldIdx,
plan.output(fieldIdx).dataType)
Review Comment:
**Non-blocking:**
Capture `plan.output` before the candidate loop and the current field's data
type once per field. Both values are invariant, but these lookups currently run
twice for every field of every result row.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
