dejankrak-db commented on code in PR #59843:
URL: https://github.com/apache/spark/pull/59843#discussion_r4399757406
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/JoinResolver.scala:
##########
@@ -275,14 +281,47 @@ class JoinResolver(
hiddenOutput = Some(newHiddenOutput)
)
- val resolvedJoin = partiallyResolvedJoin.copy(condition =
resolvedCondition)
+ val resolvedJoin = retainTopLevelJoinOutput(
+ unresolvedJoin = unresolvedJoin,
+ resolvedJoin = partiallyResolvedJoin.copy(condition = resolvedCondition),
+ conditionReferencedAttributes = conditionReferencedAttributes
+ )
cteRegistry.currentScope.tryPutWithCTE(
unresolvedOperator = unresolvedJoin,
resolvedOperator = resolvedJoin
)
}
+ /**
+ * Wraps a top-level [[Join]] in a [[Project]] with the join's output when
its condition
+ * references a column found only in the hidden output, such as the hidden
key of a USING join
+ * below, which is added to a child's output. For example:
+ *
+ * {{{
+ * FROM nt1 JOIN nt2 USING (k) JOIN nt2 x ON nt2.k = x.k
+ *
+ * Project [k#1, v1#2, v2#3, k#5, v2#6]
+ * +- Join Inner, (k#4 = k#5)
+ * :- Project [k#1, v1#2, v2#3, k#4]
+ * : +- Join Inner, (k#1 = k#4)
+ * +- ...
+ * }}}
+ */
+ private def retainTopLevelJoinOutput(
+ unresolvedJoin: Join,
+ resolvedJoin: Join,
+ conditionReferencedAttributes: HashMap[ExprId, Attribute]): LogicalPlan
= {
+ def referencesHiddenOutput: Boolean =
+
scopes.current.resolveMissingAttributesByHiddenOutput(conditionReferencedAttributes).nonEmpty
Review Comment:
Thanks, fixed in b869b85d609. The check now counts only hidden columns that
are in the join's own output (`resolvedJoin.outputSet`), which is when
fixed-point wraps the join. A LEFT SEMI or LEFT ANTI join outputs only its left
side, so a right-side hidden key no longer triggers the wrap. Added four
`using-join.sql` queries. A top-level LEFT SEMI join and a LEFT ANTI join over
a right-side USING join both stay bare joins; without the filter they fail with
`LOGICAL_PLAN_COMPARISON_MISMATCH`. A LEFT SEMI join on a left-side key and a
LEFT OUTER join on a right-side key are still wrapped in a `Project`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
