This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1fa293f681c4 [SPARK-51081][SQL] Refactor `Join.output` for single-pass
resolver
1fa293f681c4 is described below
commit 1fa293f681c49a50336eb1953e7a4ff0cdde2676
Author: Mihailo Timotic <[email protected]>
AuthorDate: Wed Feb 5 14:47:44 2025 +0800
[SPARK-51081][SQL] Refactor `Join.output` for single-pass resolver
### What changes were proposed in this pull request?
Refactor `Join.output` so that single-pass resolver can call it without
calling underlying `output`, which is recursive
### Why are the changes needed?
To avoid recursive calls in single-pass resolver.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #49794 from mihailotim-db/mihailotim-db/refactor_join_output.
Authored-by: Mihailo Timotic <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../plans/logical/basicLogicalOperators.scala | 40 +++++++++++++---------
1 file changed, 24 insertions(+), 16 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 9a62f67c2d44..5e43e6603278 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -593,6 +593,29 @@ case class Union(
copy(children = newChildren)
}
+object Join {
+ def computeOutput(
+ joinType: JoinType,
+ leftOutput: Seq[Attribute],
+ rightOutput: Seq[Attribute]
+ ): Seq[Attribute] = {
+ joinType match {
+ case j: ExistenceJoin =>
+ leftOutput :+ j.exists
+ case LeftExistence(_) =>
+ leftOutput
+ case LeftOuter | LeftSingle =>
+ leftOutput ++ rightOutput.map(_.withNullability(true))
+ case RightOuter =>
+ leftOutput.map(_.withNullability(true)) ++ rightOutput
+ case FullOuter =>
+ leftOutput.map(_.withNullability(true)) ++
rightOutput.map(_.withNullability(true))
+ case _ =>
+ leftOutput ++ rightOutput
+ }
+ }
+}
+
case class Join(
left: LogicalPlan,
right: LogicalPlan,
@@ -628,22 +651,7 @@ case class Join(
}
}
- override def output: Seq[Attribute] = {
- joinType match {
- case j: ExistenceJoin =>
- left.output :+ j.exists
- case LeftExistence(_) =>
- left.output
- case LeftOuter | LeftSingle =>
- left.output ++ right.output.map(_.withNullability(true))
- case RightOuter =>
- left.output.map(_.withNullability(true)) ++ right.output
- case FullOuter =>
- left.output.map(_.withNullability(true)) ++
right.output.map(_.withNullability(true))
- case _ =>
- left.output ++ right.output
- }
- }
+ override def output: Seq[Attribute] = Join.computeOutput(joinType,
left.output, right.output)
override def metadataOutput: Seq[Attribute] = {
joinType match {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
