This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 062bc4c25844 [SPARK-50659][SQL] Refactor Union output computation out
to reuse it in the single-pass Analyzer
062bc4c25844 is described below
commit 062bc4c258442d3752f6209a300c8a986f4e117f
Author: Vladimir Golubev <[email protected]>
AuthorDate: Wed Dec 25 14:37:03 2024 +0800
[SPARK-50659][SQL] Refactor Union output computation out to reuse it in the
single-pass Analyzer
### What changes were proposed in this pull request?
Refactor `computeOutput` out to the `Union` companion object.
### Why are the changes needed?
To reuse this piece of code in the single-pass Analyzer.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49283 from
vladimirg-db/vladimirg-db/refactor-out-union-compute-output-to-use-in-single-pass-analyzer.
Authored-by: Vladimir Golubev <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../plans/logical/basicLogicalOperators.scala | 30 ++++++++++++----------
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index a6405f18ad44..b897026a0df4 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -445,6 +445,21 @@ object Union {
def apply(left: LogicalPlan, right: LogicalPlan): Union = {
Union (left :: right :: Nil)
}
+
+ // updating nullability to make all the children consistent
+ def mergeChildOutputs(childOutputs: Seq[Seq[Attribute]]): Seq[Attribute] = {
+ childOutputs.transpose.map { attrs =>
+ val firstAttr = attrs.head
+ val nullable = attrs.exists(_.nullable)
+ val newDt = attrs.map(_.dataType).reduce(StructType.unionLikeMerge)
+ if (firstAttr.dataType == newDt) {
+ firstAttr.withNullability(nullable)
+ } else {
+ AttributeReference(firstAttr.name, newDt, nullable,
firstAttr.metadata)(
+ firstAttr.exprId, firstAttr.qualifier)
+ }
+ }
+ }
}
/**
@@ -526,20 +541,7 @@ case class Union(
private lazy val lazyOutput: Seq[Attribute] = computeOutput()
- // updating nullability to make all the children consistent
- private def computeOutput(): Seq[Attribute] = {
- children.map(_.output).transpose.map { attrs =>
- val firstAttr = attrs.head
- val nullable = attrs.exists(_.nullable)
- val newDt = attrs.map(_.dataType).reduce(StructType.unionLikeMerge)
- if (firstAttr.dataType == newDt) {
- firstAttr.withNullability(nullable)
- } else {
- AttributeReference(firstAttr.name, newDt, nullable,
firstAttr.metadata)(
- firstAttr.exprId, firstAttr.qualifier)
- }
- }
- }
+ private def computeOutput(): Seq[Attribute] =
Union.mergeChildOutputs(children.map(_.output))
/**
* Maps the constraints containing a given (original) sequence of attributes
to those with a
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
