cloud-fan commented on a change in pull request #31666:
URL: https://github.com/apache/spark/pull/31666#discussion_r611670279
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
##########
@@ -3158,26 +3179,29 @@ class Analyzer(override val catalogManager:
CatalogManager)
val rUniqueOutput = right.output.filterNot(att => rightKeys.contains(att))
// the output list looks like: join keys, columns from left, columns from
right
- val projectList = joinType match {
+ val (projectList, hiddenList) = joinType match {
case LeftOuter =>
- leftKeys ++ lUniqueOutput ++ rUniqueOutput.map(_.withNullability(true))
+ (leftKeys ++ lUniqueOutput ++
rUniqueOutput.map(_.withNullability(true)), rightKeys)
case LeftExistence(_) =>
- leftKeys ++ lUniqueOutput
+ (leftKeys ++ lUniqueOutput, Seq.empty)
case RightOuter =>
- rightKeys ++ lUniqueOutput.map(_.withNullability(true)) ++
rUniqueOutput
+ (rightKeys ++ lUniqueOutput.map(_.withNullability(true)) ++
rUniqueOutput, leftKeys)
case FullOuter =>
// in full outer join, joinCols should be non-null if there is.
val joinedCols = joinPairs.map { case (l, r) => Alias(Coalesce(Seq(l,
r)), l.name)() }
- joinedCols ++
+ (joinedCols ++
lUniqueOutput.map(_.withNullability(true)) ++
- rUniqueOutput.map(_.withNullability(true))
+ rUniqueOutput.map(_.withNullability(true)),
+ leftKeys ++ rightKeys)
case _ : InnerLike =>
- leftKeys ++ lUniqueOutput ++ rUniqueOutput
+ (leftKeys ++ lUniqueOutput ++ rUniqueOutput, rightKeys)
case _ =>
sys.error("Unsupported natural join type " + joinType)
}
- // use Project to trim unnecessary fields
- Project(projectList, Join(left, right, joinType, newCondition, hint))
+ // use Project to hide duplicated common keys
+ val project = Project(projectList, Join(left, right, joinType,
newCondition, hint))
+ project.setTagValue(Project.hiddenOutputTag,
hiddenList.map(_.asHiddenCol()))
Review comment:
I feel it's too hacky to change the hidden column propagation logic of
`Project`. How about we do a manual propagation here?
`hiddenList.map(_.asHiddenCol()) ++ project.child.metadataOutput.filter(is
hidden column)`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
