maropu commented on a change in pull request #28490:
URL: https://github.com/apache/spark/pull/28490#discussion_r436354138
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
##########
@@ -1259,6 +1259,11 @@ class Analyzer(
attr.withExprId(exprId)
}
+ private def dedupStructField(attr: Alias, structFieldMap: Map[String,
Attribute]) = {
Review comment:
Not used now?
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
##########
@@ -1481,7 +1486,35 @@ class Analyzer(
case q: LogicalPlan =>
logTrace(s"Attempting to resolve
${q.simpleString(SQLConf.get.maxToStringFields)}")
Review comment:
w/ some code cleanup;
```
case agg @ (_: Aggregate | _: GroupingSets) =>
val resolved = agg.mapExpressions(resolveExpressionTopDown(_, agg))
val hasStructField = resolved.expressions.exists {
_.collectFirst { case gsf: GetStructField => gsf }.isDefined
}
if (hasStructField) {
// Plz describe some comments why we need this handling...
val structFieldMap = mutable.Map[String, Alias]()
resolved.transformExpressionsDown {
case a @ Alias(struct: GetStructField, _) =>
if (structFieldMap.contains(struct.sql)) {
val exprId = structFieldMap.getOrElse(struct.sql, a).exprId
Alias(a.child, a.name)(exprId, a.qualifier,
a.explicitMetadata)
} else {
structFieldMap += (struct.sql -> a)
a
}
}
} else {
resolved
}
```
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
##########
@@ -1481,7 +1486,35 @@ class Analyzer(
case q: LogicalPlan =>
logTrace(s"Attempting to resolve
${q.simpleString(SQLConf.get.maxToStringFields)}")
Review comment:
Could you write this handling in an independent patten like this?
```
case agg @ (_: Aggregate | _: GroupingSets) =>
val resolved = agg.mapExpressions(resolveExpressionTopDown(_, agg))
val structFieldMap = mutable.Map[String, Alias]()
resolved.transformExpressionsDown {
// Plz describe some comments why we need this handling...
case a @ Alias(struct: GetStructField, _) =>
if (structFieldMap.contains(struct.sql)) {
val exprId = structFieldMap.getOrElse(struct.sql, a).exprId
Alias(a.child, a.name)(exprId, a.qualifier, a.explicitMetadata)
} else {
structFieldMap += (struct.sql -> a)
a
}
case e => e
}
case q: LogicalPlan =>
logTrace(s"Attempting to resolve
${q.simpleString(SQLConf.get.maxToStringFields)}")
q.mapExpressions(resolveExpressionTopDown(_, q))
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
