AngersZhuuuu commented on a change in pull request #30212:
URL: https://github.com/apache/spark/pull/30212#discussion_r533438082
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
##########
@@ -2027,65 +1952,76 @@ class Analyzer(override val catalogManager:
CatalogManager)
*/
object ResolveFunctions extends Rule[LogicalPlan] {
val trimWarningEnabled = new AtomicBoolean(true)
+
+ def resolveFunction(): PartialFunction[Expression, Expression] = {
+ case u if !u.childrenResolved => u // Skip until children are resolved.
+ case u: UnresolvedAttribute if resolver(u.name,
VirtualColumn.hiveGroupingIdName) =>
+ withPosition(u) {
+ Alias(GroupingID(Nil), VirtualColumn.hiveGroupingIdName)()
+ }
+ case u @ UnresolvedGenerator(name, children) =>
+ withPosition(u) {
+ v1SessionCatalog.lookupFunction(name, children) match {
+ case generator: Generator => generator
+ case other =>
+ failAnalysis(s"$name is expected to be a generator. However, " +
+ s"its class is ${other.getClass.getCanonicalName}, which is
not a generator.")
+ }
+ }
+ case u @ UnresolvedFunction(funcId, arguments, isDistinct, filter) =>
+ withPosition(u) {
+ v1SessionCatalog.lookupFunction(funcId, arguments) match {
+ // AggregateWindowFunctions are AggregateFunctions that can only
be evaluated within
+ // the context of a Window clause. They do not need to be wrapped
in an
+ // AggregateExpression.
+ case wf: AggregateWindowFunction =>
+ if (isDistinct || filter.isDefined) {
+ failAnalysis("DISTINCT or FILTER specified, " +
+ s"but ${wf.prettyName} is not an aggregate function")
+ } else {
+ wf
+ }
+ // We get an aggregate function, we need to wrap it in an
AggregateExpression.
+ case agg: AggregateFunction =>
+ if (filter.isDefined && !filter.get.deterministic) {
+ failAnalysis("FILTER expression is non-deterministic, " +
+ "it cannot be used in aggregate functions")
+ }
+ AggregateExpression(agg, Complete, isDistinct, filter)
+ // This function is not an aggregate function, just return the
resolved one.
+ case other if (isDistinct || filter.isDefined) =>
+ failAnalysis("DISTINCT or FILTER specified, " +
+ s"but ${other.prettyName} is not an aggregate function")
+ case e: String2TrimExpression if arguments.size == 2 =>
+ if (trimWarningEnabled.get) {
+ logWarning("Two-parameter TRIM/LTRIM/RTRIM function signatures
are deprecated." +
+ " Use SQL syntax `TRIM((BOTH | LEADING | TRAILING)? trimStr
FROM str)`" +
+ " instead.")
+ trimWarningEnabled.set(false)
+ }
+ e
+ case other =>
+ other
+ }
+ }
+ }
+
def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
// Resolve functions with concrete relations from v2 catalog.
case UnresolvedFunc(multipartIdent) =>
val funcIdent = parseSessionCatalogFunctionIdentifier(multipartIdent)
ResolvedFunc(Identifier.of(funcIdent.database.toArray,
funcIdent.funcName))
- case q: LogicalPlan =>
- q transformExpressions {
- case u if !u.childrenResolved => u // Skip until children are
resolved.
- case u: UnresolvedAttribute if resolver(u.name,
VirtualColumn.hiveGroupingIdName) =>
- withPosition(u) {
- Alias(GroupingID(Nil), VirtualColumn.hiveGroupingIdName)()
- }
- case u @ UnresolvedGenerator(name, children) =>
- withPosition(u) {
- v1SessionCatalog.lookupFunction(name, children) match {
- case generator: Generator => generator
- case other =>
- failAnalysis(s"$name is expected to be a generator. However,
" +
- s"its class is ${other.getClass.getCanonicalName}, which
is not a generator.")
- }
- }
- case u @ UnresolvedFunction(funcId, arguments, isDistinct, filter) =>
- withPosition(u) {
- v1SessionCatalog.lookupFunction(funcId, arguments) match {
- // AggregateWindowFunctions are AggregateFunctions that can
only be evaluated within
- // the context of a Window clause. They do not need to be
wrapped in an
- // AggregateExpression.
- case wf: AggregateWindowFunction =>
- if (isDistinct || filter.isDefined) {
- failAnalysis("DISTINCT or FILTER specified, " +
- s"but ${wf.prettyName} is not an aggregate function")
- } else {
- wf
- }
- // We get an aggregate function, we need to wrap it in an
AggregateExpression.
- case agg: AggregateFunction =>
- if (filter.isDefined && !filter.get.deterministic) {
- failAnalysis("FILTER expression is non-deterministic, " +
- "it cannot be used in aggregate functions")
- }
- AggregateExpression(agg, Complete, isDistinct, filter)
- // This function is not an aggregate function, just return the
resolved one.
- case other if (isDistinct || filter.isDefined) =>
- failAnalysis("DISTINCT or FILTER specified, " +
- s"but ${other.prettyName} is not an aggregate function")
- case e: String2TrimExpression if arguments.size == 2 =>
- if (trimWarningEnabled.get) {
- log.warn("Two-parameter TRIM/LTRIM/RTRIM function
signatures are deprecated." +
- " Use SQL syntax `TRIM((BOTH | LEADING | TRAILING)?
trimStr FROM str)`" +
- " instead.")
- trimWarningEnabled.set(false)
- }
- e
- case other =>
- other
- }
- }
+ case a: Aggregate =>
+ val newGroups = a.groupingExpressions.map {
+ case gs: GroupingSet =>
+
gs.withNewChildren(gs.children.map(_.transformDown(resolveFunction)))
+ case e => e
}
+ a.copy(groupingExpressions = newGroups) transformExpressions
resolveFunction
Review comment:
> Actually, I'm not sure now about why we need to update
`ResolveFunctions`. If `GroupingSet`(`Rollup`, `Cube`, and `GroupingSet`) has a
correct structure like the other expressions, I don't think we need to update
this rule at all. If it is unavoidable to update it, I think we need a
reasonable reason to do so.
emmm, without this change some test failed, later I will debug to find the
real reason.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
