dongjoon-hyun commented on a change in pull request #27803: [SPARK-31049][SQL]
Support nested adjacent generators, e.g., explode(explode(v))
URL: https://github.com/apache/spark/pull/27803#discussion_r390677763
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
##########
@@ -2206,17 +2220,21 @@ class Analyzer(
* @return (the [[Generator]], seq of output names, outer flag)
*/
def unapply(e: Expression): Option[(Generator, Seq[String], Boolean)] =
e match {
- case Alias(GeneratorOuter(g: Generator), name) if g.resolved =>
Some((g, name :: Nil, true))
- case MultiAlias(GeneratorOuter(g: Generator), names) if g.resolved =>
Some((g, names, true))
- case Alias(g: Generator, name) if g.resolved => Some((g, name :: Nil,
false))
- case MultiAlias(g: Generator, names) if g.resolved => Some((g, names,
false))
+ case Alias(GeneratorOuter(g: Generator), name) => Some((g, name ::
Nil, true))
+ case MultiAlias(GeneratorOuter(g: Generator), names) => Some((g,
names, true))
+ case Alias(g: Generator, name) => Some((g, name :: Nil, false))
+ case MultiAlias(g: Generator, names) => Some((g, names, false))
case _ => None
}
}
def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
- case Project(projectList, _) if projectList.exists(hasNestedGenerator) =>
- val nestedGenerator = projectList.find(hasNestedGenerator).get
+ // We need to resolve all the functions that might be replaced with
generators
+ // before validating a plan in this rule.
+ case p if
p.expressions.exists(_.find(_.isInstanceOf[UnresolvedFunction]).isDefined) => p
+
+ case Project(projectList, _) if
projectList.exists(hasUnsupportedNestedGenerator) =>
+ val nestedGenerator =
projectList.find(hasUnsupportedNestedGenerator).get
throw new AnalysisException("Generators are not supported when it's
nested in " +
Review comment:
Do we need to revise this message, `Generators are not supported when it's
nested`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
