[
https://issues.apache.org/jira/browse/SPARK-15021?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15265188#comment-15265188
]
Wenchen Fan commented on SPARK-15021:
-------------------------------------
I checked with hive, hive doesn't allow nested UDTF, so I don't think it's a
bug.
Do we want this feature? I'd say it's not a easy job, UDTF returns rows having
one or more fields, I'm not sure how to define the semantic of aggregation on
UDTF.
> cannot run aggregate function on explode result
> -----------------------------------------------
>
> Key: SPARK-15021
> URL: https://issues.apache.org/jira/browse/SPARK-15021
> Project: Spark
> Issue Type: New Feature
> Components: SQL
> Affects Versions: 1.6.1
> Reporter: Reynold Xin
>
> See the following repro
> {code}
> scala> spark.range(1000).map(i =>
> Array[Long](i)).selectExpr("max(explode(value))").collect()
> java.lang.UnsupportedOperationException
> at
> org.apache.spark.sql.catalyst.expressions.Generator$class.dataType(generators.scala:46)
> at
> org.apache.spark.sql.catalyst.expressions.Explode.dataType(generators.scala:106)
> at
> org.apache.spark.sql.catalyst.expressions.aggregate.Max.checkInputDataTypes(Max.scala:40)
> at
> org.apache.spark.sql.catalyst.expressions.Expression.resolved$lzycompute(Expression.scala:133)
> at
> org.apache.spark.sql.catalyst.expressions.Expression.resolved(Expression.scala:133)
> at
> org.apache.spark.sql.catalyst.expressions.Expression$$anonfun$childrenResolved$1.apply(Expression.scala:145)
> at
> org.apache.spark.sql.catalyst.expressions.Expression$$anonfun$childrenResolved$1.apply(Expression.scala:145)
> at
> scala.collection.LinearSeqOptimized$class.forall(LinearSeqOptimized.scala:83)
> at scala.collection.immutable.List.forall(List.scala:84)
> at
> org.apache.spark.sql.catalyst.expressions.Expression.childrenResolved(Expression.scala:145)
> at
> org.apache.spark.sql.catalyst.expressions.Expression.resolved$lzycompute(Expression.scala:133)
> at
> org.apache.spark.sql.catalyst.expressions.Expression.resolved(Expression.scala:133)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$$anonfun$org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveAliases$$assignAliases$1$$anonfun$apply$3.applyOrElse(Analyzer.scala:178)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$$anonfun$org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveAliases$$assignAliases$1$$anonfun$apply$3.applyOrElse(Analyzer.scala:175)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:287)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:287)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:68)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:286)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$$anonfun$org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveAliases$$assignAliases$1.apply(Analyzer.scala:175)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$$anonfun$org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveAliases$$assignAliases$1.apply(Analyzer.scala:173)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> at
> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.AbstractTraversable.map(Traversable.scala:104)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$.org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveAliases$$assignAliases(Analyzer.scala:173)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$$anonfun$apply$4.applyOrElse(Analyzer.scala:203)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$$anonfun$apply$4.applyOrElse(Analyzer.scala:191)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$resolveOperators$1.apply(LogicalPlan.scala:61)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan$$anonfun$resolveOperators$1.apply(LogicalPlan.scala:61)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:68)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperators(LogicalPlan.scala:60)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$.apply(Analyzer.scala:191)
> at
> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveAliases$.apply(Analyzer.scala:171)
> at
> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85)
> at
> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82)
> at
> scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124)
> at scala.collection.immutable.List.foldLeft(List.scala:84)
> at
> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:82)
> at
> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:74)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at
> org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:74)
> at
> org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:64)
> at
> org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:62)
> at
> org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:48)
> at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:60)
> at
> org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2447)
> at org.apache.spark.sql.Dataset.select(Dataset.scala:940)
> at org.apache.spark.sql.Dataset.selectExpr(Dataset.scala:975)
> ... 48 elided
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
