holdenk commented on code in PR #46741:
URL: https://github.com/apache/spark/pull/46741#discussion_r1633922700
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala:
##########
@@ -177,4 +120,83 @@ object RewriteWithExpression extends Rule[LogicalPlan] {
case other => other.mapChildren(rewriteWithExprAndInputPlans(_,
inputPlans))
}
}
+
+ private def genProject(
+ defs: Seq[Expression],
+ inputPlans: Array[LogicalPlan]): mutable.Map[CommonExpressionId,
Expression] = {
+ val refToExpr = mutable.HashMap.empty[CommonExpressionId, Expression]
+ val childProjections =
Array.fill(inputPlans.length)(mutable.ArrayBuffer.empty[Alias])
+ defs.zipWithIndex.foreach { case (CommonExpressionDef(child, id), index) =>
+ if (id.canonicalized) {
+ throw SparkException.internalError(
+ "Cannot rewrite canonicalized Common expression definitions")
+ }
+
+ if (CollapseProject.isCheap(child)) {
+ refToExpr(id) = child
+ } else {
Review Comment:
This seems potentially less than ideal, it looks like
CollapseProject.isCheap will return true for Python UDFS. It also _might_
violate the promise of being gauranteed to be evaluated only once (see
With.scala L26).
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala:
##########
@@ -177,4 +120,83 @@ object RewriteWithExpression extends Rule[LogicalPlan] {
case other => other.mapChildren(rewriteWithExprAndInputPlans(_,
inputPlans))
}
}
+
+ private def genProject(
+ defs: Seq[Expression],
+ inputPlans: Array[LogicalPlan]): mutable.Map[CommonExpressionId,
Expression] = {
+ val refToExpr = mutable.HashMap.empty[CommonExpressionId, Expression]
+ val childProjections =
Array.fill(inputPlans.length)(mutable.ArrayBuffer.empty[Alias])
+ defs.zipWithIndex.foreach { case (CommonExpressionDef(child, id), index) =>
+ if (id.canonicalized) {
+ throw SparkException.internalError(
+ "Cannot rewrite canonicalized Common expression definitions")
Review Comment:
Can we include the element that caused this error?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
