zml1206 commented on code in PR #49202:
URL: https://github.com/apache/spark/pull/49202#discussion_r1904995131
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala:
##########
@@ -38,8 +38,34 @@ import org.apache.spark.util.Utils
* Note: For now we only use `With` in a few `RuntimeReplaceable` expressions.
If we expand its
* usage, we should support aggregate/window functions as well.
*/
-object RewriteWithExpression extends Rule[LogicalPlan] {
+object RewriteWithExpression extends Rule[LogicalPlan] with AliasHelper {
override def apply(plan: LogicalPlan): LogicalPlan = {
+ if (!plan.containsPattern(WITH_EXPRESSION)) {
+ return plan
+ }
+ var p = plan
+ while (p.containsPattern(WITH_EXPRESSION)) {
+ p = CollapseProject(applyOnce(p))
+ }
+ rewriteAlias(p)
+ }
+
+ // Expensive common expression will be evaluated twice: once in the Filter
being pushed down,
+ // once in the Project stays up. Project reuses the attribute to avoid
secondary evaluates.
+ private def rewriteAlias(plan: LogicalPlan): LogicalPlan = {
+ plan.transformUpWithSubqueriesAndPruning(_.containsAllPatterns(PROJECT,
FILTER)) {
+ case p @ Project(projectList, child: Filter)
Review Comment:
The aggregateExpressions of aggregate are not rewritten here, because the
groupingExpressions will be changed, which may cause regression.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
