Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/21993#discussion_r208822706
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -631,19 +631,26 @@ object ColumnPruning extends Rule[LogicalPlan] {
object CollapseProject extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
- case p1 @ Project(_, p2: Project) =>
- if (haveCommonNonDeterministicOutput(p1.projectList,
p2.projectList)) {
- p1
- } else {
- p2.copy(projectList = buildCleanedProjectList(p1.projectList,
p2.projectList))
- }
+ case p1@Project(_, p2: Project) =>
--- End diff --
nit: Do we need to change this line? We can keep this line as is.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
