Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/23211#discussion_r240097255
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
---
@@ -649,13 +664,16 @@ object CollapseProject extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
case p1 @ Project(_, p2: Project) =>
- if (haveCommonNonDeterministicOutput(p1.projectList,
p2.projectList)) {
+ if (haveCommonNonDeterministicOutput(p1.projectList, p2.projectList)
||
+ ScalarSubquery.hasScalarSubquery(p1.projectList) ||
+ ScalarSubquery.hasScalarSubquery(p2.projectList)) {
--- End diff --
why do we allow it before?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
