This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 163a6e2 [SPARK-27514] Skip collapsing windows with empty window
expressions
163a6e2 is described below
commit 163a6e298213f216f74f4764e241ee6298ea30b6
Author: Yifei Huang <[email protected]>
AuthorDate: Fri Apr 19 14:04:44 2019 +0800
[SPARK-27514] Skip collapsing windows with empty window expressions
## What changes were proposed in this pull request?
A previous change moved the removal of empty window expressions to the
RemoveNoopOperations rule, which comes after the CollapseWindow rule.
Therefore, by the time we get to CollapseWindow, we aren't guaranteed that
empty windows have been removed. This change checks that the window expressions
are not empty, and only collapses the windows if both windows are non-empty.
A lengthier description and repro steps here:
https://issues.apache.org/jira/browse/SPARK-27514
## How was this patch tested?
A unit test, plus I reran the breaking case mentioned in the Jira ticket.
Closes #24411 from yifeih/yh/spark-27514.
Authored-by: Yifei Huang <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 1 +
.../spark/sql/catalyst/optimizer/CollapseWindowSuite.scala | 11 +++++++++++
2 files changed, 12 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index afdf61e..f32f2c7 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -770,6 +770,7 @@ object CollapseWindow extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
case w1 @ Window(we1, ps1, os1, w2 @ Window(we2, ps2, os2, grandChild))
if ps1 == ps2 && os1 == os2 &&
w1.references.intersect(w2.windowOutputSet).isEmpty &&
+ we1.nonEmpty && we2.nonEmpty &&
// This assumes Window contains the same type of window expressions.
This is ensured
// by ExtractWindowFunctions.
WindowFunctionType.functionType(we1.head) ==
WindowFunctionType.functionType(we2.head) =>
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala
index 52054c2..3b3b490 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CollapseWindowSuite.scala
@@ -89,4 +89,15 @@ class CollapseWindowSuite extends PlanTest {
val optimized = Optimize.execute(query.analyze)
comparePlans(optimized, expected)
}
+
+ test("Skip windows with empty window expressions") {
+ val query = testRelation
+ .window(Seq(), partitionSpec1, orderSpec1)
+ .window(Seq(sum(a).as('sum_a)), partitionSpec1, orderSpec1)
+
+ val optimized = Optimize.execute(query.analyze)
+ val correctAnswer = query.analyze
+
+ comparePlans(optimized, correctAnswer)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
