Repository: spark
Updated Branches:
refs/heads/branch-2.3 0d425c336 -> 5fbbd94d5
[SPARK-23021][SQL] AnalysisBarrier should override innerChildren to print
correct explain output
## What changes were proposed in this pull request?
`AnalysisBarrier` in the current master cuts off explain results for parsed
logical plans;
```
scala> Seq((1, 1)).toDF("a", "b").groupBy("a").count().sample(0.1).explain(true)
== Parsed Logical Plan ==
Sample 0.0, 0.1, false, -7661439431999668039
+- AnalysisBarrier Aggregate [a#5], [a#5, count(1) AS count#14L]
```
To fix this, `AnalysisBarrier` needs to override `innerChildren` and this pr
changed the output to;
```
== Parsed Logical Plan ==
Sample 0.0, 0.1, false, -5086223488015741426
+- AnalysisBarrier
+- Aggregate [a#5], [a#5, count(1) AS count#14L]
+- Project [_1#2 AS a#5, _2#3 AS b#6]
+- LocalRelation [_1#2, _2#3]
```
## How was this patch tested?
Added tests in `DataFrameSuite`.
Author: Takeshi Yamamuro <yamam...@apache.org>
Closes #20247 from maropu/SPARK-23021-2.
(cherry picked from commit 990f05c80347c6eec2ee06823cff587c9ea90b49)
Signed-off-by: gatorsmile <gatorsm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fbbd94d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fbbd94d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fbbd94d
Branch: refs/heads/branch-2.3
Commit: 5fbbd94d509dbbcfa1fe940569049f72ff4a6e89
Parents: 0d425c3
Author: Takeshi Yamamuro <yamam...@apache.org>
Authored: Sun Jan 14 22:26:21 2018 +0800
Committer: gatorsmile <gatorsm...@gmail.com>
Committed: Sun Jan 14 22:27:26 2018 +0800
----------------------------------------------------------------------
.../plans/logical/basicLogicalOperators.scala | 1 +
.../sql/hive/execution/HiveExplainSuite.scala | 17 +++++++++++++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5fbbd94d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 95e099c..a4fca79 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -903,6 +903,7 @@ case class Deduplicate(
* This analysis barrier will be removed at the end of analysis stage.
*/
case class AnalysisBarrier(child: LogicalPlan) extends LeafNode {
+ override protected def innerChildren: Seq[LogicalPlan] = Seq(child)
override def output: Seq[Attribute] = child.output
override def isStreaming: Boolean = child.isStreaming
override def doCanonicalize(): LogicalPlan = child.canonicalized
http://git-wip-us.apache.org/repos/asf/spark/blob/5fbbd94d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
index dfabf1e..a4273de 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
@@ -171,4 +171,21 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils
with TestHiveSingleto
sql("EXPLAIN EXTENDED CODEGEN SELECT 1")
}
}
+
+ test("SPARK-23021 AnalysisBarrier should not cut off explain output for
parsed logical plans") {
+ val df = Seq((1, 1)).toDF("a", "b").groupBy("a").count().limit(1)
+ val outputStream = new java.io.ByteArrayOutputStream()
+ Console.withOut(outputStream) {
+ df.explain(true)
+ }
+ assert(outputStream.toString.replaceAll("""#\d+""", "#0").contains(
+ s"""== Parsed Logical Plan ==
+ |GlobalLimit 1
+ |+- LocalLimit 1
+ | +- AnalysisBarrier
+ | +- Aggregate [a#0], [a#0, count(1) AS count#0L]
+ | +- Project [_1#0 AS a#0, _2#0 AS b#0]
+ | +- LocalRelation [_1#0, _2#0]
+ |""".stripMargin))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
