spark git commit: [SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables
Repository: spark Updated Branches: refs/heads/branch-1.5 c99f4160b -> 2239a2036 [SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables In `HiveComparisionTest`s it is possible to fail a query of the form `SELECT * FROM dest1`, where `dest1` is the query that is actually computing the incorrect results. To aid debugging this patch improves the harness to also print these query plans and their results. Author: Michael Armbrust Closes #8388 from marmbrus/generatedTables. (cherry picked from commit 5175ca0c85b10045d12c3fb57b1e52278a413ecf) Signed-off-by: Reynold Xin Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2239a203 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2239a203 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2239a203 Branch: refs/heads/branch-1.5 Commit: 2239a20368b7833ffe0059941478924c7be87bbe Parents: c99f416 Author: Michael Armbrust Authored: Mon Aug 24 23:15:27 2015 -0700 Committer: Reynold Xin Committed: Mon Aug 24 23:15:34 2015 -0700 -- .../sql/hive/execution/HiveComparisonTest.scala | 36 1 file changed, 36 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2239a203/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index 2bdb0e1..4d45249 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -19,6 +19,8 @@ package org.apache.spark.sql.hive.execution import java.io._ +import scala.util.control.NonFatal + import org.scalatest.{BeforeAndAfterAll, GivenWhenThen} import org.apache.spark.{Logging, SparkFunSuite} @@ -386,11 +388,45 @@ abstract class HiveComparisonTest hiveCacheFiles.foreach(_.delete()) } + // If this query is reading other tables that were created during this test run + // also print out the query plans and results for those. + val computedTablesMessages: String = try { +val tablesRead = new TestHive.QueryExecution(query).executedPlan.collect { + case ts: HiveTableScan => ts.relation.tableName +}.toSet + +TestHive.reset() +val executions = queryList.map(new TestHive.QueryExecution(_)) +executions.foreach(_.toRdd) +val tablesGenerated = queryList.zip(executions).flatMap { + case (q, e) => e.executedPlan.collect { +case i: InsertIntoHiveTable if tablesRead contains i.table.tableName => + (q, e, i) + } +} + +tablesGenerated.map { case (hiveql, execution, insert) => + s""" + |=== Generated Table === + |$hiveql + |$execution + |== Results == + |${insert.child.execute().collect().mkString("\n")} + """.stripMargin +}.mkString("\n") + + } catch { +case NonFatal(e) => + logError("Failed to compute generated tables", e) + s"Couldn't compute dependent tables: $e" + } + val errorMessage = s""" |Results do not match for $testCaseName: |$hiveQuery\n${hiveQuery.analyzed.output.map(_.name).mkString("\t")} |$resultComparison + |$computedTablesMessages """.stripMargin stringToFile(new File(wrongDirectory, testCaseName), errorMessage + consoleTestCase) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables
Repository: spark Updated Branches: refs/heads/master a0c0aae1d -> 5175ca0c8 [SPARK-10178] [SQL] HiveComparisionTest should print out dependent tables In `HiveComparisionTest`s it is possible to fail a query of the form `SELECT * FROM dest1`, where `dest1` is the query that is actually computing the incorrect results. To aid debugging this patch improves the harness to also print these query plans and their results. Author: Michael Armbrust Closes #8388 from marmbrus/generatedTables. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5175ca0c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5175ca0c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5175ca0c Branch: refs/heads/master Commit: 5175ca0c85b10045d12c3fb57b1e52278a413ecf Parents: a0c0aae Author: Michael Armbrust Authored: Mon Aug 24 23:15:27 2015 -0700 Committer: Reynold Xin Committed: Mon Aug 24 23:15:27 2015 -0700 -- .../sql/hive/execution/HiveComparisonTest.scala | 36 1 file changed, 36 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5175ca0c/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index 2bdb0e1..4d45249 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -19,6 +19,8 @@ package org.apache.spark.sql.hive.execution import java.io._ +import scala.util.control.NonFatal + import org.scalatest.{BeforeAndAfterAll, GivenWhenThen} import org.apache.spark.{Logging, SparkFunSuite} @@ -386,11 +388,45 @@ abstract class HiveComparisonTest hiveCacheFiles.foreach(_.delete()) } + // If this query is reading other tables that were created during this test run + // also print out the query plans and results for those. + val computedTablesMessages: String = try { +val tablesRead = new TestHive.QueryExecution(query).executedPlan.collect { + case ts: HiveTableScan => ts.relation.tableName +}.toSet + +TestHive.reset() +val executions = queryList.map(new TestHive.QueryExecution(_)) +executions.foreach(_.toRdd) +val tablesGenerated = queryList.zip(executions).flatMap { + case (q, e) => e.executedPlan.collect { +case i: InsertIntoHiveTable if tablesRead contains i.table.tableName => + (q, e, i) + } +} + +tablesGenerated.map { case (hiveql, execution, insert) => + s""" + |=== Generated Table === + |$hiveql + |$execution + |== Results == + |${insert.child.execute().collect().mkString("\n")} + """.stripMargin +}.mkString("\n") + + } catch { +case NonFatal(e) => + logError("Failed to compute generated tables", e) + s"Couldn't compute dependent tables: $e" + } + val errorMessage = s""" |Results do not match for $testCaseName: |$hiveQuery\n${hiveQuery.analyzed.output.map(_.name).mkString("\t")} |$resultComparison + |$computedTablesMessages """.stripMargin stringToFile(new File(wrongDirectory, testCaseName), errorMessage + consoleTestCase) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org