spark git commit: [SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability
Repository: spark
Updated Branches:
refs/heads/branch-1.6 93095eb29 -> fb08f7b78
[SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability
Author: Wenchen Fan
Closes #8645 from cloud-fan/test.
(cherry picked from commit a89e8b6122ee5a1517fbcf405b1686619db56696)
Signed-off-by: Andrew Or
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fb08f7b7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fb08f7b7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fb08f7b7
Branch: refs/heads/branch-1.6
Commit: fb08f7b784bc8b5e0cd110f315f72c7d9fc65e08
Parents: 93095eb
Author: Wenchen Fan
Authored: Tue Dec 15 18:29:19 2015 -0800
Committer: Andrew Or
Committed: Tue Dec 15 18:29:25 2015 -0800
--
.../apache/spark/sql/catalyst/dsl/package.scala | 7 ++--
.../catalyst/optimizer/ColumnPruningSuite.scala | 41 +++-
2 files changed, 27 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fb08f7b7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index af594c2..e509711 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -275,13 +275,14 @@ package object dsl {
def unionAll(otherPlan: LogicalPlan): LogicalPlan = Union(logicalPlan,
otherPlan)
- // TODO specify the output column names
def generate(
generator: Generator,
join: Boolean = false,
outer: Boolean = false,
-alias: Option[String] = None): LogicalPlan =
-Generate(generator, join = join, outer = outer, alias, Nil,
logicalPlan)
+alias: Option[String] = None,
+outputNames: Seq[String] = Nil): LogicalPlan =
+Generate(generator, join = join, outer = outer, alias,
+ outputNames.map(UnresolvedAttribute(_)), logicalPlan)
def insertInto(tableName: String, overwrite: Boolean = false):
LogicalPlan =
InsertIntoTable(
http://git-wip-us.apache.org/repos/asf/spark/blob/fb08f7b7/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
index 4a1e7ce..9bf61ae 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.expressions.Explode
import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.plans.logical.{Project, LocalRelation,
Generate, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
@@ -35,12 +35,11 @@ class ColumnPruningSuite extends PlanTest {
test("Column pruning for Generate when Generate.join = false") {
val input = LocalRelation('a.int, 'b.array(StringType))
-val query = Generate(Explode('b), false, false, None, 's.string :: Nil,
input).analyze
+val query = input.generate(Explode('b), join = false).analyze
+
val optimized = Optimize.execute(query)
-val correctAnswer =
- Generate(Explode('b), false, false, None, 's.string :: Nil,
-Project('b.attr :: Nil, input)).analyze
+val correctAnswer = input.select('b).generate(Explode('b), join =
false).analyze
comparePlans(optimized, correctAnswer)
}
@@ -49,16 +48,19 @@ class ColumnPruningSuite extends PlanTest {
val input = LocalRelation('a.int, 'b.int, 'c.array(StringType))
val query =
- Project(Seq('a, 's),
-Generate(Explode('c), true, false, None, 's.string :: Nil,
- input)).analyze
+ input
+.generate(Explode('c), join = true, outputNames = "explode" :: Nil)
+.select('a, 'explode)
+.analyze
+
val optimized = Optimize.execute(query)
val correctAnswer =
- Project(Seq('a, 's),
-Generate(Explode('c), true, false, None, 's.string :: Nil,
- Project(Seq('a, 'c),
-input))).analyze
+ input
+
spark git commit: [SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability
Repository: spark
Updated Branches:
refs/heads/master c5b6b398d -> a89e8b612
[SPARK-10477][SQL] using DSL in ColumnPruningSuite to improve readability
Author: Wenchen Fan
Closes #8645 from cloud-fan/test.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a89e8b61
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a89e8b61
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a89e8b61
Branch: refs/heads/master
Commit: a89e8b6122ee5a1517fbcf405b1686619db56696
Parents: c5b6b39
Author: Wenchen Fan
Authored: Tue Dec 15 18:29:19 2015 -0800
Committer: Andrew Or
Committed: Tue Dec 15 18:29:19 2015 -0800
--
.../apache/spark/sql/catalyst/dsl/package.scala | 7 ++--
.../catalyst/optimizer/ColumnPruningSuite.scala | 41 +++-
2 files changed, 27 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a89e8b61/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index af594c2..e509711 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -275,13 +275,14 @@ package object dsl {
def unionAll(otherPlan: LogicalPlan): LogicalPlan = Union(logicalPlan,
otherPlan)
- // TODO specify the output column names
def generate(
generator: Generator,
join: Boolean = false,
outer: Boolean = false,
-alias: Option[String] = None): LogicalPlan =
-Generate(generator, join = join, outer = outer, alias, Nil,
logicalPlan)
+alias: Option[String] = None,
+outputNames: Seq[String] = Nil): LogicalPlan =
+Generate(generator, join = join, outer = outer, alias,
+ outputNames.map(UnresolvedAttribute(_)), logicalPlan)
def insertInto(tableName: String, overwrite: Boolean = false):
LogicalPlan =
InsertIntoTable(
http://git-wip-us.apache.org/repos/asf/spark/blob/a89e8b61/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
--
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
index 4a1e7ce..9bf61ae 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.expressions.Explode
import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.plans.logical.{Project, LocalRelation,
Generate, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
@@ -35,12 +35,11 @@ class ColumnPruningSuite extends PlanTest {
test("Column pruning for Generate when Generate.join = false") {
val input = LocalRelation('a.int, 'b.array(StringType))
-val query = Generate(Explode('b), false, false, None, 's.string :: Nil,
input).analyze
+val query = input.generate(Explode('b), join = false).analyze
+
val optimized = Optimize.execute(query)
-val correctAnswer =
- Generate(Explode('b), false, false, None, 's.string :: Nil,
-Project('b.attr :: Nil, input)).analyze
+val correctAnswer = input.select('b).generate(Explode('b), join =
false).analyze
comparePlans(optimized, correctAnswer)
}
@@ -49,16 +48,19 @@ class ColumnPruningSuite extends PlanTest {
val input = LocalRelation('a.int, 'b.int, 'c.array(StringType))
val query =
- Project(Seq('a, 's),
-Generate(Explode('c), true, false, None, 's.string :: Nil,
- input)).analyze
+ input
+.generate(Explode('c), join = true, outputNames = "explode" :: Nil)
+.select('a, 'explode)
+.analyze
+
val optimized = Optimize.execute(query)
val correctAnswer =
- Project(Seq('a, 's),
-Generate(Explode('c), true, false, None, 's.string :: Nil,
- Project(Seq('a, 'c),
-input))).analyze
+ input
+.select('a, 'c)
+.generate(Explode('c), join = true, outputNames = "explode" :: Nil)
+
