spark git commit: [SPARK-15706][SQL] Fix Wrong Answer when using IF NOT EXISTS in INSERT OVERWRITE for DYNAMIC PARTITION
Repository: spark
Updated Branches:
refs/heads/master 5ada60614 -> e5d703bca
[SPARK-15706][SQL] Fix Wrong Answer when using IF NOT EXISTS in INSERT
OVERWRITE for DYNAMIC PARTITION
What changes were proposed in this pull request?
`IF NOT EXISTS` in `INSERT OVERWRITE` should not support dynamic partitions. If
we specify `IF NOT EXISTS`, the inserted statement is not shown in the table.
This PR is to issue an exception in this case, just like what Hive does. Also
issue an exception if users specify `IF NOT EXISTS` if users do not specify any
`PARTITION` specification.
How was this patch tested?
Added test cases into `PlanParserSuite` and `InsertIntoHiveTableSuite`
Author: gatorsmile
Closes #13447 from gatorsmile/insertIfNotExist.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e5d703bc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e5d703bc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e5d703bc
Branch: refs/heads/master
Commit: e5d703bca85c65ce329b1e202283cfa35d109146
Parents: 5ada606
Author: gatorsmile
Authored: Thu Jun 16 22:54:02 2016 -0700
Committer: Yin Huai
Committed: Thu Jun 16 22:54:02 2016 -0700
--
.../apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +-
.../spark/sql/catalyst/parser/AstBuilder.scala | 6 ++
.../plans/logical/basicLogicalOperators.scala | 1 +
.../sql/catalyst/parser/PlanParserSuite.scala | 13 ++--
.../sql/hive/InsertIntoHiveTableSuite.scala | 68
5 files changed, 85 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e5d703bc/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
--
diff --git
a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index b603196..23e925e 100644
---
a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++
b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -203,7 +203,7 @@ query
;
insertInto
-: INSERT OVERWRITE TABLE tableIdentifier partitionSpec? (IF NOT EXISTS)?
+: INSERT OVERWRITE TABLE tableIdentifier (partitionSpec (IF NOT EXISTS)?)?
| INSERT INTO TABLE? tableIdentifier partitionSpec?
;
http://git-wip-us.apache.org/repos/asf/spark/blob/e5d703bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index e380643..c7420a1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -171,6 +171,12 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
Logging {
val tableIdent = visitTableIdentifier(ctx.tableIdentifier)
val partitionKeys =
Option(ctx.partitionSpec).map(visitPartitionSpec).getOrElse(Map.empty)
+val dynamicPartitionKeys = partitionKeys.filter(_._2.isEmpty)
+if (ctx.EXISTS != null && dynamicPartitionKeys.nonEmpty) {
+ throw new ParseException(s"Dynamic partitions do not support IF NOT
EXISTS. Specified " +
+"partitions with value: " + dynamicPartitionKeys.keys.mkString("[",
",", "]"), ctx)
+}
+
InsertIntoTable(
UnresolvedRelation(tableIdent, None),
partitionKeys,
http://git-wip-us.apache.org/repos/asf/spark/blob/e5d703bc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 898784d..6c3eb3a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -377,6 +377,7 @@ case class InsertIntoTable(
}
assert(overwrite || !ifNotExists)
+ assert(partition.values.forall(_.nonEmpty) || !ifNotExists)
override lazy val resolved: Boolean =
childrenResolved && table.resolved && expectedColumns.forall { expected =>
child.output.size == expected.size && child.output.zip(expected).forall {
http://git-wip-us.apache.org/repos/asf/spark/blob/e5d703bc/sql/catalyst/src/test/scala/or
spark git commit: [SPARK-15706][SQL] Fix Wrong Answer when using IF NOT EXISTS in INSERT OVERWRITE for DYNAMIC PARTITION
Repository: spark
Updated Branches:
refs/heads/branch-2.0 3994372f4 -> b82abde06
[SPARK-15706][SQL] Fix Wrong Answer when using IF NOT EXISTS in INSERT
OVERWRITE for DYNAMIC PARTITION
What changes were proposed in this pull request?
`IF NOT EXISTS` in `INSERT OVERWRITE` should not support dynamic partitions. If
we specify `IF NOT EXISTS`, the inserted statement is not shown in the table.
This PR is to issue an exception in this case, just like what Hive does. Also
issue an exception if users specify `IF NOT EXISTS` if users do not specify any
`PARTITION` specification.
How was this patch tested?
Added test cases into `PlanParserSuite` and `InsertIntoHiveTableSuite`
Author: gatorsmile
Closes #13447 from gatorsmile/insertIfNotExist.
(cherry picked from commit e5d703bca85c65ce329b1e202283cfa35d109146)
Signed-off-by: Yin Huai
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b82abde0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b82abde0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b82abde0
Branch: refs/heads/branch-2.0
Commit: b82abde060d97bd95f4fba547545a830602a35fa
Parents: 3994372
Author: gatorsmile
Authored: Thu Jun 16 22:54:02 2016 -0700
Committer: Yin Huai
Committed: Thu Jun 16 22:54:17 2016 -0700
--
.../apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +-
.../spark/sql/catalyst/parser/AstBuilder.scala | 6 ++
.../plans/logical/basicLogicalOperators.scala | 1 +
.../sql/catalyst/parser/PlanParserSuite.scala | 13 ++--
.../sql/hive/InsertIntoHiveTableSuite.scala | 68
5 files changed, 85 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b82abde0/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
--
diff --git
a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index b603196..23e925e 100644
---
a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++
b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -203,7 +203,7 @@ query
;
insertInto
-: INSERT OVERWRITE TABLE tableIdentifier partitionSpec? (IF NOT EXISTS)?
+: INSERT OVERWRITE TABLE tableIdentifier (partitionSpec (IF NOT EXISTS)?)?
| INSERT INTO TABLE? tableIdentifier partitionSpec?
;
http://git-wip-us.apache.org/repos/asf/spark/blob/b82abde0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index e380643..c7420a1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -171,6 +171,12 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
Logging {
val tableIdent = visitTableIdentifier(ctx.tableIdentifier)
val partitionKeys =
Option(ctx.partitionSpec).map(visitPartitionSpec).getOrElse(Map.empty)
+val dynamicPartitionKeys = partitionKeys.filter(_._2.isEmpty)
+if (ctx.EXISTS != null && dynamicPartitionKeys.nonEmpty) {
+ throw new ParseException(s"Dynamic partitions do not support IF NOT
EXISTS. Specified " +
+"partitions with value: " + dynamicPartitionKeys.keys.mkString("[",
",", "]"), ctx)
+}
+
InsertIntoTable(
UnresolvedRelation(tableIdent, None),
partitionKeys,
http://git-wip-us.apache.org/repos/asf/spark/blob/b82abde0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 898784d..6c3eb3a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -377,6 +377,7 @@ case class InsertIntoTable(
}
assert(overwrite || !ifNotExists)
+ assert(partition.values.forall(_.nonEmpty) || !ifNotExists)
override lazy val resolved: Boolean =
childrenResolved && table.resolved && expectedColumns.forall { expected =>
child.output.size == expected.size && child.output.zip(expect
