Repository: spark Updated Branches: refs/heads/master 8cdb81fa8 -> 262833397
[SPARK-16358][SQL] Remove InsertIntoHiveTable From Logical Plan #### What changes were proposed in this pull request? LogicalPlan `InsertIntoHiveTable` is useless. Thus, we can remove it from the code base. #### How was this patch tested? The existing test cases Author: gatorsmile <gatorsm...@gmail.com> Closes #14037 from gatorsmile/InsertIntoHiveTable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/26283339 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/26283339 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/26283339 Branch: refs/heads/master Commit: 26283339786f38c50722a7488d0bca8573b9c352 Parents: 8cdb81f Author: gatorsmile <gatorsm...@gmail.com> Authored: Mon Jul 4 13:45:07 2016 +0800 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Mon Jul 4 13:45:07 2016 +0800 ---------------------------------------------------------------------- .../spark/sql/hive/HiveMetastoreCatalog.scala | 39 -------------------- .../apache/spark/sql/hive/HiveStrategies.scala | 7 +--- .../sql/hive/execution/HiveComparisonTest.scala | 2 - 3 files changed, 1 insertion(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/26283339/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala index 2e0b5d5..151e456 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala @@ -372,12 +372,6 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreParquet(r) => InsertIntoTable(convertToParquetRelation(r), partition, child, overwrite, ifNotExists) - // Write path - case InsertIntoHiveTable(r: MetastoreRelation, partition, child, overwrite, ifNotExists) - // Inserting into partitioned table is not supported in Parquet data source (yet). - if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreParquet(r) => - InsertIntoTable(convertToParquetRelation(r), partition, child, overwrite, ifNotExists) - // Read path case relation: MetastoreRelation if shouldConvertMetastoreParquet(relation) => val parquetRelation = convertToParquetRelation(relation) @@ -416,12 +410,6 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreOrc(r) => InsertIntoTable(convertToOrcRelation(r), partition, child, overwrite, ifNotExists) - // Write path - case InsertIntoHiveTable(r: MetastoreRelation, partition, child, overwrite, ifNotExists) - // Inserting into partitioned table is not supported in Orc data source (yet). - if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreOrc(r) => - InsertIntoTable(convertToOrcRelation(r), partition, child, overwrite, ifNotExists) - // Read path case relation: MetastoreRelation if shouldConvertMetastoreOrc(relation) => val orcRelation = convertToOrcRelation(relation) @@ -489,30 +477,3 @@ private[hive] object MetaStorePartitionedTableFileCatalog { } } } - -/** - * A logical plan representing insertion into Hive table. - * This plan ignores nullability of ArrayType, MapType, StructType unlike InsertIntoTable - * because Hive table doesn't have nullability for ARRAY, MAP, STRUCT types. - */ -private[hive] case class InsertIntoHiveTable( - table: MetastoreRelation, - partition: Map[String, Option[String]], - child: LogicalPlan, - overwrite: Boolean, - ifNotExists: Boolean) - extends LogicalPlan with Command { - - override def children: Seq[LogicalPlan] = child :: Nil - override def output: Seq[Attribute] = Seq.empty - - val numDynamicPartitions = partition.values.count(_.isEmpty) - - // This is the expected schema of the table prepared to be inserted into, - // including dynamic partition columns. - val tableOutput = table.attributes ++ table.partitionKeys.takeRight(numDynamicPartitions) - - override lazy val resolved: Boolean = childrenResolved && child.output.zip(tableOutput).forall { - case (childAttr, tableAttr) => childAttr.dataType.sameType(tableAttr.dataType) - } -} http://git-wip-us.apache.org/repos/asf/spark/blob/26283339/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index 71b180e..17956de 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -44,12 +44,7 @@ private[hive] trait HiveStrategies { def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case logical.InsertIntoTable( table: MetastoreRelation, partition, child, overwrite, ifNotExists) => - execution.InsertIntoHiveTable( - table, partition, planLater(child), overwrite, ifNotExists) :: Nil - case hive.InsertIntoHiveTable( - table: MetastoreRelation, partition, child, overwrite, ifNotExists) => - execution.InsertIntoHiveTable( - table, partition, planLater(child), overwrite, ifNotExists) :: Nil + InsertIntoHiveTable(table, partition, planLater(child), overwrite, ifNotExists) :: Nil case _ => Nil } } http://git-wip-us.apache.org/repos/asf/spark/blob/26283339/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index f5d2f02..80e75aa 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.execution.command._ -import org.apache.spark.sql.hive.{InsertIntoHiveTable => LogicalInsertIntoHiveTable} import org.apache.spark.sql.hive.test.{TestHive, TestHiveQueryExecution} /** @@ -349,7 +348,6 @@ abstract class HiveComparisonTest val containsCommands = originalQuery.analyzed.collectFirst { case _: Command => () case _: InsertIntoTable => () - case _: LogicalInsertIntoHiveTable => () }.nonEmpty if (containsCommands) { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
