Repository: spark Updated Branches: refs/heads/master 12ab7f7e8 -> aba9492d2
[SPARK-21895][SQL] Support changing database in HiveClient ## What changes were proposed in this pull request? Supporting moving tables across different database in HiveClient `alterTable` ## How was this patch tested? N/A Author: gatorsmile <[email protected]> Closes #19104 from gatorsmile/alterTable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aba9492d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aba9492d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aba9492d Branch: refs/heads/master Commit: aba9492d25e285d00033c408e9bfdd543ee12f72 Parents: 12ab7f7 Author: gatorsmile <[email protected]> Authored: Fri Sep 1 13:21:06 2017 -0700 Committer: gatorsmile <[email protected]> Committed: Fri Sep 1 13:21:06 2017 -0700 ---------------------------------------------------------------------- .../spark/sql/hive/HiveExternalCatalog.scala | 2 +- .../spark/sql/hive/client/HiveClient.scala | 11 ++++-- .../spark/sql/hive/client/HiveClientImpl.scala | 7 +++- .../spark/sql/hive/client/VersionsSuite.scala | 41 +++++++++++++++++++- 4 files changed, 53 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/aba9492d/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 34af37c..96dc983 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -512,7 +512,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat identifier = TableIdentifier(newName, Some(db)), storage = storageWithNewPath) - client.alterTable(oldName, newTable) + client.alterTable(db, oldName, newTable) } private def getLocationFromStorageProps(table: CatalogTable): Option[String] = { http://git-wip-us.apache.org/repos/asf/spark/blob/aba9492d/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala index 8cff0ca..ee3eb2e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala @@ -90,10 +90,15 @@ private[hive] trait HiveClient { def dropTable(dbName: String, tableName: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit /** Alter a table whose name matches the one specified in `table`, assuming it exists. */ - final def alterTable(table: CatalogTable): Unit = alterTable(table.identifier.table, table) + final def alterTable(table: CatalogTable): Unit = { + alterTable(table.database, table.identifier.table, table) + } - /** Updates the given table with new metadata, optionally renaming the table. */ - def alterTable(tableName: String, table: CatalogTable): Unit + /** + * Updates the given table with new metadata, optionally renaming the table or + * moving across different database. + */ + def alterTable(dbName: String, tableName: String, table: CatalogTable): Unit /** Creates a new database with the given name. */ def createDatabase(database: CatalogDatabase, ignoreIfExists: Boolean): Unit http://git-wip-us.apache.org/repos/asf/spark/blob/aba9492d/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 7c0b9bf..69dac7b 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -495,7 +495,10 @@ private[hive] class HiveClientImpl( shim.dropTable(client, dbName, tableName, true, ignoreIfNotExists, purge) } - override def alterTable(tableName: String, table: CatalogTable): Unit = withHiveState { + override def alterTable( + dbName: String, + tableName: String, + table: CatalogTable): Unit = withHiveState { // getTableOption removes all the Hive-specific properties. Here, we fill them back to ensure // these properties are still available to the others that share the same Hive metastore. // If users explicitly alter these Hive-specific properties through ALTER TABLE DDL, we respect @@ -503,7 +506,7 @@ private[hive] class HiveClientImpl( val hiveTable = toHiveTable( table.copy(properties = table.ignoredProperties ++ table.properties), Some(userName)) // Do not use `table.qualifiedName` here because this may be a rename - val qualifiedTableName = s"${table.database}.$tableName" + val qualifiedTableName = s"$dbName.$tableName" shim.alterTable(client, qualifiedTableName, hiveTable) } http://git-wip-us.apache.org/repos/asf/spark/blob/aba9492d/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala index cbbe869..1d9c8da 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala @@ -232,12 +232,49 @@ class VersionsSuite extends SparkFunSuite with Logging { assert(client.getTable("default", "src").properties.contains("changed")) } - test(s"$version: alterTable(tableName: String, table: CatalogTable)") { + test(s"$version: alterTable(dbName: String, tableName: String, table: CatalogTable)") { val newTable = client.getTable("default", "src").copy(properties = Map("changedAgain" -> "")) - client.alterTable("src", newTable) + client.alterTable("default", "src", newTable) assert(client.getTable("default", "src").properties.contains("changedAgain")) } + test(s"$version: alterTable - rename") { + val newTable = client.getTable("default", "src") + .copy(identifier = TableIdentifier("tgt", database = Some("default"))) + assert(!client.tableExists("default", "tgt")) + + client.alterTable("default", "src", newTable) + + assert(client.tableExists("default", "tgt")) + assert(!client.tableExists("default", "src")) + } + + test(s"$version: alterTable - change database") { + val tempDB = CatalogDatabase( + "temporary", description = "test create", tempDatabasePath, Map()) + client.createDatabase(tempDB, ignoreIfExists = true) + + val newTable = client.getTable("default", "tgt") + .copy(identifier = TableIdentifier("tgt", database = Some("temporary"))) + assert(!client.tableExists("temporary", "tgt")) + + client.alterTable("default", "tgt", newTable) + + assert(client.tableExists("temporary", "tgt")) + assert(!client.tableExists("default", "tgt")) + } + + test(s"$version: alterTable - change database and table names") { + val newTable = client.getTable("temporary", "tgt") + .copy(identifier = TableIdentifier("src", database = Some("default"))) + assert(!client.tableExists("default", "src")) + + client.alterTable("temporary", "tgt", newTable) + + assert(client.tableExists("default", "src")) + assert(!client.tableExists("temporary", "tgt")) + } + test(s"$version: listTables(database)") { assert(client.listTables("default") === Seq("src", "temporary")) } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
