viirya commented on a change in pull request #24938: [SPARK-27946][SQL] Hive
DDL to Spark DDL conversion USING "show create table"
URL: https://github.com/apache/spark/pull/24938#discussion_r309991310
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
##########
@@ -195,4 +203,214 @@ class HiveShowCreateTableSuite extends
ShowCreateTableSuite with TestHiveSinglet
hiveContext.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog]
.client.runSqlHive(ddl)
}
+
+ private def checkCreateSparkTable(tableName: String): Unit = {
+ val table = TableIdentifier(tableName, Some("default"))
+ val db = table.database.get
+ val hiveTable = spark.sharedState.externalCatalog.getTable(db, table.table)
+ val shownSparkDDL = sql(s"SHOW CREATE TABLE ${table.quotedString} AS
SPARK").head().getString(0)
+ // Drops original Hive table.
+ sql(s"DROP TABLE ${table.quotedString}")
+
+ try {
+ sql(shownSparkDDL)
+ val actual = spark.sharedState.externalCatalog.getTable(db, table.table)
+ val shownDDL = sql(s"SHOW CREATE TABLE
${table.quotedString}").head().getString(0)
+
+ // Drops created Spark table using `SHOW CREATE TABLE AS SPARK`.
+ sql(s"DROP TABLE ${table.quotedString}")
+
+ sql(shownDDL)
+ val expected = spark.sharedState.externalCatalog.getTable(db,
table.table)
+
+ checkCatalogTables(expected, actual)
+ checkHiveCatalogTables(hiveTable, actual)
+ } finally {
+ sql(s"DROP TABLE IF EXISTS ${table.table}")
+ }
+ }
+
+ private def checkHiveCatalogTables(expected: CatalogTable, actual:
CatalogTable): Unit = {
+ def normalize(table: CatalogTable): CatalogTable = {
+ val nondeterministicProps = Set(
+ "CreateTime",
+ "transient_lastDdlTime",
+ "grantTime",
+ "lastUpdateTime",
+ "last_modified_by",
+ "last_modified_time",
+ "Owner:",
+ // The following are hive specific schema parameters which we do not
need to match exactly.
+ "totalNumberFiles",
+ "maxFileSize",
+ "minFileSize"
+ )
+
+ table.copy(
+ createTime = 0L,
+ lastAccessTime = 0L,
+ properties =
table.properties.filterKeys(!nondeterministicProps.contains(_)),
+ stats = None,
+ ignoredProperties = Map.empty,
+ storage = CatalogStorageFormat.empty,
+ provider = None,
+ tracksPartitionsInCatalog = false
+ )
+ }
+ assert(normalize(actual) == normalize(expected))
+ }
+
+ test("simple hive table as spark") {
+ withTable("t1") {
+ sql(
+ s"""CREATE TABLE t1 (
Review comment:
yes, fixed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
