cloud-fan commented on a change in pull request #25306: [SPARK-28573][SQL]
Convert InsertIntoTable(HiveTableRelation) to DataSource inserting for
partitioned table
URL: https://github.com/apache/spark/pull/25306#discussion_r319814740
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
##########
@@ -228,4 +229,64 @@ class HiveOrcQuerySuite extends OrcQueryTest with
TestHiveSingleton {
}
}
}
+
+ private def getCachedDataSourceTable(table: TableIdentifier) = {
+
spark.sessionState.catalog.asInstanceOf[HiveSessionCatalog].metastoreCatalog
+ .getCachedDataSourceTable(table)
+ }
+
+ private def checkCached(tableIdentifier: TableIdentifier): Unit = {
+ getCachedDataSourceTable(tableIdentifier) match {
+ case null => fail(s"Converted ${tableIdentifier.table} should be cached
in the cache.")
+ case LogicalRelation(_: HadoopFsRelation, _, _, _) => // OK
+ case other =>
+ fail(
+ s"The cached ${tableIdentifier.table} should be a HadoopFsRelation.
" +
+ s"However, $other is returned form the cache.")
+ }
+ }
+
+ test("SPARK-28573 ORC conversation could be applied for partitioned table
insertion") {
+ withTempView("single") {
+ val singleRowDF = Seq((0, "foo")).toDF("key", "value")
+ singleRowDF.createOrReplaceTempView("single")
+ Seq("true", "false").foreach { conversion =>
+ withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true",
+ HiveUtils.CONVERT_INSERTING_PARTITIONED_TABLE.key -> conversion) {
+ withTable("dummy_orc_partitioned") {
+ withTempPath { dir =>
+ spark.sessionState.refreshTable("dummy_orc_partitioned")
+ val path = dir.getCanonicalPath
+ spark.sql(
+ s"""
+ |CREATE TABLE dummy_orc_partitioned(key INT, value STRING)
+ |PARTITIONED by (`date` STRING)
+ |STORED AS ORC
+ |LOCATION '${dir.toURI}'
Review comment:
we don't need to specify the table location, and save the `withTempPath`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
