Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/16500#discussion_r96108163
--- Diff:
sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala ---
@@ -609,6 +609,63 @@ class ParquetMetastoreSuite extends
ParquetPartitioningTest {
}
}
+ test("Explicitly added partitions should be readable after load") {
+ withTable("test_added_partitions") {
+ withTempDir { src =>
+ val newPartitionDir = src.getCanonicalPath
+ spark.range(2).selectExpr("cast(id as string)").toDF("a").write
+ .mode("overwrite")
+ .parquet(newPartitionDir)
+
+ sql(
+ """
+ |CREATE TABLE test_added_partitions (a STRING)
+ |PARTITIONED BY (b INT)
+ |STORED AS PARQUET
+ """.stripMargin)
+
+ // Create partition without data files and check whether it can be
read
+ sql(s"ALTER TABLE test_added_partitions ADD PARTITION (b='1')")
+ // This table fetch is to fill the cache with zero leaf files
+ checkAnswer(spark.table("test_added_partitions"), Seq.empty)
+
+ sql(
+ s"""
+ |LOAD DATA LOCAL INPATH '$newPartitionDir' OVERWRITE
+ |INTO TABLE test_added_partitions PARTITION(b='1')
+ """.stripMargin)
+
+ checkAnswer(
+ spark.table("test_added_partitions"),
+ Seq(("0", 1), ("1", 1)).toDF("a", "b"))
--- End diff --
we usually write `checkAnswer(df, Row("0", 1) :: Row("1", 1) :: Nil)`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
