Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/16672#discussion_r100734735
--- Diff:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
---
@@ -1431,4 +1432,133 @@ class HiveDDLSuite
}
}
}
+
+ test("insert data to a data source table which has a not existed
location should succeed") {
+ withTable("t") {
+ withTempDir { dir =>
+ spark.sql(
+ s"""CREATE TABLE t(a string, b int)
+ |USING parquet
+ |OPTIONS(path "file:${dir.getCanonicalPath}")
+ """.stripMargin)
+ var table =
spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+ val expectedPath = s"file:${dir.getAbsolutePath.stripSuffix("/")}"
+ assert(table.location.stripSuffix("/") == expectedPath)
+
+ dir.delete
+ assert(!new File(table.location).exists())
+ spark.sql("INSERT INTO TABLE t SELECT 'c', 1")
+ checkAnswer(spark.table("t"), Row("c", 1) :: Nil)
+
+ Utils.deleteRecursively(dir)
+ assert(!new File(table.location).exists())
+ spark.sql("INSERT OVERWRITE TABLE t SELECT 'c', 1")
+ checkAnswer(spark.table("t"), Row("c", 1) :: Nil)
+
+ var newDir = dir.getAbsolutePath.stripSuffix("/") + "/x"
+ spark.sql(s"ALTER TABLE t SET LOCATION '$newDir'")
+ spark.sessionState.catalog.refreshTable(TableIdentifier("t"))
+
+ table =
spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+ assert(table.location == newDir)
+ assert(!new File(newDir).exists())
+
+ spark.sql("INSERT INTO TABLE t SELECT 'c', 1")
+ checkAnswer(spark.table("t"), Row("c", 1) :: Nil)
+ }
+ }
+ }
+
+ test("insert into a data source table with no existed partition location
should succeed") {
+ withTable("t") {
+ withTempDir { dir =>
+ spark.sql(
+ s"""CREATE TABLE t(a int, b int, c int, d int)
+ |USING parquet
+ |PARTITIONED BY(a, b)
+ |LOCATION "file:${dir.getCanonicalPath}"
+ """.stripMargin)
+ var table =
spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+ val expectedPath = s"file:${dir.getAbsolutePath.stripSuffix("/")}"
+ assert(table.location.stripSuffix("/") == expectedPath)
+
+ spark.sql("INSERT INTO TABLE t PARTITION(a=1, b=2) SELECT 3, 4")
+ checkAnswer(spark.table("t"), Row(3, 4, 1, 2) :: Nil)
+
+ val partLoc = new File(s"${dir.getAbsolutePath}/a=1")
--- End diff --
A general comment about the test cases. Can you please check whether the
directory exists after the insert? It can help others confirm the path is
correct
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
