This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new f1f1bebe3e95 [SPARK-56558][SQL] CTAS IF NOT EXISTS bypasses non-empty
location check when table does not exist in catalog
f1f1bebe3e95 is described below
commit f1f1bebe3e9514d2d4b82950d7bab7c6d3d46559
Author: Pablo Langa <[email protected]>
AuthorDate: Fri Aug 7 20:49:42 2026 +0800
[SPARK-56558][SQL] CTAS IF NOT EXISTS bypasses non-empty location check
when table does not exist in catalog
### What changes were proposed in this pull request?
CREATE TABLE IF NOT EXISTS ... LOCATION ... AS SELECT silently overwrites
data in a non-empty directory when the table does not yet exist in the catalog,
bypassing the protection introduced in
https://github.com/apache/spark/pull/32411.
IF NOT EXISTS semantics apply only to the catalog: if the table already
exists, the command is a no-op. If the table does not exist in the catalog, the
non-empty location check must still be enforced, identical to a plain CREATE
TABLE ... AS SELECT.
This PR aligns the behaviour across the two scenarios CREATE TABLE and
CREATE TABLE IF NOT EXISTS when the table do not exists.
### Why are the changes needed?
Aling the behavior between CREATE TABLE and CREATE TABLE IF NOT EXISTS
### Does this PR introduce _any_ user-facing change?
Yes, now CREATE TABLE IF NOT EXISTS will fail if the location is not empty
like CREATE TABLE
### How was this patch tested?
Tests and manually
### Was this patch authored or co-authored using generative AI tooling?
yes, co-authored by Claude Code
Closes #55444 from planga82/fix/create_table_existing_location.
Authored-by: Pablo Langa <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 44551f920250dda0ce9b50374bbd912c4c28a029)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/execution/command/DataWritingCommand.scala | 4 ++--
.../execution/datasources/TableLocationSuite.scala | 23 +++++++++++++++++++++-
.../spark/sql/hive/execution/SQLQuerySuite.scala | 23 ++++++++++++++++++++++
3 files changed, 47 insertions(+), 3 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
index 90050b25e954..e5ff8cd5a564 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
@@ -103,14 +103,14 @@ object DataWritingCommand {
}
/**
* When execute CTAS operators, and the location is not empty, throw
[[AnalysisException]].
- * For CTAS, the SaveMode is always [[ErrorIfExists]]
*
* @param tablePath Table location.
* @param saveMode Save mode of the table.
* @param hadoopConf Configuration.
*/
def assertEmptyRootPath(tablePath: URI, saveMode: SaveMode, hadoopConf:
Configuration): Unit = {
- if (saveMode == SaveMode.ErrorIfExists &&
!SQLConf.get.allowNonEmptyLocationInCTAS) {
+ if ((saveMode == SaveMode.ErrorIfExists || saveMode == SaveMode.Ignore) &&
+ !SQLConf.get.allowNonEmptyLocationInCTAS) {
val filePath = new org.apache.hadoop.fs.Path(tablePath)
val fs = filePath.getFileSystem(hadoopConf)
if (fs.exists(filePath) &&
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/TableLocationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/TableLocationSuite.scala
index dacb745f49ae..5aa165d22787 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/TableLocationSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/TableLocationSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources
import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
@@ -76,4 +76,25 @@ class TableLocationSuite extends SharedSparkSession {
}
}
}
+
+ test("SPARK-56558: CTAS IF NOT EXISTS should be with non-existent or empty
location") {
+ withSQLConf(SQLConf.ALLOW_NON_EMPTY_LOCATION_IN_CTAS.key -> "false") {
+ withTempDir { dir =>
+ val tempLocation = dir.getCanonicalPath
+ withTable("ctas1", "ctas2") {
+ sql(s"CREATE TABLE ctas1 USING parquet LOCATION
'$tempLocation/ctas1' " +
+ "AS SELECT 1 AS ID")
+ // Table ctas2 does not exist in the catalog, so IF NOT EXISTS must
not skip the
+ // non-empty location check and overwrite the data of table ctas1.
+ val m = intercept[AnalysisException] {
+ sql(s"CREATE TABLE IF NOT EXISTS ctas2 USING parquet LOCATION
'$tempLocation/ctas1' " +
+ "AS SELECT 2 AS ID")
+ }.getMessage
+ assert(m.contains("CREATE-TABLE-AS-SELECT cannot create table with
location to a " +
+ "non-empty directory"))
+ checkAnswer(spark.table("ctas1"), Row(1))
+ }
+ }
+ }
+ }
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 68f46cbcb112..74d317ba1e58 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -594,6 +594,29 @@ abstract class SQLQuerySuiteBase extends QueryTest with
TestHiveSingleton {
}
}
+ test("SPARK-56558: CTAS IF NOT EXISTS Hive Table should be with non-existent
" +
+ "or empty location") {
+ withSQLConf(SQLConf.ALLOW_NON_EMPTY_LOCATION_IN_CTAS.key -> "false") {
+ withTempDir { dir =>
+ val tempLocation = dir.toURI.toString
+ withTable("ctas1", "ctas_with_existing_location") {
+ sql(s"CREATE TABLE ctas1(id string) stored as rcfile LOCATION
'$tempLocation/ctas1'")
+ sql("INSERT INTO TABLE ctas1 SELECT 'A' ")
+ // The target table does not exist in the catalog, so IF NOT EXISTS
must not skip the
+ // non-empty location check and overwrite the data of table ctas1.
+ val m = intercept[AnalysisException] {
+ sql(s"""CREATE TABLE IF NOT EXISTS ctas_with_existing_location
stored as rcfile
+ |LOCATION '$tempLocation'
+ |AS SELECT key k, value FROM src ORDER BY k,
value""".stripMargin)
+ }.getMessage
+ assert(m.contains("CREATE-TABLE-AS-SELECT cannot create " +
+ "table with location to a non-empty directory"))
+ checkAnswer(spark.table("ctas1"), Row("A"))
+ }
+ }
+ }
+ }
+
test("CTAS with serde") {
withTable("ctas1", "ctas2", "ctas3", "ctas4", "ctas5") {
sql("CREATE TABLE ctas1 AS SELECT key k, value FROM src ORDER BY k,
value")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
