This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 643085910317 [SPARK-60544][SQL] Run the dynamic partition checks 
before `saveAsTable` overwrite drops the existing Hive table
643085910317 is described below

commit 6430859103171c1f72377cef2337f953cf299cdb
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Oct 25 11:52:32 2026 -0700

    [SPARK-60544][SQL] Run the dynamic partition checks before `saveAsTable` 
overwrite drops the existing Hive table
    
    ### What changes were proposed in this pull request?
    
    This PR makes `CreateHiveTableAsSelectCommand` run the dynamic partition 
checks of the Hive write on the new table before it drops the existing table 
for `saveAsTable` with `SaveMode.Overwrite` and `format("hive")`.
    
    `CreateHiveTableAsSelectCommand` now mixes in `V1WritesHiveUtils` and calls 
`getDynamicPartitionColumns` with the new table definition and the partition 
spec of the CTAS, right after `validateHiveSerdeTable` (SPARK-60515) and before 
the drop. `InsertIntoHiveTable` calls the same method for the table that the 
command creates, so the checks and their errors are the same.
    
    ### Why are the changes needed?
    
    Since SPARK-60515, `CreateHiveTableAsSelectCommand` runs the checks of Hive 
on the new table before it drops the existing table. However, the dynamic 
partition checks in `V1WritesHiveUtils.getDynamicPartitionColumns` still run 
later, when the command builds the `InsertIntoHiveTable` for the table that it 
has just created. Under the default `hive.exec.dynamic.partition.mode=strict`, 
or with `hive.exec.dynamic.partition=false`, a partitioned overwrite fails 
there, and the existing table [...]
    
    ```scala
    spark.range(3).write.format("hive").saveAsTable("t")
    spark.range(5).selectExpr("id", "id % 2 AS 
p").write.format("hive").partitionBy("p").mode("overwrite").saveAsTable("t")
    // SparkException: Dynamic partition strict mode requires at least one 
static partition column. To turn this off set 
hive.exec.dynamic.partition.mode=nonstrict
    spark.catalog.tableExists("t")
    // false
    ```
    
    It happens with `option("fileFormat", "parquet")` too, because 
`RelationConversions` converts only a non-partitioned Hive CTAS to a data 
source write. The checks only need the partition columns of the new table and 
the Hive conf, so they can run before the drop.
    
    peter-toth reported this in the review of #59757 
(https://github.com/apache/spark/pull/59757#discussion_r4370977080).
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. When the dynamic partition checks fail, such an overwrite keeps the 
existing table and its data. The error is the same as before. Valid overwrites 
are unchanged.
    
    ### How was this patch tested?
    
    Added a test to `MetastoreDataSourcesSuite` for 
`hive.exec.dynamic.partition.mode=strict` and 
`hive.exec.dynamic.partition=false` with the `textfile` and `parquet` file 
formats. It fails without this fix. The existing test `create partitioned hive 
serde table as select` in `HiveDDLSuite` covers a valid partitioned overwrite 
of an existing table.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #59784 from dongjoon-hyun/SPARK-60544.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../execution/CreateHiveTableAsSelectCommand.scala | 10 +++++---
 .../spark/sql/hive/MetastoreDataSourcesSuite.scala | 27 +++++++++++++++++++++-
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
index 280978b8543e..7e7e94da1e4b 100644
--- 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
+++ 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala
@@ -40,7 +40,7 @@ case class CreateHiveTableAsSelectCommand(
     query: LogicalPlan,
     outputColumnNames: Seq[String],
     mode: SaveMode)
-  extends LeafRunnableCommand with CTEInChildren {
+  extends LeafRunnableCommand with CTEInChildren with V1WritesHiveUtils {
   assert(query.resolved)
   override def innerChildren: Seq[LogicalPlan] = query :: Nil
 
@@ -75,9 +75,13 @@ case class CreateHiveTableAsSelectCommand(
         // `SaveAsV1TableCommand` runs this command with `SaveMode.Overwrite` 
to replace the
         // table. Drop it here, after the analysis and the checks of the new 
table, because the
         // new table is created before the write.
+        val qualifiedNewTableDesc =
+          newTableDesc.copy(identifier = 
catalog.qualifyIdentifier(tableIdentifier))
         
sparkSession.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog]
-          .validateHiveSerdeTable(
-            newTableDesc.copy(identifier = 
catalog.qualifyIdentifier(tableIdentifier)))
+          .validateHiveSerdeTable(qualifiedNewTableDesc)
+        // The dynamic partition checks of the write, e.g., the strict mode.
+        getDynamicPartitionColumns(qualifiedNewTableDesc,
+          qualifiedNewTableDesc.partitionColumnNames.map(_ -> None).toMap, 
query)
         catalog.dropTable(tableIdentifier, ignoreIfNotExists = true, purge = 
false)
       }
       // TODO ideally, we should get the output data ready first and then
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index c339b8130689..71fd3e9b28fc 100644
--- 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -23,9 +23,10 @@ import java.time.LocalTime
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hive.ql.ErrorMsg
 import org.apache.logging.log4j.Level
 
-import org.apache.spark.SparkThrowable
+import org.apache.spark.{SparkException, SparkThrowable}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, 
CatalogTable, CatalogTableType}
@@ -1371,6 +1372,30 @@ class MetastoreDataSourcesSuite extends QueryTest
     }
   }
 
+  test("SPARK-60544: saveAsTable with mode Overwrite keeps the existing hive 
table when " +
+      "the dynamic partition checks fail") {
+    Seq(
+      ("hive.exec.dynamic.partition.mode", "strict", 
ErrorMsg.DYNAMIC_PARTITION_STRICT_MODE),
+      ("hive.exec.dynamic.partition", "false", 
ErrorMsg.DYNAMIC_PARTITION_DISABLED)
+    ).foreach { case (key, value, errorMsg) =>
+      // A partitioned Hive CTAS is not converted to a data source write, even 
with Parquet.
+      Seq("textfile", "parquet").foreach { fileFormat =>
+        withSQLConf(key -> value) {
+          withTable("t") {
+            spark.range(3).write.format("hive").saveAsTable("t")
+            val e = intercept[SparkException] {
+              spark.range(5).selectExpr("id", "id % 2 AS 
p").write.format("hive")
+                .option("fileFormat", fileFormat).partitionBy("p")
+                .mode(SaveMode.Overwrite).saveAsTable("t")
+            }
+            assert(e.getMessage === errorMsg.getMsg)
+            checkAnswer(table("t"), spark.range(3).toDF())
+          }
+        }
+      }
+    }
+  }
+
   test("insertInto - source and target are the same table") {
     val tableName = "tab1"
     withTable(tableName) {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to