This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new d97e693fc3ea [SPARK-60129][SQL] Do not overwrite existing partitions 
in InMemoryCatalog.createPartitions with ignoreIfExists
d97e693fc3ea is described below

commit d97e693fc3eac8896300a8fd3178ca160a959b5b
Author: Kumar Sambhav <[email protected]>
AuthorDate: Sat Oct 24 06:31:12 2026 +0000

    [SPARK-60129][SQL] Do not overwrite existing partitions in 
InMemoryCatalog.createPartitions with ignoreIfExists
    
    ### What changes were proposed in this pull request?
    
    In `InMemoryCatalog.createPartitions`, skip partitions that already exist 
when `ignoreIfExists = true`, instead of overwriting their metadata. Previously 
`ignoreIfExists` only disabled the duplicate check, and the subsequent loop 
still replaced every existing partition (including its location) with the new 
definition.
    
    This aligns `InMemoryCatalog` with `HiveExternalCatalog`, which passes 
`ifNotExists` to the Hive metastore and leaves existing partitions untouched.
    
    ### Why are the changes needed?
    
    `ALTER TABLE .. RECOVER PARTITIONS` calls `createPartitions(..., 
ignoreIfExists = true)` for every partition directory it finds under the table 
root. With the in-memory catalog, after the table root is changed via `ALTER 
TABLE .. SET LOCATION`, recovering partitions silently repoints 
already-registered partitions to the new root, so queries return different data:
    
    ```sql
    CREATE TABLE t (id INT, value STRING) USING PARQUET PARTITIONED BY (p 
STRING)
      LOCATION '/tmp/spark_t_old';
    INSERT INTO t VALUES (1, 'old-a', 'a'), (2, 'old-b', 'b');
    
    CREATE TABLE t_new (id INT, value STRING) USING PARQUET PARTITIONED BY (p 
STRING)
      LOCATION '/tmp/spark_t_new';
    INSERT INTO t_new VALUES (3, 'new-b', 'b');
    
    ALTER TABLE t SET LOCATION '/tmp/spark_t_new';
    ALTER TABLE t RECOVER PARTITIONS;
    SELECT id, value, p FROM t ORDER BY id;
    -- 1  old-a  a
    -- 3  new-b  b    <- partition p=b now points to /tmp/spark_t_new/p=b
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, it fixes a bug. With the in-memory catalog 
(`spark.sql.catalogImplementation=in-memory`), `ALTER TABLE .. RECOVER 
PARTITIONS` no longer changes the location of existing partitions. For the 
example above, the last query now returns:
    
    ```
    1  old-a  a
    2  old-b  b
    ```
    
    This matches the behavior of the Hive catalog.
    
    ### How was this patch tested?
    
    Added new tests:
    - `ExternalCatalogSuite`: "create partitions with ignoreIfExists does not 
alter existing partitions". It runs for both `InMemoryCatalogSuite` and 
`HiveExternalCatalogSuite`.
    - v1 `AlterTableRecoverPartitionsSuiteBase`: "SPARK-60129: keep locations 
of existing partitions". It runs for the in-memory (sequential and parallel) 
and Hive v1 suites.
    
    Both new tests fail without the fix and pass with it:
    
    ```
    build/sbt 'catalyst/testOnly *InMemoryCatalogSuite'
    build/sbt 'sql/testOnly 
org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitions*'
    build/sbt 'hive/testOnly *HiveExternalCatalogSuite 
org.apache.spark.sql.hive.execution.command.AlterTableRecoverPartitionsSuite'
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Co-authored: (claude-opus-5-5)
    
    Closes #59728 from sambhav37/SPARK-60129.
    
    Authored-by: Kumar Sambhav <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
    (cherry picked from commit 8e877428ff3c3c8468e5ffcd23e1adfe9be4ca66)
    Signed-off-by: Wenchen Fan <[email protected]>
---
 .../sql/catalyst/catalog/InMemoryCatalog.scala      |  6 ++++--
 .../sql/catalyst/catalog/ExternalCatalogSuite.scala | 11 +++++++++++
 .../v1/AlterTableRecoverPartitionsSuite.scala       | 21 ++++++++++++++++++++-
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
index 1aa5c483db88..4d67b3f4cfe8 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
@@ -441,13 +441,15 @@ class InMemoryCatalog(
       ignoreIfExists: Boolean): Unit = synchronized {
     requireTableExists(db, table)
     val existingParts = catalog(db).tables(table).partitions
-    val parts = toCatalogPartitionSpec(newParts)
+    val allParts = toCatalogPartitionSpec(newParts)
     if (!ignoreIfExists) {
-      val dupSpecs = parts.collect { case p if existingParts.contains(p.spec) 
=> p.spec }
+      val dupSpecs = allParts.collect { case p if 
existingParts.contains(p.spec) => p.spec }
       if (dupSpecs.nonEmpty) {
         throw new PartitionsAlreadyExistException(db = db, table = table, 
specs = dupSpecs)
       }
     }
+    // Keep existing partitions untouched, as the Hive metastore does with 
`ifNotExists`.
+    val parts = allParts.filterNot(p => existingParts.contains(p.spec))
 
     val tableMeta = getTable(db, table)
     val partitionColumnNames = tableMeta.partitionColumnNames
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index f06128f23679..065ec23ec9d3 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -372,6 +372,17 @@ abstract class ExternalCatalogSuite extends SparkFunSuite {
     catalog.createPartitions("db2", "tbl2", Seq(part1), ignoreIfExists = true)
   }
 
+  test("create partitions with ignoreIfExists does not alter existing 
partitions") {
+    val catalog = newBasicCatalog()
+    val oldLocation = catalog.getPartition("db2", "tbl2", part1.spec).location
+    val newLocation = newUriForPartition(Seq("a=1", "b=2"))
+    val newPart1 = part1.copy(storage = storageFormat.copy(locationUri = 
Some(newLocation)))
+    val newPart = CatalogTablePartition(Map("a" -> "7", "b" -> "8"), 
storageFormat)
+    catalog.createPartitions("db2", "tbl2", Seq(newPart1, newPart), 
ignoreIfExists = true)
+    assert(catalog.getPartition("db2", "tbl2", part1.spec).location == 
oldLocation)
+    assert(catalog.getPartitionOption("db2", "tbl2", newPart.spec).isDefined)
+  }
+
   test("create partitions without location") {
     val catalog = newBasicCatalog()
     val table = CatalogTable(
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
index 54c0e7883ccd..8142882820a5 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command.v1
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.internal.config.RDD_PARALLEL_LISTING_THRESHOLD
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.execution.command
 
 /**
@@ -115,6 +115,25 @@ trait AlterTableRecoverPartitionsSuiteBase extends 
command.AlterTableRecoverPart
     }
   }
 
+  test("SPARK-60129: keep locations of existing partitions") {
+    withNamespaceAndTable("ns", "tbl") { t =>
+      withTempDir { newRoot =>
+        sql(s"CREATE TABLE $t (id INT, p STRING) $defaultUsing PARTITIONED BY 
(p)")
+        sql(s"INSERT INTO $t PARTITION (p='b') SELECT 1")
+        checkAnswer(sql(s"SELECT id, p FROM $t"), Row(1, "b"))
+
+        val partDir = new Path(newRoot.getCanonicalPath, "p=b")
+        val fs = partDir.getFileSystem(spark.sessionState.newHadoopConf())
+        fs.mkdirs(partDir)
+        sql(s"ALTER TABLE $t SET LOCATION '${newRoot.toURI}'")
+        sql(s"ALTER TABLE $t RECOVER PARTITIONS")
+
+        checkPartitions(t, Map("p" -> "b"))
+        checkAnswer(sql(s"SELECT id, p FROM $t"), Row(1, "b"))
+      }
+    }
+  }
+
   test("ALTER TABLE .. RECOVER PARTITIONS is not allowed for non-partitioned 
table") {
     withTable("tbl") {
       sql("CREATE TABLE tbl(col1 int, col2 string) USING parquet")


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to