This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new d97e693fc3ea [SPARK-60129][SQL] Do not overwrite existing partitions
in InMemoryCatalog.createPartitions with ignoreIfExists
d97e693fc3ea is described below
commit d97e693fc3eac8896300a8fd3178ca160a959b5b
Author: Kumar Sambhav <[email protected]>
AuthorDate: Sat Oct 24 06:31:12 2026 +0000
[SPARK-60129][SQL] Do not overwrite existing partitions in
InMemoryCatalog.createPartitions with ignoreIfExists
### What changes were proposed in this pull request?
In `InMemoryCatalog.createPartitions`, skip partitions that already exist
when `ignoreIfExists = true`, instead of overwriting their metadata. Previously
`ignoreIfExists` only disabled the duplicate check, and the subsequent loop
still replaced every existing partition (including its location) with the new
definition.
This aligns `InMemoryCatalog` with `HiveExternalCatalog`, which passes
`ifNotExists` to the Hive metastore and leaves existing partitions untouched.
### Why are the changes needed?
`ALTER TABLE .. RECOVER PARTITIONS` calls `createPartitions(...,
ignoreIfExists = true)` for every partition directory it finds under the table
root. With the in-memory catalog, after the table root is changed via `ALTER
TABLE .. SET LOCATION`, recovering partitions silently repoints
already-registered partitions to the new root, so queries return different data:
```sql
CREATE TABLE t (id INT, value STRING) USING PARQUET PARTITIONED BY (p
STRING)
LOCATION '/tmp/spark_t_old';
INSERT INTO t VALUES (1, 'old-a', 'a'), (2, 'old-b', 'b');
CREATE TABLE t_new (id INT, value STRING) USING PARQUET PARTITIONED BY (p
STRING)
LOCATION '/tmp/spark_t_new';
INSERT INTO t_new VALUES (3, 'new-b', 'b');
ALTER TABLE t SET LOCATION '/tmp/spark_t_new';
ALTER TABLE t RECOVER PARTITIONS;
SELECT id, value, p FROM t ORDER BY id;
-- 1 old-a a
-- 3 new-b b <- partition p=b now points to /tmp/spark_t_new/p=b
```
### Does this PR introduce _any_ user-facing change?
Yes, it fixes a bug. With the in-memory catalog
(`spark.sql.catalogImplementation=in-memory`), `ALTER TABLE .. RECOVER
PARTITIONS` no longer changes the location of existing partitions. For the
example above, the last query now returns:
```
1 old-a a
2 old-b b
```
This matches the behavior of the Hive catalog.
### How was this patch tested?
Added new tests:
- `ExternalCatalogSuite`: "create partitions with ignoreIfExists does not
alter existing partitions". It runs for both `InMemoryCatalogSuite` and
`HiveExternalCatalogSuite`.
- v1 `AlterTableRecoverPartitionsSuiteBase`: "SPARK-60129: keep locations
of existing partitions". It runs for the in-memory (sequential and parallel)
and Hive v1 suites.
Both new tests fail without the fix and pass with it:
```
build/sbt 'catalyst/testOnly *InMemoryCatalogSuite'
build/sbt 'sql/testOnly
org.apache.spark.sql.execution.command.v1.AlterTableRecoverPartitions*'
build/sbt 'hive/testOnly *HiveExternalCatalogSuite
org.apache.spark.sql.hive.execution.command.AlterTableRecoverPartitionsSuite'
```
### Was this patch authored or co-authored using generative AI tooling?
Co-authored: (claude-opus-5-5)
Closes #59728 from sambhav37/SPARK-60129.
Authored-by: Kumar Sambhav <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 8e877428ff3c3c8468e5ffcd23e1adfe9be4ca66)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/catalyst/catalog/InMemoryCatalog.scala | 6 ++++--
.../sql/catalyst/catalog/ExternalCatalogSuite.scala | 11 +++++++++++
.../v1/AlterTableRecoverPartitionsSuite.scala | 21 ++++++++++++++++++++-
3 files changed, 35 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
index 1aa5c483db88..4d67b3f4cfe8 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
@@ -441,13 +441,15 @@ class InMemoryCatalog(
ignoreIfExists: Boolean): Unit = synchronized {
requireTableExists(db, table)
val existingParts = catalog(db).tables(table).partitions
- val parts = toCatalogPartitionSpec(newParts)
+ val allParts = toCatalogPartitionSpec(newParts)
if (!ignoreIfExists) {
- val dupSpecs = parts.collect { case p if existingParts.contains(p.spec)
=> p.spec }
+ val dupSpecs = allParts.collect { case p if
existingParts.contains(p.spec) => p.spec }
if (dupSpecs.nonEmpty) {
throw new PartitionsAlreadyExistException(db = db, table = table,
specs = dupSpecs)
}
}
+ // Keep existing partitions untouched, as the Hive metastore does with
`ifNotExists`.
+ val parts = allParts.filterNot(p => existingParts.contains(p.spec))
val tableMeta = getTable(db, table)
val partitionColumnNames = tableMeta.partitionColumnNames
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index f06128f23679..065ec23ec9d3 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -372,6 +372,17 @@ abstract class ExternalCatalogSuite extends SparkFunSuite {
catalog.createPartitions("db2", "tbl2", Seq(part1), ignoreIfExists = true)
}
+ test("create partitions with ignoreIfExists does not alter existing
partitions") {
+ val catalog = newBasicCatalog()
+ val oldLocation = catalog.getPartition("db2", "tbl2", part1.spec).location
+ val newLocation = newUriForPartition(Seq("a=1", "b=2"))
+ val newPart1 = part1.copy(storage = storageFormat.copy(locationUri =
Some(newLocation)))
+ val newPart = CatalogTablePartition(Map("a" -> "7", "b" -> "8"),
storageFormat)
+ catalog.createPartitions("db2", "tbl2", Seq(newPart1, newPart),
ignoreIfExists = true)
+ assert(catalog.getPartition("db2", "tbl2", part1.spec).location ==
oldLocation)
+ assert(catalog.getPartitionOption("db2", "tbl2", newPart.spec).isDefined)
+ }
+
test("create partitions without location") {
val catalog = newBasicCatalog()
val table = CatalogTable(
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
index 54c0e7883ccd..8142882820a5 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/AlterTableRecoverPartitionsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command.v1
import org.apache.hadoop.fs.Path
import org.apache.spark.internal.config.RDD_PARALLEL_LISTING_THRESHOLD
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.execution.command
/**
@@ -115,6 +115,25 @@ trait AlterTableRecoverPartitionsSuiteBase extends
command.AlterTableRecoverPart
}
}
+ test("SPARK-60129: keep locations of existing partitions") {
+ withNamespaceAndTable("ns", "tbl") { t =>
+ withTempDir { newRoot =>
+ sql(s"CREATE TABLE $t (id INT, p STRING) $defaultUsing PARTITIONED BY
(p)")
+ sql(s"INSERT INTO $t PARTITION (p='b') SELECT 1")
+ checkAnswer(sql(s"SELECT id, p FROM $t"), Row(1, "b"))
+
+ val partDir = new Path(newRoot.getCanonicalPath, "p=b")
+ val fs = partDir.getFileSystem(spark.sessionState.newHadoopConf())
+ fs.mkdirs(partDir)
+ sql(s"ALTER TABLE $t SET LOCATION '${newRoot.toURI}'")
+ sql(s"ALTER TABLE $t RECOVER PARTITIONS")
+
+ checkPartitions(t, Map("p" -> "b"))
+ checkAnswer(sql(s"SELECT id, p FROM $t"), Row(1, "b"))
+ }
+ }
+ }
+
test("ALTER TABLE .. RECOVER PARTITIONS is not allowed for non-partitioned
table") {
withTable("tbl") {
sql("CREATE TABLE tbl(col1 int, col2 string) USING parquet")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]