JingsongLi commented on code in PR #11226:
URL: https://github.com/apache/paimon/pull/11226#discussion_r4643567782


##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends 
SupportsAtomicPartitionManagement with L
       new SparkRow(currentPartitionRowType, 
rowConverter(row).asInstanceOf[Row]))
   }
 
+  /**
+   * The first partition column whose value in `spec` does not parse back to 
its value in `row`. A
+   * spec holds each value as a string, and a blank one is written as the 
default partition name,
+   * which parses back as NULL.
+   */
+  private def columnNotParsedBack(row: InternalRow, spec: JMap[String, 
String]): Option[String] = {
+    val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+    val specRowType = TypeUtils.project(table.rowType, names.asJava)
+    val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+      spec,
+      specRowType,
+      CoreOptions.fromMap(table.options()).partitionDefaultName())
+    names.zipWithIndex
+      .find {
+        case (name, index) =>
+          val parsedValue =
+            DataConverter.fromPaimon(parsed.get(name), 
specRowType.getField(name).`type`())
+          !Objects.deepEquals(row.get(index, partitionSchema(name).dataType), 
parsedValue)
+      }
+      .map(_._1)
+  }
+
+  /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+  private def isText(value: Any, text: String): Boolean = value match {
+    case s: UTF8String => s.toString == text
+    case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+    case _ => false
+  }
+
+  /** The first column of `row` whose value is the default partition name, 
which is not NULL. */
+  private def columnHoldingDefaultPartitionName(row: InternalRow): 
Option[String] = {
+    val defaultPartitionName = 
CoreOptions.fromMap(table.options()).partitionDefaultName()
+    partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+      case (field, i) if isText(row.get(i, field.dataType), 
defaultPartitionName) => field.name
+    }
+  }
+
+  /**
+   * The first `numFields` values of a partition row, as a key that compares 
them the way
+   * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by 
content.
+   */
+  private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+    val values = (0 until numFields).map {
+      i =>
+        row.get(i, partitionSchema.fields(i).dataType) match {
+          case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+          case value => value
+        }
+    }
+    new JArrayList[AnyRef](values.asJava)
+  }
+
+  /** The specs of the partitions to drop by name; each must address its own 
partition. */
+  private def toRemovedPaimonPartitions(rows: Array[InternalRow]): 
Array[JMap[String, String]] = {
+    val specs = toPaimonPartitions(rows)
+    rows.zip(specs).foreach {
+      case (row, spec) =>
+        columnNotParsedBack(row, spec).foreach {
+          name =>
+            val nullPartitionHint = if 
(columnHoldingDefaultPartitionName(row).contains(name)) {
+              s" If you meant the NULL partition, use $name = NULL."
+            } else {
+              ""
+            }
+            throw new IllegalArgumentException(
+              s"Partition spec $spec of table ${table.name()} does not name 
the partition to " +
+                s"remove: the value of partition column $name does not parse 
back from it." +
+                nullPartitionHint)
+        }
+    }
+    specs
+  }
+
+  /**
+   * Removes the data of the partitions in the latest snapshot that the rows 
address, matched by
+   * value: a full row addresses its own partition, a prefix row every 
partition under it.
+   */
+  private def truncateStoredPartitions(
+      fileStoreTable: FileStoreTable,
+      rows: Array[InternalRow]): Unit = {
+    if (rows.isEmpty) {
+      return
+    }
+    // A batch scan can hide files, such as level 0 of a deletion-vector 
primary-key table; the
+    // snapshot reader lists every partition of the latest snapshot.
+    val stored = fileStoreTable
+      .newSnapshotReader()
+      .withMode(ScanMode.ALL)
+      .partitions()

Review Comment:
   **[P2] Match a partial DROP against the snapshot being committed**
   
   I exercised the Spark partition API used by `ALTER TABLE ... DROP 
PARTITION(p1='x')`. Starting with `x/a`, I committed an append to both `x/a` 
and new `x/b` immediately after `.partitions()` returned. The DROP removed all 
of `x/a` including the new row, and left `x/b`. With the append committed 
before the baseline DROP evaluates its predicate, both are removed. If the DROP 
were first, both newly written rows would survive. Keep the typed prefix 
predicate through the core commit and retry so it matches each candidate 
snapshot.



##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends 
SupportsAtomicPartitionManagement with L
       new SparkRow(currentPartitionRowType, 
rowConverter(row).asInstanceOf[Row]))
   }
 
+  /**
+   * The first partition column whose value in `spec` does not parse back to 
its value in `row`. A
+   * spec holds each value as a string, and a blank one is written as the 
default partition name,
+   * which parses back as NULL.
+   */
+  private def columnNotParsedBack(row: InternalRow, spec: JMap[String, 
String]): Option[String] = {
+    val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+    val specRowType = TypeUtils.project(table.rowType, names.asJava)
+    val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+      spec,
+      specRowType,
+      CoreOptions.fromMap(table.options()).partitionDefaultName())
+    names.zipWithIndex
+      .find {
+        case (name, index) =>
+          val parsedValue =
+            DataConverter.fromPaimon(parsed.get(name), 
specRowType.getField(name).`type`())
+          !Objects.deepEquals(row.get(index, partitionSchema(name).dataType), 
parsedValue)
+      }
+      .map(_._1)
+  }
+
+  /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+  private def isText(value: Any, text: String): Boolean = value match {
+    case s: UTF8String => s.toString == text
+    case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+    case _ => false
+  }
+
+  /** The first column of `row` whose value is the default partition name, 
which is not NULL. */
+  private def columnHoldingDefaultPartitionName(row: InternalRow): 
Option[String] = {
+    val defaultPartitionName = 
CoreOptions.fromMap(table.options()).partitionDefaultName()
+    partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+      case (field, i) if isText(row.get(i, field.dataType), 
defaultPartitionName) => field.name
+    }
+  }
+
+  /**
+   * The first `numFields` values of a partition row, as a key that compares 
them the way
+   * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by 
content.
+   */
+  private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+    val values = (0 until numFields).map {
+      i =>
+        row.get(i, partitionSchema.fields(i).dataType) match {
+          case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+          case value => value
+        }
+    }
+    new JArrayList[AnyRef](values.asJava)
+  }
+
+  /** The specs of the partitions to drop by name; each must address its own 
partition. */
+  private def toRemovedPaimonPartitions(rows: Array[InternalRow]): 
Array[JMap[String, String]] = {
+    val specs = toPaimonPartitions(rows)
+    rows.zip(specs).foreach {
+      case (row, spec) =>
+        columnNotParsedBack(row, spec).foreach {
+          name =>
+            val nullPartitionHint = if 
(columnHoldingDefaultPartitionName(row).contains(name)) {
+              s" If you meant the NULL partition, use $name = NULL."
+            } else {
+              ""
+            }
+            throw new IllegalArgumentException(
+              s"Partition spec $spec of table ${table.name()} does not name 
the partition to " +
+                s"remove: the value of partition column $name does not parse 
back from it." +
+                nullPartitionHint)
+        }
+    }
+    specs
+  }
+
+  /**
+   * Removes the data of the partitions in the latest snapshot that the rows 
address, matched by
+   * value: a full row addresses its own partition, a prefix row every 
partition under it.
+   */
+  private def truncateStoredPartitions(
+      fileStoreTable: FileStoreTable,
+      rows: Array[InternalRow]): Unit = {
+    if (rows.isEmpty) {
+      return
+    }
+    // A batch scan can hide files, such as level 0 of a deletion-vector 
primary-key table; the
+    // snapshot reader lists every partition of the latest snapshot.
+    val stored = fileStoreTable
+      .newSnapshotReader()
+      .withMode(ScanMode.ALL)
+      .partitions()
+      .asScala
+      .map(partition => (partition, DataConverter.fromPaimon(partition, 
partitionRowType)))
+    val storedByPrefix = rows
+      .map(_.numFields)
+      .distinct
+      .map {
+        numFields =>
+          numFields -> stored.groupBy { case (_, values) => 
partitionKey(values, numFields) }
+      }
+      .toMap
+    val partitions = rows.toSeq.flatMap {
+      row =>
+        val matched = storedByPrefix(row.numFields)
+          .getOrElse(partitionKey(row, row.numFields), Seq.empty)
+          .map(_._1)
+        if (matched.isEmpty) {
+          columnHoldingDefaultPartitionName(row).foreach {
+            name =>
+              throw new IllegalArgumentException(
+                s"No partition of table ${table.name()} has the default 
partition name as the " +
+                  s"value of partition column $name. If you meant the NULL 
partition, use " +
+                  s"$name = NULL.")
+          }
+        }
+        matched
+    }.distinct
+    if (partitions.nonEmpty) {
+      logInfo(
+        "Try to remove the data of partitions: " + partitions
+          
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/", 
200))
+          .mkString(","))
+      // A static overwrite with no data removes the data of the given 
partitions.
+      val target =
+        fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key -> 
"false").asJava)
+      val commit = target.newCommit(CoreOptions.createCommitUser(new 
Options(target.options())))
+      try {
+        commit
+          .withOperation(Snapshot.Operation.TRUNCATE)
+          .withOverwriteStaticPartitions(partitions.asJava)
+          .commit(Collections.emptyList[CommitMessage]())
+      } finally {
+        commit.close()
+      }
+    }
+  }
+
   override def dropPartitions(rows: Array[InternalRow]): Boolean = {
     table match {
       case fileStoreTable: FileStoreTable =>
-        val partitions = toPaimonPartitions(rows).toSeq.asJava
-        logInfo("Try to drop partitions: " + partitions.asScala.mkString(","))
-        val partitionModification = 
fileStoreTable.catalogEnvironment().partitionModification()
+        // The catalog drops the partitions it keeps in a metastore by name.
+        val options = fileStoreTable.coreOptions()
+        val partitionModification =
+          if (options.partitionedTableInMetastore() || 
options.tagToPartitionField() != null) {

Review Comment:
   **[P2] Preserve the advertised catalog partition modification callback**
   
   Spark previously honored any non-null 
`CatalogEnvironment.partitionModification()` for DROP. A custom catalog can 
implement `Catalog.supportsPartitionModification()` and propagate logical 
partition drops to a mirror even when `metastore.partitioned-table` and 
`tag-to-partition-field` are unset. Both flags now default to the direct commit 
path, so the override silently stops receiving DROP and the mirror can diverge. 
Please preserve dispatch to the catalog or give catalogs an explicit typed 
partition-drop capability, with a test using a catalog that advertises the 
capability without these Hive options.



##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends 
SupportsAtomicPartitionManagement with L
       new SparkRow(currentPartitionRowType, 
rowConverter(row).asInstanceOf[Row]))
   }
 
+  /**
+   * The first partition column whose value in `spec` does not parse back to 
its value in `row`. A
+   * spec holds each value as a string, and a blank one is written as the 
default partition name,
+   * which parses back as NULL.
+   */
+  private def columnNotParsedBack(row: InternalRow, spec: JMap[String, 
String]): Option[String] = {
+    val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+    val specRowType = TypeUtils.project(table.rowType, names.asJava)
+    val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+      spec,
+      specRowType,
+      CoreOptions.fromMap(table.options()).partitionDefaultName())
+    names.zipWithIndex
+      .find {
+        case (name, index) =>
+          val parsedValue =
+            DataConverter.fromPaimon(parsed.get(name), 
specRowType.getField(name).`type`())
+          !Objects.deepEquals(row.get(index, partitionSchema(name).dataType), 
parsedValue)
+      }
+      .map(_._1)
+  }
+
+  /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+  private def isText(value: Any, text: String): Boolean = value match {
+    case s: UTF8String => s.toString == text
+    case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+    case _ => false
+  }
+
+  /** The first column of `row` whose value is the default partition name, 
which is not NULL. */
+  private def columnHoldingDefaultPartitionName(row: InternalRow): 
Option[String] = {
+    val defaultPartitionName = 
CoreOptions.fromMap(table.options()).partitionDefaultName()
+    partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+      case (field, i) if isText(row.get(i, field.dataType), 
defaultPartitionName) => field.name
+    }
+  }
+
+  /**
+   * The first `numFields` values of a partition row, as a key that compares 
them the way
+   * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by 
content.
+   */
+  private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+    val values = (0 until numFields).map {
+      i =>
+        row.get(i, partitionSchema.fields(i).dataType) match {
+          case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+          case value => value
+        }
+    }
+    new JArrayList[AnyRef](values.asJava)
+  }
+
+  /** The specs of the partitions to drop by name; each must address its own 
partition. */
+  private def toRemovedPaimonPartitions(rows: Array[InternalRow]): 
Array[JMap[String, String]] = {
+    val specs = toPaimonPartitions(rows)
+    rows.zip(specs).foreach {
+      case (row, spec) =>
+        columnNotParsedBack(row, spec).foreach {
+          name =>
+            val nullPartitionHint = if 
(columnHoldingDefaultPartitionName(row).contains(name)) {
+              s" If you meant the NULL partition, use $name = NULL."
+            } else {
+              ""
+            }
+            throw new IllegalArgumentException(
+              s"Partition spec $spec of table ${table.name()} does not name 
the partition to " +
+                s"remove: the value of partition column $name does not parse 
back from it." +
+                nullPartitionHint)
+        }
+    }
+    specs
+  }
+
+  /**
+   * Removes the data of the partitions in the latest snapshot that the rows 
address, matched by
+   * value: a full row addresses its own partition, a prefix row every 
partition under it.
+   */
+  private def truncateStoredPartitions(
+      fileStoreTable: FileStoreTable,
+      rows: Array[InternalRow]): Unit = {
+    if (rows.isEmpty) {
+      return
+    }
+    // A batch scan can hide files, such as level 0 of a deletion-vector 
primary-key table; the
+    // snapshot reader lists every partition of the latest snapshot.
+    val stored = fileStoreTable
+      .newSnapshotReader()
+      .withMode(ScanMode.ALL)
+      .partitions()
+      .asScala
+      .map(partition => (partition, DataConverter.fromPaimon(partition, 
partitionRowType)))

Review Comment:
   **[P1] Preserve partition timestamp precision before matching a DROP**
   
   I used the Java writer to store two distinct partitions in a Parquet table 
with a TIMESTAMP(9) partition key: `2024-06-01 00:00:00.123456000` and 
`2024-06-01 00:00:00.123456789`. With `partition.legacy-name=false` and UTC, 
Spark 3.5.9 `ALTER TABLE t DROP PARTITION (p='2024-06-01 00:00:00.123456')` 
leaves the second partition on master and removes both on this head. Converting 
each stored key to Spark discards the sub-microsecond digits before grouping. 
Convert the SQL selector to its Paimon type, then match against the original 
partition values at stored precision.



##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends 
SupportsAtomicPartitionManagement with L
       new SparkRow(currentPartitionRowType, 
rowConverter(row).asInstanceOf[Row]))
   }
 
+  /**
+   * The first partition column whose value in `spec` does not parse back to 
its value in `row`. A
+   * spec holds each value as a string, and a blank one is written as the 
default partition name,
+   * which parses back as NULL.
+   */
+  private def columnNotParsedBack(row: InternalRow, spec: JMap[String, 
String]): Option[String] = {
+    val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+    val specRowType = TypeUtils.project(table.rowType, names.asJava)
+    val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+      spec,
+      specRowType,
+      CoreOptions.fromMap(table.options()).partitionDefaultName())
+    names.zipWithIndex
+      .find {
+        case (name, index) =>
+          val parsedValue =
+            DataConverter.fromPaimon(parsed.get(name), 
specRowType.getField(name).`type`())
+          !Objects.deepEquals(row.get(index, partitionSchema(name).dataType), 
parsedValue)
+      }
+      .map(_._1)
+  }
+
+  /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+  private def isText(value: Any, text: String): Boolean = value match {
+    case s: UTF8String => s.toString == text
+    case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+    case _ => false
+  }
+
+  /** The first column of `row` whose value is the default partition name, 
which is not NULL. */
+  private def columnHoldingDefaultPartitionName(row: InternalRow): 
Option[String] = {
+    val defaultPartitionName = 
CoreOptions.fromMap(table.options()).partitionDefaultName()
+    partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+      case (field, i) if isText(row.get(i, field.dataType), 
defaultPartitionName) => field.name
+    }
+  }
+
+  /**
+   * The first `numFields` values of a partition row, as a key that compares 
them the way
+   * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by 
content.
+   */
+  private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+    val values = (0 until numFields).map {
+      i =>
+        row.get(i, partitionSchema.fields(i).dataType) match {
+          case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+          case value => value
+        }
+    }
+    new JArrayList[AnyRef](values.asJava)
+  }
+
+  /** The specs of the partitions to drop by name; each must address its own 
partition. */
+  private def toRemovedPaimonPartitions(rows: Array[InternalRow]): 
Array[JMap[String, String]] = {
+    val specs = toPaimonPartitions(rows)
+    rows.zip(specs).foreach {
+      case (row, spec) =>
+        columnNotParsedBack(row, spec).foreach {
+          name =>
+            val nullPartitionHint = if 
(columnHoldingDefaultPartitionName(row).contains(name)) {
+              s" If you meant the NULL partition, use $name = NULL."
+            } else {
+              ""
+            }
+            throw new IllegalArgumentException(
+              s"Partition spec $spec of table ${table.name()} does not name 
the partition to " +
+                s"remove: the value of partition column $name does not parse 
back from it." +
+                nullPartitionHint)
+        }
+    }
+    specs
+  }
+
+  /**
+   * Removes the data of the partitions in the latest snapshot that the rows 
address, matched by
+   * value: a full row addresses its own partition, a prefix row every 
partition under it.
+   */
+  private def truncateStoredPartitions(
+      fileStoreTable: FileStoreTable,
+      rows: Array[InternalRow]): Unit = {
+    if (rows.isEmpty) {
+      return
+    }
+    // A batch scan can hide files, such as level 0 of a deletion-vector 
primary-key table; the
+    // snapshot reader lists every partition of the latest snapshot.
+    val stored = fileStoreTable
+      .newSnapshotReader()
+      .withMode(ScanMode.ALL)
+      .partitions()
+      .asScala
+      .map(partition => (partition, DataConverter.fromPaimon(partition, 
partitionRowType)))
+    val storedByPrefix = rows
+      .map(_.numFields)
+      .distinct
+      .map {
+        numFields =>
+          numFields -> stored.groupBy { case (_, values) => 
partitionKey(values, numFields) }
+      }
+      .toMap
+    val partitions = rows.toSeq.flatMap {
+      row =>
+        val matched = storedByPrefix(row.numFields)
+          .getOrElse(partitionKey(row, row.numFields), Seq.empty)
+          .map(_._1)
+        if (matched.isEmpty) {
+          columnHoldingDefaultPartitionName(row).foreach {
+            name =>
+              throw new IllegalArgumentException(
+                s"No partition of table ${table.name()} has the default 
partition name as the " +
+                  s"value of partition column $name. If you meant the NULL 
partition, use " +
+                  s"$name = NULL.")
+          }
+        }
+        matched
+    }.distinct
+    if (partitions.nonEmpty) {
+      logInfo(
+        "Try to remove the data of partitions: " + partitions
+          
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/", 
200))
+          .mkString(","))
+      // A static overwrite with no data removes the data of the given 
partitions.
+      val target =
+        fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key -> 
"false").asJava)
+      val commit = target.newCommit(CoreOptions.createCommitUser(new 
Options(target.options())))
+      try {
+        commit
+          .withOperation(Snapshot.Operation.TRUNCATE)
+          .withOverwriteStaticPartitions(partitions.asJava)
+          .commit(Collections.emptyList[CommitMessage]())

Review Comment:
   **[P1] Preserve the scope of explicit partition DROP and TRUNCATE**
   
   I inserted partitions `2020-01-01` and `2099-01-01`, enabled a seven-day 
partition TTL, then explicitly targeted only `2099-01-01`. Both Spark 3.5.9 
`ALTER TABLE ... DROP PARTITION` and `TRUNCATE TABLE ... PARTITION` preserve 
the other partition on master. Both leave the table empty on this head. The 
empty overwrite invokes table-wide partition expiration through 
`TableCommitImpl.maintain`. Please add a typed drop path that retains existing 
DDL maintenance semantics, and assert that a non-target partition survives both 
operations.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to