JingsongLi commented on code in PR #11226:
URL: https://github.com/apache/paimon/pull/11226#discussion_r4643567782
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
Review Comment:
**[P2] Match a partial DROP against the snapshot being committed**
I exercised the Spark partition API used by `ALTER TABLE ... DROP
PARTITION(p1='x')`. Starting with `x/a`, I committed an append to both `x/a`
and new `x/b` immediately after `.partitions()` returned. The DROP removed all
of `x/a` including the new row, and left `x/b`. With the append committed
before the baseline DROP evaluates its predicate, both are removed. If the DROP
were first, both newly written rows would survive. Keep the typed prefix
predicate through the core commit and retry so it matches each candidate
snapshot.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
+ .asScala
+ .map(partition => (partition, DataConverter.fromPaimon(partition,
partitionRowType)))
+ val storedByPrefix = rows
+ .map(_.numFields)
+ .distinct
+ .map {
+ numFields =>
+ numFields -> stored.groupBy { case (_, values) =>
partitionKey(values, numFields) }
+ }
+ .toMap
+ val partitions = rows.toSeq.flatMap {
+ row =>
+ val matched = storedByPrefix(row.numFields)
+ .getOrElse(partitionKey(row, row.numFields), Seq.empty)
+ .map(_._1)
+ if (matched.isEmpty) {
+ columnHoldingDefaultPartitionName(row).foreach {
+ name =>
+ throw new IllegalArgumentException(
+ s"No partition of table ${table.name()} has the default
partition name as the " +
+ s"value of partition column $name. If you meant the NULL
partition, use " +
+ s"$name = NULL.")
+ }
+ }
+ matched
+ }.distinct
+ if (partitions.nonEmpty) {
+ logInfo(
+ "Try to remove the data of partitions: " + partitions
+
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/",
200))
+ .mkString(","))
+ // A static overwrite with no data removes the data of the given
partitions.
+ val target =
+ fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key ->
"false").asJava)
+ val commit = target.newCommit(CoreOptions.createCommitUser(new
Options(target.options())))
+ try {
+ commit
+ .withOperation(Snapshot.Operation.TRUNCATE)
+ .withOverwriteStaticPartitions(partitions.asJava)
+ .commit(Collections.emptyList[CommitMessage]())
+ } finally {
+ commit.close()
+ }
+ }
+ }
+
override def dropPartitions(rows: Array[InternalRow]): Boolean = {
table match {
case fileStoreTable: FileStoreTable =>
- val partitions = toPaimonPartitions(rows).toSeq.asJava
- logInfo("Try to drop partitions: " + partitions.asScala.mkString(","))
- val partitionModification =
fileStoreTable.catalogEnvironment().partitionModification()
+ // The catalog drops the partitions it keeps in a metastore by name.
+ val options = fileStoreTable.coreOptions()
+ val partitionModification =
+ if (options.partitionedTableInMetastore() ||
options.tagToPartitionField() != null) {
Review Comment:
**[P2] Preserve the advertised catalog partition modification callback**
Spark previously honored any non-null
`CatalogEnvironment.partitionModification()` for DROP. A custom catalog can
implement `Catalog.supportsPartitionModification()` and propagate logical
partition drops to a mirror even when `metastore.partitioned-table` and
`tag-to-partition-field` are unset. Both flags now default to the direct commit
path, so the override silently stops receiving DROP and the mirror can diverge.
Please preserve dispatch to the catalog or give catalogs an explicit typed
partition-drop capability, with a test using a catalog that advertises the
capability without these Hive options.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
+ .asScala
+ .map(partition => (partition, DataConverter.fromPaimon(partition,
partitionRowType)))
Review Comment:
**[P1] Preserve partition timestamp precision before matching a DROP**
I used the Java writer to store two distinct partitions in a Parquet table
with a TIMESTAMP(9) partition key: `2024-06-01 00:00:00.123456000` and
`2024-06-01 00:00:00.123456789`. With `partition.legacy-name=false` and UTC,
Spark 3.5.9 `ALTER TABLE t DROP PARTITION (p='2024-06-01 00:00:00.123456')`
leaves the second partition on master and removes both on this head. Converting
each stored key to Spark discards the sub-microsecond digits before grouping.
Convert the SQL selector to its Paimon type, then match against the original
partition values at stored precision.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
+ .asScala
+ .map(partition => (partition, DataConverter.fromPaimon(partition,
partitionRowType)))
+ val storedByPrefix = rows
+ .map(_.numFields)
+ .distinct
+ .map {
+ numFields =>
+ numFields -> stored.groupBy { case (_, values) =>
partitionKey(values, numFields) }
+ }
+ .toMap
+ val partitions = rows.toSeq.flatMap {
+ row =>
+ val matched = storedByPrefix(row.numFields)
+ .getOrElse(partitionKey(row, row.numFields), Seq.empty)
+ .map(_._1)
+ if (matched.isEmpty) {
+ columnHoldingDefaultPartitionName(row).foreach {
+ name =>
+ throw new IllegalArgumentException(
+ s"No partition of table ${table.name()} has the default
partition name as the " +
+ s"value of partition column $name. If you meant the NULL
partition, use " +
+ s"$name = NULL.")
+ }
+ }
+ matched
+ }.distinct
+ if (partitions.nonEmpty) {
+ logInfo(
+ "Try to remove the data of partitions: " + partitions
+
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/",
200))
+ .mkString(","))
+ // A static overwrite with no data removes the data of the given
partitions.
+ val target =
+ fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key ->
"false").asJava)
+ val commit = target.newCommit(CoreOptions.createCommitUser(new
Options(target.options())))
+ try {
+ commit
+ .withOperation(Snapshot.Operation.TRUNCATE)
+ .withOverwriteStaticPartitions(partitions.asJava)
+ .commit(Collections.emptyList[CommitMessage]())
Review Comment:
**[P1] Preserve the scope of explicit partition DROP and TRUNCATE**
I inserted partitions `2020-01-01` and `2099-01-01`, enabled a seven-day
partition TTL, then explicitly targeted only `2099-01-01`. Both Spark 3.5.9
`ALTER TABLE ... DROP PARTITION` and `TRUNCATE TABLE ... PARTITION` preserve
the other partition on master. Both leave the table empty on this head. The
empty overwrite invokes table-wide partition expiration through
`TableCommitImpl.maintain`. Please add a typed drop path that retains existing
DDL maintenance semantics, and assert that a non-target partition survives both
operations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]