LuciferYang commented on code in PR #11226:
URL: https://github.com/apache/paimon/pull/11226#discussion_r4652369518
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
+ .asScala
+ .map(partition => (partition, DataConverter.fromPaimon(partition,
partitionRowType)))
Review Comment:
Reproduced with your data. Where they commit directly, DROP and TRUNCATE now
convert the selector to its Paimon type, with timestamps truncated to the
column's precision, and give the commit a predicate on the stored partition
values, so `p = '2024-06-01 00:00:00.123456'` removes only the `.123456000`
partition. The new test writes both partitions through the Java API and checks
that `.123456789` survives.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
+ .asScala
+ .map(partition => (partition, DataConverter.fromPaimon(partition,
partitionRowType)))
+ val storedByPrefix = rows
+ .map(_.numFields)
+ .distinct
+ .map {
+ numFields =>
+ numFields -> stored.groupBy { case (_, values) =>
partitionKey(values, numFields) }
+ }
+ .toMap
+ val partitions = rows.toSeq.flatMap {
+ row =>
+ val matched = storedByPrefix(row.numFields)
+ .getOrElse(partitionKey(row, row.numFields), Seq.empty)
+ .map(_._1)
+ if (matched.isEmpty) {
+ columnHoldingDefaultPartitionName(row).foreach {
+ name =>
+ throw new IllegalArgumentException(
+ s"No partition of table ${table.name()} has the default
partition name as the " +
+ s"value of partition column $name. If you meant the NULL
partition, use " +
+ s"$name = NULL.")
+ }
+ }
+ matched
+ }.distinct
+ if (partitions.nonEmpty) {
+ logInfo(
+ "Try to remove the data of partitions: " + partitions
+
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/",
200))
+ .mkString(","))
+ // A static overwrite with no data removes the data of the given
partitions.
+ val target =
+ fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key ->
"false").asJava)
+ val commit = target.newCommit(CoreOptions.createCommitUser(new
Options(target.options())))
+ try {
+ commit
+ .withOperation(Snapshot.Operation.TRUNCATE)
+ .withOverwriteStaticPartitions(partitions.asJava)
+ .commit(Collections.emptyList[CommitMessage]())
Review Comment:
The empty overwrite is gone. Where they commit directly, DROP and TRUNCATE
pass the typed predicate to a new internal
`TableCommitImpl.truncatePartitions(PartitionPredicate)`, which goes through
the same `FileStoreCommitImpl.dropPartitions` overwrite as the spec overload
and, like it, runs no maintenance. The new test sets your TTL options and
checks that `2020-01-01` survives both a TRUNCATE and a DROP of other
partitions.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
Review Comment:
The prefix predicate now goes into the commit, which matches it against the
snapshot each attempt commits on, as the spec path does, instead of a partition
list taken beforehand. `FileStoreCommitTest` checks that a `dt`-only predicate
also drops an `hr` partition the first commit did not write.
##########
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala:
##########
@@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType,
rowConverter(row).asInstanceOf[Row]))
}
+ /**
+ * The first partition column whose value in `spec` does not parse back to
its value in `row`. A
+ * spec holds each value as a string, and a blank one is written as the
default partition name,
+ * which parses back as NULL.
+ */
+ private def columnNotParsedBack(row: InternalRow, spec: JMap[String,
String]): Option[String] = {
+ val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
+ val specRowType = TypeUtils.project(table.rowType, names.asJava)
+ val parsed = InternalRowPartitionComputer.convertSpecToInternal(
+ spec,
+ specRowType,
+ CoreOptions.fromMap(table.options()).partitionDefaultName())
+ names.zipWithIndex
+ .find {
+ case (name, index) =>
+ val parsedValue =
+ DataConverter.fromPaimon(parsed.get(name),
specRowType.getField(name).`type`())
+ !Objects.deepEquals(row.get(index, partitionSchema(name).dataType),
parsedValue)
+ }
+ .map(_._1)
+ }
+
+ /** Whether the Spark value is a STRING or BINARY value equal to `text`. */
+ private def isText(value: Any, text: String): Boolean = value match {
+ case s: UTF8String => s.toString == text
+ case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
+ case _ => false
+ }
+
+ /** The first column of `row` whose value is the default partition name,
which is not NULL. */
+ private def columnHoldingDefaultPartitionName(row: InternalRow):
Option[String] = {
+ val defaultPartitionName =
CoreOptions.fromMap(table.options()).partitionDefaultName()
+ partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
+ case (field, i) if isText(row.get(i, field.dataType),
defaultPartitionName) => field.name
+ }
+ }
+
+ /**
+ * The first `numFields` values of a partition row, as a key that compares
them the way
+ * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by
content.
+ */
+ private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
+ val values = (0 until numFields).map {
+ i =>
+ row.get(i, partitionSchema.fields(i).dataType) match {
+ case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
+ case value => value
+ }
+ }
+ new JArrayList[AnyRef](values.asJava)
+ }
+
+ /** The specs of the partitions to drop by name; each must address its own
partition. */
+ private def toRemovedPaimonPartitions(rows: Array[InternalRow]):
Array[JMap[String, String]] = {
+ val specs = toPaimonPartitions(rows)
+ rows.zip(specs).foreach {
+ case (row, spec) =>
+ columnNotParsedBack(row, spec).foreach {
+ name =>
+ val nullPartitionHint = if
(columnHoldingDefaultPartitionName(row).contains(name)) {
+ s" If you meant the NULL partition, use $name = NULL."
+ } else {
+ ""
+ }
+ throw new IllegalArgumentException(
+ s"Partition spec $spec of table ${table.name()} does not name
the partition to " +
+ s"remove: the value of partition column $name does not parse
back from it." +
+ nullPartitionHint)
+ }
+ }
+ specs
+ }
+
+ /**
+ * Removes the data of the partitions in the latest snapshot that the rows
address, matched by
+ * value: a full row addresses its own partition, a prefix row every
partition under it.
+ */
+ private def truncateStoredPartitions(
+ fileStoreTable: FileStoreTable,
+ rows: Array[InternalRow]): Unit = {
+ if (rows.isEmpty) {
+ return
+ }
+ // A batch scan can hide files, such as level 0 of a deletion-vector
primary-key table; the
+ // snapshot reader lists every partition of the latest snapshot.
+ val stored = fileStoreTable
+ .newSnapshotReader()
+ .withMode(ScanMode.ALL)
+ .partitions()
+ .asScala
+ .map(partition => (partition, DataConverter.fromPaimon(partition,
partitionRowType)))
+ val storedByPrefix = rows
+ .map(_.numFields)
+ .distinct
+ .map {
+ numFields =>
+ numFields -> stored.groupBy { case (_, values) =>
partitionKey(values, numFields) }
+ }
+ .toMap
+ val partitions = rows.toSeq.flatMap {
+ row =>
+ val matched = storedByPrefix(row.numFields)
+ .getOrElse(partitionKey(row, row.numFields), Seq.empty)
+ .map(_._1)
+ if (matched.isEmpty) {
+ columnHoldingDefaultPartitionName(row).foreach {
+ name =>
+ throw new IllegalArgumentException(
+ s"No partition of table ${table.name()} has the default
partition name as the " +
+ s"value of partition column $name. If you meant the NULL
partition, use " +
+ s"$name = NULL.")
+ }
+ }
+ matched
+ }.distinct
+ if (partitions.nonEmpty) {
+ logInfo(
+ "Try to remove the data of partitions: " + partitions
+
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/",
200))
+ .mkString(","))
+ // A static overwrite with no data removes the data of the given
partitions.
+ val target =
+ fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key ->
"false").asJava)
+ val commit = target.newCommit(CoreOptions.createCommitUser(new
Options(target.options())))
+ try {
+ commit
+ .withOperation(Snapshot.Operation.TRUNCATE)
+ .withOverwriteStaticPartitions(partitions.asJava)
+ .commit(Collections.emptyList[CommitMessage]())
+ } finally {
+ commit.close()
+ }
+ }
+ }
+
override def dropPartitions(rows: Array[InternalRow]): Boolean = {
table match {
case fileStoreTable: FileStoreTable =>
- val partitions = toPaimonPartitions(rows).toSeq.asJava
- logInfo("Try to drop partitions: " + partitions.asScala.mkString(","))
- val partitionModification =
fileStoreTable.catalogEnvironment().partitionModification()
+ // The catalog drops the partitions it keeps in a metastore by name.
+ val options = fileStoreTable.coreOptions()
+ val partitionModification =
+ if (options.partitionedTableInMetastore() ||
options.tagToPartitionField() != null) {
Review Comment:
Restored: any non-null `partitionModification()` gets the DROP again. A new
test gives a table a partition handler without the Hive options and checks the
DROP reaches it. That handler takes names, so a value whose name does not parse
back, such as a blank string, is rejected before anything is dropped; the Hive
test checks this with and without `metastore.partitioned-table`. A typed drop
would avoid the rejection, but it needs a new method on the public `Catalog`
interface; I'd keep that for a separate change unless you'd rather have it here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]