JingsongLi commented on code in PR #11272: URL: https://github.com/apache/paimon/pull/11272#discussion_r4643327158
########## paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java: ########## @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.append.dataevolution; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.serializer.VersionedSerializer; +import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; +import org.apache.paimon.deletionvectors.DeletionVector; +import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexFileMetaSerializer; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.io.DataOutputSerializer; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.DeletionFile; +import org.apache.paimon.utils.Range; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.TreeMap; + +import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET; +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; +import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow; +import static org.apache.paimon.utils.SerializationUtils.readCount; +import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow; + +/** + * A partition-local batch of complete anchor rewrites and uniquely owned old index files. + * + * <p>New anchor bitmaps share rolling output files. Retained entries are written separately for + * each old index, preserving the conflict boundaries of subsequent deletes. + */ +public class DataEvolutionDeletionVectorRewriteTask { + + final BinaryRow partition; + final List<Rewrite> rewrites = new ArrayList<>(); + final List<IndexRewrite> indexes = new ArrayList<>(); + + DataEvolutionDeletionVectorRewriteTask(BinaryRow partition) { + this.partition = partition; + } + + public BinaryRow partition() { + return partition; + } + + public int workCount() { + return rewrites.size() + indexes.size(); + } + + public CommitMessage execute(FileStoreTable table) { + DeletionVectorsIndexFile index = + table.store().newIndexFileHandler().dvIndex(partition, UNAWARE_BUCKET); + boolean bitmap64 = table.coreOptions().deletionVectorBitmap64(); + Iterator<Map.Entry<String, DeletionVector>> bitmaps = + new Iterator<Map.Entry<String, DeletionVector>>() { + private int rewritePosition; + private Iterator<Map.Entry<String, DeletionVector>> current = + Collections.emptyIterator(); + + @Override + public boolean hasNext() { + while (!current.hasNext()) { + if (rewritePosition < rewrites.size()) { + current = + rewrite(rewrites.get(rewritePosition++), index, bitmap64) + .entrySet() + .iterator(); + } else { + return false; + } + } + return true; + } + + @Override + public Map.Entry<String, DeletionVector> next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return current.next(); + } + }; + List<IndexFileMeta> added = new ArrayList<>(); + try { + added.addAll(index.writeWithRolling(bitmaps)); + for (IndexRewrite old : indexes) { + Map<String, DeletionFile> retained = + index.toDeletionFiles(Collections.singletonList(old.file)); + old.removed.forEach(retained::remove); + added.addAll(index.writeWithRolling(index.readDeletionVector(retained))); + } + } catch (RuntimeException failure) { + for (IndexFileMeta file : added) { + try { + table.fileIO().deleteQuietly(index.path(file)); + } catch (RuntimeException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } + throw failure; + } + List<IndexFileMeta> deleted = new ArrayList<>(indexes.size()); + indexes.forEach(old -> deleted.add(old.file)); + return new CommitMessageImpl( + partition, + UNAWARE_BUCKET, + null, + DataIncrement.emptyIncrement(), + new CompactIncrement( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + added, + deleted)); + } + + private static Map<String, DeletionVector> rewrite( + Rewrite rewrite, DeletionVectorsIndexFile index, boolean bitmap64) { + Map<String, Map<String, DeletionFile>> byIndex = new LinkedHashMap<>(); + Map<String, Range> inputRanges = new LinkedHashMap<>(); + for (Source source : rewrite.sources) { + byIndex.computeIfAbsent(source.deletion.path(), ignored -> new LinkedHashMap<>()) + .put(source.anchor, source.deletion); + inputRanges.put(source.anchor, source.range); + } + TreeMap<Long, Map.Entry<String, Range>> outputs = new TreeMap<>(); + rewrite.outputs.entrySet().forEach(output -> outputs.put(output.getValue().from, output)); + Map<String, DeletionVector> result = new LinkedHashMap<>(); + for (Map<String, DeletionFile> inputs : byIndex.values()) { + index.readDeletionVector(inputs) + .forEach( + (anchor, bitmap) -> + move( + bitmap, + inputRanges.get(anchor), + outputs, + result, + bitmap64)); + } + return result; + } + + static void move( + DeletionVector bitmap, + Range input, + TreeMap<Long, Map.Entry<String, Range>> outputs, + Map<String, DeletionVector> result, + boolean bitmap64) { + if (bitmap.isEmpty()) { + return; + } + Map.Entry<String, Range> first = outputs.floorEntry(input.from).getValue(); + if (input.equals(first.getValue())) { Review Comment: **[P2] Respect the configured bitmap format on equal-range rewrites.** I wrote two rows to a Data Evolution append table, deleted one with the default 32-bit DV, and set `deletion-vectors.bitmap64=true`. Equal-range compaction committed and the surviving row remained readable. A subsequent SQL DELETE on that row failed with `Only instance with the same class type can be merged`. The new index reused the old bitmap even though the output writer was configured for 64-bit vectors. Reuse is safe when the formats match. For a mismatch, convert by position or reject the rewrite before publishing the new files. ########## docs/docs/multimodal-table/data-evolution-maintenance.mdx: ########## @@ -89,6 +89,20 @@ Ordinary compaction **does not physically remove rows hidden by deletion vectors**. Removing their positions would change the alignment with untouched column files. +### Recovering Files After a Failed Rewrite + +Flink and Spark rewrite DV indexes on workers. A failed worker attempt cleans +up the new DV files it has tracked. Files from tasks that already succeeded, +or from a lost worker process, can remain unreferenced if the overall rewrite +fails before committing. + +After the failed job and its retries have stopped, use +[Remove Orphan Files](../maintenance/manage-snapshots#remove-orphan-files) Review Comment: **[P2] Cover recovery when DV indexes use an external root.** I reran the failed distributed rewrite fixture with `global-index.external-path` set to a separate local directory. The previous snapshot stayed current and the completed Spark task left one external DV index behind. Running the orphan cleaner with a cutoff after the failure removed zero external files. The local and Spark cleaners share `OrphanFilesClean.listPaimonFileDirs`, which scans the table index directory and external data directories but omits this external index root. Provide a recovery path that can identify this table's outputs safely, or document the limitation and how to reclaim those files. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
