JingsongLi commented on code in PR #11272:
URL: https://github.com/apache/paimon/pull/11272#discussion_r4643327158


##########
paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java:
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append.dataevolution;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.serializer.VersionedSerializer;
+import org.apache.paimon.deletionvectors.Bitmap64DeletionVector;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexFileMetaSerializer;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.io.DataInputDeserializer;
+import org.apache.paimon.io.DataOutputSerializer;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.source.DeletionFile;
+import org.apache.paimon.utils.Range;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TreeMap;
+
+import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.Preconditions.checkState;
+import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
+import static org.apache.paimon.utils.SerializationUtils.readCount;
+import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
+
+/**
+ * A partition-local batch of complete anchor rewrites and uniquely owned old 
index files.
+ *
+ * <p>New anchor bitmaps share rolling output files. Retained entries are 
written separately for
+ * each old index, preserving the conflict boundaries of subsequent deletes.
+ */
+public class DataEvolutionDeletionVectorRewriteTask {
+
+    final BinaryRow partition;
+    final List<Rewrite> rewrites = new ArrayList<>();
+    final List<IndexRewrite> indexes = new ArrayList<>();
+
+    DataEvolutionDeletionVectorRewriteTask(BinaryRow partition) {
+        this.partition = partition;
+    }
+
+    public BinaryRow partition() {
+        return partition;
+    }
+
+    public int workCount() {
+        return rewrites.size() + indexes.size();
+    }
+
+    public CommitMessage execute(FileStoreTable table) {
+        DeletionVectorsIndexFile index =
+                table.store().newIndexFileHandler().dvIndex(partition, 
UNAWARE_BUCKET);
+        boolean bitmap64 = table.coreOptions().deletionVectorBitmap64();
+        Iterator<Map.Entry<String, DeletionVector>> bitmaps =
+                new Iterator<Map.Entry<String, DeletionVector>>() {
+                    private int rewritePosition;
+                    private Iterator<Map.Entry<String, DeletionVector>> 
current =
+                            Collections.emptyIterator();
+
+                    @Override
+                    public boolean hasNext() {
+                        while (!current.hasNext()) {
+                            if (rewritePosition < rewrites.size()) {
+                                current =
+                                        
rewrite(rewrites.get(rewritePosition++), index, bitmap64)
+                                                .entrySet()
+                                                .iterator();
+                            } else {
+                                return false;
+                            }
+                        }
+                        return true;
+                    }
+
+                    @Override
+                    public Map.Entry<String, DeletionVector> next() {
+                        if (!hasNext()) {
+                            throw new NoSuchElementException();
+                        }
+                        return current.next();
+                    }
+                };
+        List<IndexFileMeta> added = new ArrayList<>();
+        try {
+            added.addAll(index.writeWithRolling(bitmaps));
+            for (IndexRewrite old : indexes) {
+                Map<String, DeletionFile> retained =
+                        
index.toDeletionFiles(Collections.singletonList(old.file));
+                old.removed.forEach(retained::remove);
+                
added.addAll(index.writeWithRolling(index.readDeletionVector(retained)));
+            }
+        } catch (RuntimeException failure) {
+            for (IndexFileMeta file : added) {
+                try {
+                    table.fileIO().deleteQuietly(index.path(file));
+                } catch (RuntimeException cleanupFailure) {
+                    failure.addSuppressed(cleanupFailure);
+                }
+            }
+            throw failure;
+        }
+        List<IndexFileMeta> deleted = new ArrayList<>(indexes.size());
+        indexes.forEach(old -> deleted.add(old.file));
+        return new CommitMessageImpl(
+                partition,
+                UNAWARE_BUCKET,
+                null,
+                DataIncrement.emptyIncrement(),
+                new CompactIncrement(
+                        Collections.emptyList(),
+                        Collections.emptyList(),
+                        Collections.emptyList(),
+                        added,
+                        deleted));
+    }
+
+    private static Map<String, DeletionVector> rewrite(
+            Rewrite rewrite, DeletionVectorsIndexFile index, boolean bitmap64) 
{
+        Map<String, Map<String, DeletionFile>> byIndex = new LinkedHashMap<>();
+        Map<String, Range> inputRanges = new LinkedHashMap<>();
+        for (Source source : rewrite.sources) {
+            byIndex.computeIfAbsent(source.deletion.path(), ignored -> new 
LinkedHashMap<>())
+                    .put(source.anchor, source.deletion);
+            inputRanges.put(source.anchor, source.range);
+        }
+        TreeMap<Long, Map.Entry<String, Range>> outputs = new TreeMap<>();
+        rewrite.outputs.entrySet().forEach(output -> 
outputs.put(output.getValue().from, output));
+        Map<String, DeletionVector> result = new LinkedHashMap<>();
+        for (Map<String, DeletionFile> inputs : byIndex.values()) {
+            index.readDeletionVector(inputs)
+                    .forEach(
+                            (anchor, bitmap) ->
+                                    move(
+                                            bitmap,
+                                            inputRanges.get(anchor),
+                                            outputs,
+                                            result,
+                                            bitmap64));
+        }
+        return result;
+    }
+
+    static void move(
+            DeletionVector bitmap,
+            Range input,
+            TreeMap<Long, Map.Entry<String, Range>> outputs,
+            Map<String, DeletionVector> result,
+            boolean bitmap64) {
+        if (bitmap.isEmpty()) {
+            return;
+        }
+        Map.Entry<String, Range> first = 
outputs.floorEntry(input.from).getValue();
+        if (input.equals(first.getValue())) {

Review Comment:
   **[P2] Respect the configured bitmap format on equal-range rewrites.** I 
wrote two rows to a Data Evolution append table, deleted one with the default 
32-bit DV, and set `deletion-vectors.bitmap64=true`. Equal-range compaction 
committed and the surviving row remained readable. A subsequent SQL DELETE on 
that row failed with `Only instance with the same class type can be merged`. 
The new index reused the old bitmap even though the output writer was 
configured for 64-bit vectors. Reuse is safe when the formats match. For a 
mismatch, convert by position or reject the rewrite before publishing the new 
files.



##########
docs/docs/multimodal-table/data-evolution-maintenance.mdx:
##########
@@ -89,6 +89,20 @@ Ordinary compaction **does not physically remove rows hidden 
by deletion
 vectors**. Removing their positions would change the alignment with untouched
 column files.
 
+### Recovering Files After a Failed Rewrite
+
+Flink and Spark rewrite DV indexes on workers. A failed worker attempt cleans
+up the new DV files it has tracked. Files from tasks that already succeeded,
+or from a lost worker process, can remain unreferenced if the overall rewrite
+fails before committing.
+
+After the failed job and its retries have stopped, use
+[Remove Orphan Files](../maintenance/manage-snapshots#remove-orphan-files)

Review Comment:
   **[P2] Cover recovery when DV indexes use an external root.** I reran the 
failed distributed rewrite fixture with `global-index.external-path` set to a 
separate local directory. The previous snapshot stayed current and the 
completed Spark task left one external DV index behind. Running the orphan 
cleaner with a cutoff after the failure removed zero external files. The local 
and Spark cleaners share `OrphanFilesClean.listPaimonFileDirs`, which scans the 
table index directory and external data directories but omits this external 
index root. Provide a recovery path that can identify this table's outputs 
safely, or document the limitation and how to reclaim those files.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to