JustinBinber commented on code in PR #11156:
URL: https://github.com/apache/paimon/pull/11156#discussion_r4644587740


##########
paimon-core/src/main/java/org/apache/paimon/table/source/DataFileMetaStore.java:
##########
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataFileMetaSerializer;
+import org.apache.paimon.io.DataInputDeserializer;
+import org.apache.paimon.io.DataInputViewStreamWrapper;
+import org.apache.paimon.io.DataOutputSerializer;
+
+import 
org.apache.paimon.shade.guava30.com.google.common.base.FinalizablePhantomReference;
+import 
org.apache.paimon.shade.guava30.com.google.common.base.FinalizableReferenceQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.AbstractList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Driver-local, explicitly owned storage for replaying split metadata without 
retaining POJOs.
+ *
+ * <p>Append is single-threaded. After sealing, immutable slices support 
independent concurrent
+ * readers. The owner must outlive every traversal and Task serialization; 
closing invalidates all
+ * slices. Serialized splits contain the metadata itself, never these local 
file paths.
+ */
+public final class DataFileMetaStore implements Closeable {
+
+    private static final Logger LOG = 
LoggerFactory.getLogger(DataFileMetaStore.class);
+
+    private static final int BUFFER_SIZE = 64 * 1024;
+
+    private final long maxDiskBytes;
+    private Path dataPath;
+    private Path indexPath;
+    private BufferedOutputStream dataOutput;
+    private DataOutputStream indexOutput;
+    private FileChannel dataInput;
+    private FileChannel indexInput;
+    private DataFileMetaSerializer writer = new DataFileMetaSerializer();
+    private DataOutputSerializer encoded = new DataOutputSerializer(1024);
+    private int count;
+    private long dataBytes;
+    private volatile boolean sealed;
+    private volatile boolean closed;
+    private volatile CleanupReference cleanupReference;
+
+    public DataFileMetaStore(Path directory, long maxDiskBytes) throws 
IOException {
+        if (maxDiskBytes < Long.BYTES) {
+            throw new IllegalArgumentException("Metadata store limit must 
include an end offset.");
+        }
+        this.maxDiskBytes = maxDiskBytes;
+        try {
+            dataPath = Files.createTempFile(directory, "paimon-metadata-", 
".data");
+            indexPath = Files.createTempFile(directory, "paimon-metadata-", 
".index");
+            dataOutput = new 
BufferedOutputStream(Files.newOutputStream(dataPath), BUFFER_SIZE);
+            indexOutput =
+                    new DataOutputStream(
+                            new BufferedOutputStream(
+                                    Files.newOutputStream(indexPath), 
BUFFER_SIZE));
+        } catch (IOException | RuntimeException | Error failure) {
+            abort(failure);
+            throw failure;
+        }
+    }
+
+    public void append(DataFileMeta file) throws IOException {
+        ensureOpen();
+        if (sealed) {
+            throw new IllegalStateException("Metadata store is sealed.");
+        }
+        try {
+            if (count == Integer.MAX_VALUE) {
+                throw new IOException("Metadata store exceeds the maximum List 
size.");
+            }
+            encoded.clear();
+            writer.serialize(file, encoded);
+            long nextDataBytes = Math.addExact(dataBytes, encoded.length());
+            // Include this record's offset and the final end offset, even 
before sealing.
+            long nextIndexBytes = ((long) count + 2) * Long.BYTES;
+            if (nextIndexBytes > maxDiskBytes || nextDataBytes > maxDiskBytes 
- nextIndexBytes) {
+                throw new IOException("Metadata store disk limit exceeded: " + 
maxDiskBytes);
+            }
+            dataOutput.write(encoded.getSharedBuffer(), 0, encoded.length());
+            indexOutput.writeLong(dataBytes);
+            dataBytes = nextDataBytes;
+            count++;
+        } catch (IOException | RuntimeException | Error failure) {
+            abort(failure);
+            throw failure;
+        }
+    }
+
+    public void seal() throws IOException {
+        ensureOpen();
+        if (sealed) {
+            return;
+        }
+        try {
+            indexOutput.writeLong(dataBytes);
+            dataOutput.close();
+            indexOutput.close();
+            dataInput = FileChannel.open(dataPath, StandardOpenOption.READ);
+            indexInput = FileChannel.open(indexPath, StandardOpenOption.READ);
+            dataOutput = null;
+            indexOutput = null;
+            writer = null;
+            encoded = null;
+            sealed = true;
+        } catch (IOException | RuntimeException | Error failure) {
+            abort(failure);
+            throw failure;
+        }
+    }
+
+    public List<DataFileMeta> slice(int fromInclusive, int toExclusive) {
+        ensureReadable();
+        checkRange(fromInclusive, toExclusive, count);
+        return new Slice(this, fromInclusive, toExclusive - fromInclusive);
+    }
+
+    /** Encoded data and index bytes, including the reserved final offset, not 
filesystem blocks. */
+    public long serializedBytes() {
+        return dataBytes + ((long) count + 1) * Long.BYTES;
+    }
+
+    void requireFresh() {
+        ensureOpen();
+        if (sealed || count != 0) {
+            throw new IllegalStateException("Metadata output store must be 
fresh and writable.");
+        }
+    }
+
+    /**
+     * Enables backup cleanup for a sealed plan whose lifetime follows its 
reachable slices.
+     * Explicit owners should still close promptly. Collection and backup 
cleanup have no timing
+     * guarantee; this is not a substitute for a disk budget or 
planning-failure cleanup.
+     */
+    public synchronized DataFileMetaStore withAutomaticCleanup() {
+        ensureReadable();
+        if (cleanupReference == null) {
+            cleanupReference =
+                    new CleanupReference(this, dataInput, indexInput, 
dataPath, indexPath);
+        }
+        return this;
+    }
+
+    private void ensureOpen() {
+        if (closed) {
+            throw new IllegalStateException("Metadata store is closed.");
+        }
+    }
+
+    private void ensureReadable() {
+        ensureOpen();
+        if (!sealed) {
+            throw new IllegalStateException("Metadata store must be sealed 
before reading.");
+        }
+    }
+
+    private long offset(int record) throws IOException {
+        ByteBuffer offset = ByteBuffer.allocate(Long.BYTES);
+        readFully(indexInput, offset, (long) record * Long.BYTES);
+        return offset.getLong(0);
+    }
+
+    private void checkOffsets(long from, long to) throws IOException {
+        if (from < 0 || to <= from || to > dataBytes) {
+            throw new IOException("Invalid metadata store offsets: " + from + 
", " + to);
+        }
+    }
+
+    private DataFileMeta get(int record) {
+        ensureReadable();
+        try {
+            long from = offset(record);
+            long to = offset(record + 1);
+            checkOffsets(from, to);
+            if (to - from > Integer.MAX_VALUE) {
+                throw new IOException("Metadata record exceeds the maximum 
array size.");
+            }
+            byte[] bytes = new byte[(int) (to - from)];
+            readFully(dataInput, ByteBuffer.wrap(bytes), from);
+            return new DataFileMetaSerializer().deserialize(new 
DataInputDeserializer(bytes));
+        } catch (IOException failure) {
+            throw new UncheckedIOException(failure);
+        } finally {
+            keepAlive();
+        }
+    }
+
+    private Iterator<DataFileMeta> iterator(int fromRecord, int size) {
+        ensureReadable();
+        if (size == 0) {
+            return Collections.<DataFileMeta>emptyList().iterator();
+        }
+        try {
+            long from = offset(fromRecord);
+            long to = offset(fromRecord + size);
+            checkOffsets(from, to);
+            DataInputViewStreamWrapper input =
+                    new DataInputViewStreamWrapper(
+                            new BufferedInputStream(rangeInput(from, to), 
BUFFER_SIZE));
+            return new Iterator<DataFileMeta>() {
+                private final DataFileMetaSerializer reader = new 
DataFileMetaSerializer();
+                private int remaining = size;
+
+                @Override
+                public boolean hasNext() {
+                    ensureReadable();
+                    return remaining > 0;
+                }
+
+                @Override
+                public DataFileMeta next() {
+                    if (!hasNext()) {
+                        throw new NoSuchElementException();
+                    }
+                    try {
+                        DataFileMeta value = reader.deserialize(input);
+                        remaining--;
+                        return value;
+                    } catch (IOException failure) {
+                        throw new UncheckedIOException(failure);
+                    } finally {
+                        keepAlive();
+                    }
+                }
+            };
+        } catch (IOException failure) {
+            throw new UncheckedIOException(failure);
+        }
+    }
+
+    private InputStream rangeInput(long from, long to) {
+        // Positional reads share only the owner's channels, never a mutable 
cursor or extra FD.
+        return new InputStream() {
+            private long position = from;
+
+            @Override
+            public int read() throws IOException {
+                byte[] one = new byte[1];
+                return read(one, 0, 1) < 0 ? -1 : one[0] & 0xff;
+            }
+
+            @Override
+            public int read(byte[] bytes, int offset, int length) throws 
IOException {
+                ensureReadable();
+                if (offset < 0 || length < 0 || offset > bytes.length - 
length) {
+                    throw new IndexOutOfBoundsException();
+                }
+                if (length == 0) {
+                    return 0;
+                }
+                if (position == to) {
+                    return -1;
+                }
+                int requested = (int) Math.min(length, to - position);
+                int read = dataInput.read(ByteBuffer.wrap(bytes, offset, 
requested), position);
+                if (read <= 0) {
+                    throw new EOFException("Incomplete metadata store data.");
+                }
+                position += read;
+                return read;
+            }
+        };
+    }
+
+    private static void readFully(FileChannel channel, ByteBuffer buffer, long 
position)
+            throws IOException {
+        while (buffer.hasRemaining()) {
+            int read = channel.read(buffer, position);

Review Comment:
   Reproduced both shared-channel failures and fixed in `e0be601617`. Replay 
now owns two fixed RandomAccessFile handles; seek/read and owner close share 
synchronization, and cancellation preserves the interrupt flag without closing 
another slice's resources. Reads check cancellation between bounded chunks, 
including buffered iteration. Independent sequential/indexed replay succeeds 
after another consumer is interrupted. All 18 store cases pass, including 
cleanup and concurrent replay/close controls. Iterators acquire no extra 
descriptors. Physical reads serialize per store; cancellation still waits for 
an active native read to return, and this change makes no SQL performance claim.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to