JingsongLi commented on code in PR #11156:
URL: https://github.com/apache/paimon/pull/11156#discussion_r4643732810
##########
paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java:
##########
@@ -83,6 +92,122 @@
/** Test for table with data evolution. */
public class DataEvolutionTableTest extends DataEvolutionTestBase {
+ @Test
+ public void testStreamingCommitConflictPathEnforcesEncodedBudget() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ long firstRowId = writeFullRows(table, 1);
+ long before = table.snapshotManager().latestSnapshot().id();
+ Map<String, String> options = new HashMap<>();
+ options.put(
+
CoreOptions.DATA_EVOLUTION_METADATA_MEMORY_OPTIMIZATION_MODE.key(),
"streaming");
+
options.put(CoreOptions.DATA_EVOLUTION_METADATA_MEMORY_MAX_DISK_SIZE.key(), "8
b");
+ FileStoreTable streaming = table.copy(options);
+ BatchWriteBuilder builder = streaming.newBatchWriteBuilder();
+ try (BatchTableWrite write =
+ builder.newWrite()
+ .withWriteType(
+
table.rowType().project(Collections.singletonList("f2")));
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(GenericRow.of(BinaryString.fromString("updated")));
+ List<CommitMessage> messages = write.prepareCommit();
+ setFirstRowId(messages, firstRowId);
+ CommitMessageImpl update =
+ ((CommitMessageImpl)
messages.get(0)).withCheckFromSnapshot(before);
+ assertThatThrownBy(() ->
commit.commit(Collections.singletonList(update)))
+ .hasStackTraceContaining("Metadata merge encoded limit");
+ }
+
assertThat(table.snapshotManager().latestSnapshot().id()).isEqualTo(before);
+ }
+
+ @Test
+ public void testStreamingCommitEntriesPinSnapshotAndReadDelta() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ List<String> firstNames =
+ writeOneFullRowAndCollectNewFiles(table).stream()
+ .map(DataFileMeta::fileName)
+ .collect(Collectors.toList());
+ Snapshot firstSnapshot = table.snapshotManager().latestSnapshot();
+ List<String> secondNames = new ArrayList<>();
+ List<String> scannedNames = new ArrayList<>();
+ FileStoreScan initiallyLatest = table.store().newScan();
+
+ initiallyLatest.forEachSimpleEntry(
+ entry -> {
+ scannedNames.add(entry.fileName());
+ if (secondNames.isEmpty()) {
+ secondNames.addAll(
+
writeOneFullRowAndCollectNewFiles(table).stream()
+ .map(DataFileMeta::fileName)
+ .collect(Collectors.toList()));
+ }
+ });
+
+ Snapshot secondSnapshot = table.snapshotManager().latestSnapshot();
+ assertThat(secondSnapshot.id()).isEqualTo(firstSnapshot.id() + 1);
+ assertThat(scannedNames).isEqualTo(firstNames);
+ List<SimpleFileEntry> pinned = new ArrayList<>();
+
table.store().newScan().withSnapshot(firstSnapshot).forEachSimpleEntry(pinned::add);
+
assertThat(pinned.stream().map(SimpleFileEntry::fileName).collect(Collectors.toList()))
+ .isEqualTo(firstNames);
+
+ List<SimpleFileEntry> delta = new ArrayList<>();
+ table.store()
+ .newScan()
+ .withSnapshot(secondSnapshot)
+ .withKind(ScanMode.DELTA)
+ .forEachSimpleEntry(delta::add);
+
assertThat(delta.stream().map(SimpleFileEntry::fileName).collect(Collectors.toList()))
+ .isEqualTo(secondNames);
+ assertThat(delta)
+ .isEqualTo(
+ table.store()
+ .newScan()
+ .withSnapshot(secondSnapshot)
+ .withKind(ScanMode.DELTA)
+ .readSimpleEntries());
+ assertThat(table.store().newScan().readSimpleEntries().size())
+ .isEqualTo(firstNames.size() + secondNames.size());
+
+ CommitScanner commitScanner =
+ new CommitScanner(
Review Comment:
**[P1] Update the CommitScanner test construction for current master**
After merging this head into master `6beb23d690`, the normal Core test
reactor fails to compile because #11290 added a SchemaManager argument. Pass
the table's schema manager in this test and verify the integration with current
master.
##########
paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java:
##########
@@ -306,6 +314,145 @@ private List<SimpleFileEntry> scanChangedRowRanges(
.values());
}
+ /**
+ * Produces the ordinary DE base scan's input without retaining its
complete metadata. Returns
+ * whether the caller must keep the first base record per identity,
matching the supplemental
+ * scan branch's left-first deduplication. Range and referenced-name state
still grow with the
+ * changes; the consumer must not retain the entire decoded base.
+ */
+ public boolean forEachBaseDataFile(
+ Snapshot latestSnapshot,
+ List<BinaryRow> changedPartitions,
+ List<ManifestEntry> deltaFiles,
+ List<IndexManifestEntry> indexFiles,
+ ThrowingConsumer<SimpleFileEntry, Exception> consumer)
+ throws Exception {
+ List<Range> changedRowRanges = changedRowRanges(deltaFiles,
indexFiles);
+ Set<String> referencedDataFiles = referencedDataFiles(deltaFiles,
indexFiles);
+ if (changedRowRanges.isEmpty()) {
+ if (!referencedDataFiles.isEmpty()) {
+ commitScanner()
+ .forEachEntryFromDataFiles(
+ latestSnapshot, changedPartitions,
referencedDataFiles, consumer);
+ }
+ return false;
+ }
+
+ commitScanner()
+ .forEachEntryFromChangedRowRanges(
+ latestSnapshot,
+ changedPartitions,
+ changedRowRanges,
+ entry -> {
+ referencedDataFiles.remove(entry.fileName());
+ consumer.accept(entry);
+ });
+ if (referencedDataFiles.isEmpty()) {
+ return false;
+ }
+ commitScanner()
+ .forEachEntryFromDataFiles(
+ latestSnapshot, changedPartitions,
referencedDataFiles, consumer);
+ // Selecting this branch triggers deduplication even when the second
scan emits no rows.
+ return true;
+ }
+
+ public boolean supportsStreamingConflicts() {
+ return streamingConflictsSupported;
+ }
+
+ /**
+ * Checks the no-DV successful path without complete base/delta/merged
metadata collections.
+ * Possible conflicts fall back to the original checker after closing
spill resources,
+ * preserving its diagnostics, error priority and throw/return contract.
That diagnostic path
+ * retains the original memory boundary. Buffers too small for both
sorters also use that
+ * boundary instead of increasing the configured sort-buffer budget.
Callback and DV callers
+ * keep the ordinary path.
+ */
+ public Optional<RuntimeException> checkConflictsStreaming(
+ CoreOptions options,
+ @Nullable IOManager ioManager,
+ Snapshot latestSnapshot,
+ List<BinaryRow> changedPartitions,
+ List<ManifestEntry> deltaFiles,
+ List<IndexManifestEntry> indexFiles,
+ @Nullable RowIdConflictChecker rowIdConflictChecker,
+ CommitKind commitKind) {
+ boolean ordinary =
+ !supportsStreamingConflicts() ||
!FileEntryMergeBuffer.supportsBufferSize(options);
+ if (!ordinary) {
+ boolean globalIndexes =
+ indexFiles.stream()
+ .anyMatch(
+ entry ->
+ entry.kind() == FileKind.ADD
+ &&
entry.indexFile().globalIndexMeta() != null);
+ DataEvolutionStreamingChecks checks =
+ new DataEvolutionStreamingChecks(
+ commitKind,
+ rowIdCheckFromSnapshot != null || commitKind ==
CommitKind.COMPACT,
+ globalIndexes);
+ try (FileEntryMergeBuffer buffer = new
FileEntryMergeBuffer(options, ioManager)) {
+ boolean deduplicate =
+ forEachBaseDataFile(
+ latestSnapshot,
+ changedPartitions,
+ deltaFiles,
+ indexFiles,
+ entry -> {
+ checks.addRawBase(entry);
+ buffer.writeBase(entry);
+ });
+ for (ManifestEntry entry : deltaFiles) {
+ buffer.writeDelta(SimpleFileEntry.from(entry));
+ }
+ try {
+ buffer.forEachMerged(deduplicate, checks::addBase,
checks::addMerged);
+ } catch (IllegalStateException invalidIdentity) {
+ // The original checker selects bucket errors before
identity errors and
+ // distinguishes delta exceptions from combined conflict
return values.
+ ordinary = true;
+ }
+ ordinary =
+ ordinary
+ || checks.mayConflict(
+ deduplicate,
+ deltaFiles,
+ indexFiles,
+ latestSnapshot.nextRowId());
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed streaming data evolution
conflict check.", e);
+ } finally {
+ checks.clear();
+ }
+ }
+ if (ordinary) {
+ List<SimpleFileEntry> base =
+ scanBaseDataFiles(
+ latestSnapshot,
+ changedPartitions,
+ deltaFiles,
+ indexFiles,
+ commitKind,
+ null,
+ false);
+ return checkConflicts(
+ latestSnapshot,
+ base,
+ SimpleFileEntry.from(deltaFiles),
+ indexFiles,
+ rowIdConflictChecker,
+ commitKind);
+ }
+ return checkForRowIdFromSnapshot(
Review Comment:
**[P1] Check centroid source changes on successful streaming validation**
On the same integration, I extended the master's
`testCentroidIndexChecksContiguousSourceRange` fixture to exercise both paths.
A partial update of the indexed column after training leaves the covered RowIDs
unchanged. The ordinary checker still returns `Centroid index source rows
changed`, but the streaming checker returns success. This exit bypasses
`checkCentroidSourceChanges`. Run that check after the streaming summary or
send centroid index ADD commits through the ordinary checker.
##########
paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java:
##########
@@ -183,6 +263,90 @@ public Iterator<ManifestEntry> readManifestEntries(
return filtered.iterator();
}
+ /**
+ * Streams complete, filtered RowID groups from one pinned batch snapshot.
+ *
+ * <p>The consumer should encode each group before requesting the next.
Sort buffers are
+ * bounded, but DELETE identities and the largest connected group still
grow with their input.
+ * The caller owns the supplied IO manager; this method only closes its
own sorting channels.
+ */
+ @Nullable
+ public Snapshot forEachFileGroup(
+ IOManager ioManager, ThrowingConsumer<List<ManifestEntry>,
Exception> consumer)
+ throws Exception {
+ if (scanMode != ScanMode.ALL) {
+ throw new UnsupportedOperationException(
+ "Incremental DE group planning only supports batch ALL
scans.");
+ }
+ long started = System.nanoTime();
+ ManifestsReader.Result manifests = readManifests();
+ long[] resultedFiles = {0};
+ Long rowLimit =
+ limit != null
+ && limit > 0
+ && inputFilter == null
+ && rowRangeIndex == null
+ && !deletionVectorsEnabled
+ ? limit
+ : null;
+ long[] remainingRows = {rowLimit == null ? 0 : rowLimit};
+ CompactFileIdentifierSet deleted = new CompactFileIdentifierSet();
+ ReusableIdentifier identifier = new ReusableIdentifier();
+ Function<ManifestEntry, ManifestEntry> converter =
+ createManifestEntryConverter(Function.identity());
+ try (DataEvolutionManifestEntryProducer producer =
Review Comment:
**[P2] Handle valid two-page buffers in scan planning**
A Spark 3.5.9 Data Evolution table with `page-size=64 kb` and
`sort-spill-buffer-size=128 kb` accepts INSERT. A plain SELECT returns the
stored row in off mode and throws IllegalArgumentException in streaming mode.
Both modes return the row with 192 KB. The revised commit fallback handles
small buffers before building its sorters, while the scan path still constructs
a sorter that requires at least three pages. Apply a planner fallback or
explicitly validate this boundary and cover SELECT with both buffer sizes.
##########
paimon-core/src/main/java/org/apache/paimon/table/source/DataFileMetaStore.java:
##########
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataFileMetaSerializer;
+import org.apache.paimon.io.DataInputDeserializer;
+import org.apache.paimon.io.DataInputViewStreamWrapper;
+import org.apache.paimon.io.DataOutputSerializer;
+
+import
org.apache.paimon.shade.guava30.com.google.common.base.FinalizablePhantomReference;
+import
org.apache.paimon.shade.guava30.com.google.common.base.FinalizableReferenceQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.AbstractList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Driver-local, explicitly owned storage for replaying split metadata without
retaining POJOs.
+ *
+ * <p>Append is single-threaded. After sealing, immutable slices support
independent concurrent
+ * readers. The owner must outlive every traversal and Task serialization;
closing invalidates all
+ * slices. Serialized splits contain the metadata itself, never these local
file paths.
+ */
+public final class DataFileMetaStore implements Closeable {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(DataFileMetaStore.class);
+
+ private static final int BUFFER_SIZE = 64 * 1024;
+
+ private final long maxDiskBytes;
+ private Path dataPath;
+ private Path indexPath;
+ private BufferedOutputStream dataOutput;
+ private DataOutputStream indexOutput;
+ private FileChannel dataInput;
+ private FileChannel indexInput;
+ private DataFileMetaSerializer writer = new DataFileMetaSerializer();
+ private DataOutputSerializer encoded = new DataOutputSerializer(1024);
+ private int count;
+ private long dataBytes;
+ private volatile boolean sealed;
+ private volatile boolean closed;
+ private volatile CleanupReference cleanupReference;
+
+ public DataFileMetaStore(Path directory, long maxDiskBytes) throws
IOException {
+ if (maxDiskBytes < Long.BYTES) {
+ throw new IllegalArgumentException("Metadata store limit must
include an end offset.");
+ }
+ this.maxDiskBytes = maxDiskBytes;
+ try {
+ dataPath = Files.createTempFile(directory, "paimon-metadata-",
".data");
+ indexPath = Files.createTempFile(directory, "paimon-metadata-",
".index");
+ dataOutput = new
BufferedOutputStream(Files.newOutputStream(dataPath), BUFFER_SIZE);
+ indexOutput =
+ new DataOutputStream(
+ new BufferedOutputStream(
+ Files.newOutputStream(indexPath),
BUFFER_SIZE));
+ } catch (IOException | RuntimeException | Error failure) {
+ abort(failure);
+ throw failure;
+ }
+ }
+
+ public void append(DataFileMeta file) throws IOException {
+ ensureOpen();
+ if (sealed) {
+ throw new IllegalStateException("Metadata store is sealed.");
+ }
+ try {
+ if (count == Integer.MAX_VALUE) {
+ throw new IOException("Metadata store exceeds the maximum List
size.");
+ }
+ encoded.clear();
+ writer.serialize(file, encoded);
+ long nextDataBytes = Math.addExact(dataBytes, encoded.length());
+ // Include this record's offset and the final end offset, even
before sealing.
+ long nextIndexBytes = ((long) count + 2) * Long.BYTES;
+ if (nextIndexBytes > maxDiskBytes || nextDataBytes > maxDiskBytes
- nextIndexBytes) {
+ throw new IOException("Metadata store disk limit exceeded: " +
maxDiskBytes);
+ }
+ dataOutput.write(encoded.getSharedBuffer(), 0, encoded.length());
+ indexOutput.writeLong(dataBytes);
+ dataBytes = nextDataBytes;
+ count++;
+ } catch (IOException | RuntimeException | Error failure) {
+ abort(failure);
+ throw failure;
+ }
+ }
+
+ public void seal() throws IOException {
+ ensureOpen();
+ if (sealed) {
+ return;
+ }
+ try {
+ indexOutput.writeLong(dataBytes);
+ dataOutput.close();
+ indexOutput.close();
+ dataInput = FileChannel.open(dataPath, StandardOpenOption.READ);
+ indexInput = FileChannel.open(indexPath, StandardOpenOption.READ);
+ dataOutput = null;
+ indexOutput = null;
+ writer = null;
+ encoded = null;
+ sealed = true;
+ } catch (IOException | RuntimeException | Error failure) {
+ abort(failure);
+ throw failure;
+ }
+ }
+
+ public List<DataFileMeta> slice(int fromInclusive, int toExclusive) {
+ ensureReadable();
+ checkRange(fromInclusive, toExclusive, count);
+ return new Slice(this, fromInclusive, toExclusive - fromInclusive);
+ }
+
+ /** Encoded data and index bytes, including the reserved final offset, not
filesystem blocks. */
+ public long serializedBytes() {
+ return dataBytes + ((long) count + 1) * Long.BYTES;
+ }
+
+ void requireFresh() {
+ ensureOpen();
+ if (sealed || count != 0) {
+ throw new IllegalStateException("Metadata output store must be
fresh and writable.");
+ }
+ }
+
+ /**
+ * Enables backup cleanup for a sealed plan whose lifetime follows its
reachable slices.
+ * Explicit owners should still close promptly. Collection and backup
cleanup have no timing
+ * guarantee; this is not a substitute for a disk budget or
planning-failure cleanup.
+ */
+ public synchronized DataFileMetaStore withAutomaticCleanup() {
+ ensureReadable();
+ if (cleanupReference == null) {
+ cleanupReference =
+ new CleanupReference(this, dataInput, indexInput,
dataPath, indexPath);
+ }
+ return this;
+ }
+
+ private void ensureOpen() {
+ if (closed) {
+ throw new IllegalStateException("Metadata store is closed.");
+ }
+ }
+
+ private void ensureReadable() {
+ ensureOpen();
+ if (!sealed) {
+ throw new IllegalStateException("Metadata store must be sealed
before reading.");
+ }
+ }
+
+ private long offset(int record) throws IOException {
+ ByteBuffer offset = ByteBuffer.allocate(Long.BYTES);
+ readFully(indexInput, offset, (long) record * Long.BYTES);
+ return offset.getLong(0);
+ }
+
+ private void checkOffsets(long from, long to) throws IOException {
+ if (from < 0 || to <= from || to > dataBytes) {
+ throw new IOException("Invalid metadata store offsets: " + from +
", " + to);
+ }
+ }
+
+ private DataFileMeta get(int record) {
+ ensureReadable();
+ try {
+ long from = offset(record);
+ long to = offset(record + 1);
+ checkOffsets(from, to);
+ if (to - from > Integer.MAX_VALUE) {
+ throw new IOException("Metadata record exceeds the maximum
array size.");
+ }
+ byte[] bytes = new byte[(int) (to - from)];
+ readFully(dataInput, ByteBuffer.wrap(bytes), from);
+ return new DataFileMetaSerializer().deserialize(new
DataInputDeserializer(bytes));
+ } catch (IOException failure) {
+ throw new UncheckedIOException(failure);
+ } finally {
+ keepAlive();
+ }
+ }
+
+ private Iterator<DataFileMeta> iterator(int fromRecord, int size) {
+ ensureReadable();
+ if (size == 0) {
+ return Collections.<DataFileMeta>emptyList().iterator();
+ }
+ try {
+ long from = offset(fromRecord);
+ long to = offset(fromRecord + size);
+ checkOffsets(from, to);
+ DataInputViewStreamWrapper input =
+ new DataInputViewStreamWrapper(
+ new BufferedInputStream(rangeInput(from, to),
BUFFER_SIZE));
+ return new Iterator<DataFileMeta>() {
+ private final DataFileMetaSerializer reader = new
DataFileMetaSerializer();
+ private int remaining = size;
+
+ @Override
+ public boolean hasNext() {
+ ensureReadable();
+ return remaining > 0;
+ }
+
+ @Override
+ public DataFileMeta next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ try {
+ DataFileMeta value = reader.deserialize(input);
+ remaining--;
+ return value;
+ } catch (IOException failure) {
+ throw new UncheckedIOException(failure);
+ } finally {
+ keepAlive();
+ }
+ }
+ };
+ } catch (IOException failure) {
+ throw new UncheckedIOException(failure);
+ }
+ }
+
+ private InputStream rangeInput(long from, long to) {
+ // Positional reads share only the owner's channels, never a mutable
cursor or extra FD.
+ return new InputStream() {
+ private long position = from;
+
+ @Override
+ public int read() throws IOException {
+ byte[] one = new byte[1];
+ return read(one, 0, 1) < 0 ? -1 : one[0] & 0xff;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws
IOException {
+ ensureReadable();
+ if (offset < 0 || length < 0 || offset > bytes.length -
length) {
+ throw new IndexOutOfBoundsException();
+ }
+ if (length == 0) {
+ return 0;
+ }
+ if (position == to) {
+ return -1;
+ }
+ int requested = (int) Math.min(length, to - position);
+ int read = dataInput.read(ByteBuffer.wrap(bytes, offset,
requested), position);
+ if (read <= 0) {
+ throw new EOFException("Incomplete metadata store data.");
+ }
+ position += read;
+ return read;
+ }
+ };
+ }
+
+ private static void readFully(FileChannel channel, ByteBuffer buffer, long
position)
+ throws IOException {
+ while (buffer.hasRemaining()) {
+ int read = channel.read(buffer, position);
Review Comment:
**[P2] Isolate interrupt handling between replay slices**
On JDK 8 I sealed a real store with two records and obtained two slices. An
interrupted consumer reading one slice receives ClosedByInterruptException. A
separate uncancelled thread replaying the other slice then receives
ClosedChannelException even though the store owner remains open. Spark retains
these slices in planned splits for later serialization. Positional reads share
an interruptible channel, so one cancelled traversal permanently invalidates
the rest. Use interrupt-isolated replay handles and cover cancellation followed
by independent replay.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]