JustinBinber commented on code in PR #11156:
URL: https://github.com/apache/paimon/pull/11156#discussion_r4644585709
##########
paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java:
##########
@@ -83,6 +92,122 @@
/** Test for table with data evolution. */
public class DataEvolutionTableTest extends DataEvolutionTestBase {
+ @Test
+ public void testStreamingCommitConflictPathEnforcesEncodedBudget() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ long firstRowId = writeFullRows(table, 1);
+ long before = table.snapshotManager().latestSnapshot().id();
+ Map<String, String> options = new HashMap<>();
+ options.put(
+
CoreOptions.DATA_EVOLUTION_METADATA_MEMORY_OPTIMIZATION_MODE.key(),
"streaming");
+
options.put(CoreOptions.DATA_EVOLUTION_METADATA_MEMORY_MAX_DISK_SIZE.key(), "8
b");
+ FileStoreTable streaming = table.copy(options);
+ BatchWriteBuilder builder = streaming.newBatchWriteBuilder();
+ try (BatchTableWrite write =
+ builder.newWrite()
+ .withWriteType(
+
table.rowType().project(Collections.singletonList("f2")));
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(GenericRow.of(BinaryString.fromString("updated")));
+ List<CommitMessage> messages = write.prepareCommit();
+ setFirstRowId(messages, firstRowId);
+ CommitMessageImpl update =
+ ((CommitMessageImpl)
messages.get(0)).withCheckFromSnapshot(before);
+ assertThatThrownBy(() ->
commit.commit(Collections.singletonList(update)))
+ .hasStackTraceContaining("Metadata merge encoded limit");
+ }
+
assertThat(table.snapshotManager().latestSnapshot().id()).isEqualTo(before);
+ }
+
+ @Test
+ public void testStreamingCommitEntriesPinSnapshotAndReadDelta() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ List<String> firstNames =
+ writeOneFullRowAndCollectNewFiles(table).stream()
+ .map(DataFileMeta::fileName)
+ .collect(Collectors.toList());
+ Snapshot firstSnapshot = table.snapshotManager().latestSnapshot();
+ List<String> secondNames = new ArrayList<>();
+ List<String> scannedNames = new ArrayList<>();
+ FileStoreScan initiallyLatest = table.store().newScan();
+
+ initiallyLatest.forEachSimpleEntry(
+ entry -> {
+ scannedNames.add(entry.fileName());
+ if (secondNames.isEmpty()) {
+ secondNames.addAll(
+
writeOneFullRowAndCollectNewFiles(table).stream()
+ .map(DataFileMeta::fileName)
+ .collect(Collectors.toList()));
+ }
+ });
+
+ Snapshot secondSnapshot = table.snapshotManager().latestSnapshot();
+ assertThat(secondSnapshot.id()).isEqualTo(firstSnapshot.id() + 1);
+ assertThat(scannedNames).isEqualTo(firstNames);
+ List<SimpleFileEntry> pinned = new ArrayList<>();
+
table.store().newScan().withSnapshot(firstSnapshot).forEachSimpleEntry(pinned::add);
+
assertThat(pinned.stream().map(SimpleFileEntry::fileName).collect(Collectors.toList()))
+ .isEqualTo(firstNames);
+
+ List<SimpleFileEntry> delta = new ArrayList<>();
+ table.store()
+ .newScan()
+ .withSnapshot(secondSnapshot)
+ .withKind(ScanMode.DELTA)
+ .forEachSimpleEntry(delta::add);
+
assertThat(delta.stream().map(SimpleFileEntry::fileName).collect(Collectors.toList()))
+ .isEqualTo(secondNames);
+ assertThat(delta)
+ .isEqualTo(
+ table.store()
+ .newScan()
+ .withSnapshot(secondSnapshot)
+ .withKind(ScanMode.DELTA)
+ .readSimpleEntries());
+ assertThat(table.store().newScan().readSimpleEntries().size())
+ .isEqualTo(firstNames.size() + secondNames.size());
+
+ CommitScanner commitScanner =
+ new CommitScanner(
Review Comment:
Fixed in `e0be601617`. Integrated upstream `71082ea86a` and passed the
table's actual SchemaManager to the updated CommitScanner constructor. The
normal Core reactor compiles and 120 focused cases pass. The inherited
pinned-snapshot/DELTA scanner control also passes with a temporary
local-filesystem fixture adapter: the original traceable URI is not Windows
compatible; that adapter and its bytecode were removed and are not part of this
PR. The normal 27-module Spark 3.5.9 reactor passes all 18 selected read/MERGE
SQL cases. Fresh automatic CI is pending for this head.
##########
paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java:
##########
@@ -306,6 +314,145 @@ private List<SimpleFileEntry> scanChangedRowRanges(
.values());
}
+ /**
+ * Produces the ordinary DE base scan's input without retaining its
complete metadata. Returns
+ * whether the caller must keep the first base record per identity,
matching the supplemental
+ * scan branch's left-first deduplication. Range and referenced-name state
still grow with the
+ * changes; the consumer must not retain the entire decoded base.
+ */
+ public boolean forEachBaseDataFile(
+ Snapshot latestSnapshot,
+ List<BinaryRow> changedPartitions,
+ List<ManifestEntry> deltaFiles,
+ List<IndexManifestEntry> indexFiles,
+ ThrowingConsumer<SimpleFileEntry, Exception> consumer)
+ throws Exception {
+ List<Range> changedRowRanges = changedRowRanges(deltaFiles,
indexFiles);
+ Set<String> referencedDataFiles = referencedDataFiles(deltaFiles,
indexFiles);
+ if (changedRowRanges.isEmpty()) {
+ if (!referencedDataFiles.isEmpty()) {
+ commitScanner()
+ .forEachEntryFromDataFiles(
+ latestSnapshot, changedPartitions,
referencedDataFiles, consumer);
+ }
+ return false;
+ }
+
+ commitScanner()
+ .forEachEntryFromChangedRowRanges(
+ latestSnapshot,
+ changedPartitions,
+ changedRowRanges,
+ entry -> {
+ referencedDataFiles.remove(entry.fileName());
+ consumer.accept(entry);
+ });
+ if (referencedDataFiles.isEmpty()) {
+ return false;
+ }
+ commitScanner()
+ .forEachEntryFromDataFiles(
+ latestSnapshot, changedPartitions,
referencedDataFiles, consumer);
+ // Selecting this branch triggers deduplication even when the second
scan emits no rows.
+ return true;
+ }
+
+ public boolean supportsStreamingConflicts() {
+ return streamingConflictsSupported;
+ }
+
+ /**
+ * Checks the no-DV successful path without complete base/delta/merged
metadata collections.
+ * Possible conflicts fall back to the original checker after closing
spill resources,
+ * preserving its diagnostics, error priority and throw/return contract.
That diagnostic path
+ * retains the original memory boundary. Buffers too small for both
sorters also use that
+ * boundary instead of increasing the configured sort-buffer budget.
Callback and DV callers
+ * keep the ordinary path.
+ */
+ public Optional<RuntimeException> checkConflictsStreaming(
+ CoreOptions options,
+ @Nullable IOManager ioManager,
+ Snapshot latestSnapshot,
+ List<BinaryRow> changedPartitions,
+ List<ManifestEntry> deltaFiles,
+ List<IndexManifestEntry> indexFiles,
+ @Nullable RowIdConflictChecker rowIdConflictChecker,
+ CommitKind commitKind) {
+ boolean ordinary =
+ !supportsStreamingConflicts() ||
!FileEntryMergeBuffer.supportsBufferSize(options);
+ if (!ordinary) {
+ boolean globalIndexes =
+ indexFiles.stream()
+ .anyMatch(
+ entry ->
+ entry.kind() == FileKind.ADD
+ &&
entry.indexFile().globalIndexMeta() != null);
+ DataEvolutionStreamingChecks checks =
+ new DataEvolutionStreamingChecks(
+ commitKind,
+ rowIdCheckFromSnapshot != null || commitKind ==
CommitKind.COMPACT,
+ globalIndexes);
+ try (FileEntryMergeBuffer buffer = new
FileEntryMergeBuffer(options, ioManager)) {
+ boolean deduplicate =
+ forEachBaseDataFile(
+ latestSnapshot,
+ changedPartitions,
+ deltaFiles,
+ indexFiles,
+ entry -> {
+ checks.addRawBase(entry);
+ buffer.writeBase(entry);
+ });
+ for (ManifestEntry entry : deltaFiles) {
+ buffer.writeDelta(SimpleFileEntry.from(entry));
+ }
+ try {
+ buffer.forEachMerged(deduplicate, checks::addBase,
checks::addMerged);
+ } catch (IllegalStateException invalidIdentity) {
+ // The original checker selects bucket errors before
identity errors and
+ // distinguishes delta exceptions from combined conflict
return values.
+ ordinary = true;
+ }
+ ordinary =
+ ordinary
+ || checks.mayConflict(
+ deduplicate,
+ deltaFiles,
+ indexFiles,
+ latestSnapshot.nextRowId());
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed streaming data evolution
conflict check.", e);
+ } finally {
+ checks.clear();
+ }
+ }
+ if (ordinary) {
+ List<SimpleFileEntry> base =
+ scanBaseDataFiles(
+ latestSnapshot,
+ changedPartitions,
+ deltaFiles,
+ indexFiles,
+ commitKind,
+ null,
+ false);
+ return checkConflicts(
+ latestSnapshot,
+ base,
+ SimpleFileEntry.from(deltaFiles),
+ indexFiles,
+ rowIdConflictChecker,
+ commitKind);
+ }
+ return checkForRowIdFromSnapshot(
Review Comment:
Reproduced and fixed in `e0be601617`. After the successful streaming
range/existence summary, the checker now calls the existing
checkCentroidSourceChanges before the snapshot RowID check. The ordinary
fallback and its error ordering are preserved. The regression keeps the covered
RowIDs unchanged: an update of the indexed source column returns the same
source-rows-changed error as ordinary checking, while an unrelated-column
update remains accepted. Both controls pass, without collecting the legacy base
metadata on the streaming path.
##########
paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java:
##########
@@ -183,6 +263,90 @@ public Iterator<ManifestEntry> readManifestEntries(
return filtered.iterator();
}
+ /**
+ * Streams complete, filtered RowID groups from one pinned batch snapshot.
+ *
+ * <p>The consumer should encode each group before requesting the next.
Sort buffers are
+ * bounded, but DELETE identities and the largest connected group still
grow with their input.
+ * The caller owns the supplied IO manager; this method only closes its
own sorting channels.
+ */
+ @Nullable
+ public Snapshot forEachFileGroup(
+ IOManager ioManager, ThrowingConsumer<List<ManifestEntry>,
Exception> consumer)
+ throws Exception {
+ if (scanMode != ScanMode.ALL) {
+ throw new UnsupportedOperationException(
+ "Incremental DE group planning only supports batch ALL
scans.");
+ }
+ long started = System.nanoTime();
+ ManifestsReader.Result manifests = readManifests();
+ long[] resultedFiles = {0};
+ Long rowLimit =
+ limit != null
+ && limit > 0
+ && inputFilter == null
+ && rowRangeIndex == null
+ && !deletionVectorsEnabled
+ ? limit
+ : null;
+ long[] remainingRows = {rowLimit == null ? 0 : rowLimit};
+ CompactFileIdentifierSet deleted = new CompactFileIdentifierSet();
+ ReusableIdentifier identifier = new ReusableIdentifier();
+ Function<ManifestEntry, ManifestEntry> converter =
+ createManifestEntryConverter(Function.identity());
+ try (DataEvolutionManifestEntryProducer producer =
Review Comment:
Reproduced and fixed in `e0be601617`. Before allocating the external sorter,
budgets below three pages use the ordinary planning body and seal the unused
replay store. This avoids recursively re-entering the store-bound read method
and preserves owner cleanup. Real Core and Spark SELECT controls pass at both
128 KiB and 192 KiB; Spark also checks predicates and LIMIT. The three-page
case still produces replay metadata, while the two-page store remains empty and
readable/sealed. The documented fallback retains the ordinary planner's heap
boundary and does not increase the configured buffer budget.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]