JustinBinber commented on code in PR #11156:
URL: https://github.com/apache/paimon/pull/11156#discussion_r4630497115


##########
paimon-core/src/main/java/org/apache/paimon/operation/FileEntryMergeBuffer.java:
##########
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.operation;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.disk.IOManager;
+import org.apache.paimon.manifest.FileEntry;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.SimpleFileEntry;
+import org.apache.paimon.manifest.SimpleFileEntryWithDV;
+import org.apache.paimon.operation.ManifestEntryExternalSort.BinaryRowSorter;
+import 
org.apache.paimon.operation.ManifestEntryExternalSort.ExternalSortConfig;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.MutableObjectIterator;
+import org.apache.paimon.utils.ThrowingConsumer;
+
+import javax.annotation.Nullable;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.Preconditions.checkState;
+import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
+import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
+
+/**
+ * Spillable equivalent of validating delta entries and then merging base and 
delta entries with
+ * {@link FileEntry#mergeEntries(Iterable)}. Survivors retain the original 
insertion order.
+ *
+ * <p>The two sorters share one configured sort-buffer budget. Reduction 
retains only the current
+ * identity's state and the first invalid histories, rather than a map of all 
file identities. The
+ * encoded-size limit covers input and survivor rows; it is a logical metadata 
budget, not a bound
+ * on physical spill I/O (merging runs may temporarily coexist). A single 
entry still has to fit in
+ * the sort buffer. The caller owns this buffer until consumption finishes, 
including on failure.
+ */
+public final class FileEntryMergeBuffer implements AutoCloseable {
+
+    private static final RowType ROW_TYPE =
+            DataTypes.ROW(
+                    DataTypes.BYTES(), // complete identity, including DV 
subtype
+                    DataTypes.INT(), // base before delta
+                    DataTypes.BIGINT(), // original ordinal within source
+                    DataTypes.TINYINT(), // kind
+                    DataTypes.INT(), // total buckets (not part of identity)
+                    DataTypes.BYTES(), // min key
+                    DataTypes.BYTES(), // max key
+                    DataTypes.BIGINT(), // row count
+                    DataTypes.BIGINT()); // nullable first row id
+
+    private final IOManager ioManager;
+    private final boolean ownedIOManager;
+    private final BinaryRowSorter identities;
+    private final BinaryRowSorter survivors;
+    private final InternalRowSerializer serializer = new 
InternalRowSerializer(ROW_TYPE);
+    private final long maxEncodedSize;
+    private final long[] ordinals = new long[2];
+    private long encodedSize;
+    private boolean finished;
+    private boolean closed;
+    @Nullable private Failure deltaFailure;
+    @Nullable private Failure combinedFailure;
+
+    public FileEntryMergeBuffer(CoreOptions options, @Nullable IOManager 
ioManager)
+            throws Exception {
+        maxEncodedSize = options.dataEvolutionMetadataMaxDiskSize();
+        ExternalSortConfig configured = ExternalSortConfig.from(options, 
ioManager);
+        long pages = configured.bufferSize / configured.pageSize;
+        checkArgument(pages >= 6, "Metadata merge requires at least six 
sort-buffer pages.");

Review Comment:
   Fixed in 
[`62c90e2279`](https://github.com/apache/paimon/pull/11156/commits/62c90e227975ef2e4345ffce9a8ed6f3e46a5d33).
 The sole production caller now checks whether the configured budget supplies 
at least six complete pages before creating the two streaming sorters. Smaller 
buffers use the existing ordinary conflict checker. Each sorter's three-page 
guard stays intact and the configured memory budget is not increased.
   
   A real DE update reproduces the failure only for streaming/four pages in the 
original implementation. Four- and six-page tests now pass across 
off/lightweight/streaming: snapshot 2 is published, untouched-column files and 
reconstructed values are retained, and the stale same-column update is 
rejected. Boundary, spill, cleanup, encoded-budget and conflict regressions 
also pass. The full selected Core reactor is 81/81 on JDK8. The documented 
limit is that this fallback keeps the ordinary checker's heap usage.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to