This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20598-c2c6bfb83e158c53e7a3c3052b28cf6a179f4107 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit fca82d3c0cac72e7a9913b8b46076ffac7520749 Author: Shuo Cheng <[email protected]> AuthorDate: Thu Nov 12 02:51:33 2026 +0000 fix(common): pass correct keys to the update processor in the file group reader's sorted record buffer (#20598) --- .../SortedKeyBasedFileGroupRecordBuffer.java | 2 +- .../TestSortedKeyBasedFileGroupRecordBuffer.java | 50 +++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java index a04a64f11584..ac7233a759e4 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/buffer/SortedKeyBasedFileGroupRecordBuffer.java @@ -75,7 +75,7 @@ class SortedKeyBasedFileGroupRecordBuffer<T> extends KeyBasedFileGroupRecordBuff } // Handle the case where the next record is only present in the log records BufferedRecord<T> nextLogRecord = records.remove(nextLogRecordKey); - nextRecord = updateProcessor.processUpdate(recordKey, null, nextLogRecord, nextLogRecord.isDelete()); + nextRecord = updateProcessor.processUpdate(nextLogRecordKey, null, nextLogRecord, nextLogRecord.isDelete()); if (nextRecord != null) { // If the next log record does not result in a deletion, or we are emitting deletes, we can return it // and queue the base record, which is already read from the iterator, for the next iteration diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java index efb682098ee5..d1418101c13e 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/buffer/TestSortedKeyBasedFileGroupRecordBuffer.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.HoodieReaderContext; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.DeleteRecord; +import org.apache.hudi.common.model.HoodieOperation; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.schema.internal.InternalSchema; import org.apache.hudi.common.table.HoodieTableConfig; @@ -33,6 +34,7 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.PartialUpdateMode; import org.apache.hudi.common.table.log.block.HoodieDataBlock; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.table.read.BaseFileUpdateCallback; import org.apache.hudi.common.table.read.FileGroupReaderSchemaHandler; import org.apache.hudi.common.table.read.HoodieReadStats; import org.apache.hudi.common.table.read.InputSplit; @@ -46,6 +48,8 @@ import org.apache.hudi.storage.StorageConfiguration; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.util.ArrayList; @@ -60,8 +64,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; class TestSortedKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuffer { @@ -225,7 +233,47 @@ class TestSortedKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuf assertTrue(actualRecords.indexOf(bmpRecord) < actualRecords.indexOf(supplementaryRecord)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void readLogOnlyRecordsBeforeBaseWithCallbacks(boolean emitDeletes) throws IOException { + HoodieReaderContext<TestRecord> readerContext = mock(HoodieReaderContext.class, RETURNS_DEEP_STUBS); + BaseFileUpdateCallback<TestRecord> callback = mock(BaseFileUpdateCallback.class); + SortedKeyBasedFileGroupRecordBuffer<TestRecord> buffer = buildSortedKeyBasedFileGroupRecordBuffer( + readerContext, new HoodieReadStats(), emitDeletes, Option.of(callback)); + try { + buffer.setBaseFileIterator(ClosableIterator.wrap(Arrays.asList(testRecord3, testRecord5).iterator())); + HoodieDataBlock dataBlock = mock(HoodieDataBlock.class); + when(dataBlock.getSchema()).thenReturn(HoodieTestDataGenerator.HOODIE_SCHEMA); + when(dataBlock.getEngineRecordIterator(readerContext)).thenReturn( + ClosableIterator.wrap(Arrays.asList(testRecord6, testRecord2, testRecord1).iterator())); + HoodieDeleteBlock deleteBlock = mock(HoodieDeleteBlock.class); + mockDeleteRecords(deleteBlock, DeleteRecord.create("4", "")); + buffer.processDataBlock(dataBlock, Option.empty()); + buffer.processDeleteBlock(deleteBlock); + + // Keys 1 and 2 precede the first base record; delete 4 precedes the second. + // Key 6 is emitted by the log-only tail after the base iterator is exhausted. + List<TestRecord> expected = emitDeletes + ? Arrays.asList(testRecord1, testRecord2, testRecord3, testRecord4, testRecord5, testRecord6) + : Arrays.asList(testRecord1, testRecord2, testRecord3, testRecord5, testRecord6); + assertEquals(expected, getActualRecordsForSortedKeyBased(buffer)); + verify(callback).onInsert(eq("1"), any()); + verify(callback).onInsert(eq("2"), any()); + verify(callback).onDelete(eq("4"), isNull(), eq(HoodieOperation.DELETE)); + verify(callback).onInsert(eq("6"), any()); + verifyNoMoreInteractions(callback); + } finally { + buffer.close(); + } + } + private SortedKeyBasedFileGroupRecordBuffer<TestRecord> buildSortedKeyBasedFileGroupRecordBuffer(HoodieReaderContext<TestRecord> mockReaderContext, HoodieReadStats readStats) { + return buildSortedKeyBasedFileGroupRecordBuffer(mockReaderContext, readStats, false, Option.empty()); + } + + private SortedKeyBasedFileGroupRecordBuffer<TestRecord> buildSortedKeyBasedFileGroupRecordBuffer( + HoodieReaderContext<TestRecord> mockReaderContext, HoodieReadStats readStats, + boolean emitDeletes, Option<BaseFileUpdateCallback<TestRecord>> callback) { when(mockReaderContext.getSchemaHandler().getRequiredSchema()).thenReturn(HoodieTestDataGenerator.HOODIE_SCHEMA); when(mockReaderContext.getSchemaHandler().getInternalSchema()).thenReturn(InternalSchema.getEmptyInternalSchema()); when(mockReaderContext.getSchemaHandler().getSchemaEvolutionTransformer(any(), any())).thenReturn(Option.empty()); @@ -243,7 +291,7 @@ class TestSortedKeyBasedFileGroupRecordBuffer extends BaseTestFileGroupRecordBuf Option<PartialUpdateMode> partialUpdateModeOpt = Option.empty(); TypedProperties props = new TypedProperties(); when(mockReaderContext.getPayloadClasses(any())).thenReturn(Option.empty()); - UpdateProcessor<TestRecord> updateProcessor = UpdateProcessor.create(readStats, mockReaderContext, false, Option.empty(), props); + UpdateProcessor<TestRecord> updateProcessor = UpdateProcessor.create(readStats, mockReaderContext, emitDeletes, callback, props); return new SortedKeyBasedFileGroupRecordBuffer<>( mockReaderContext, recordMergeMode, partialUpdateModeOpt, props, Collections.emptyList(), updateProcessor); }
