This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20734-40ea87502e765acc7a30b6052659f5244ff35c7d in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 408718220c76bff329fe992126320b63782d3a64 Author: Danny Chan <[email protected]> AuthorDate: Fri Nov 20 05:11:21 2026 +0000 fix: keep Avro merge record contexts instance-local (#20734) * fix(common): keep log scanner record contexts instance-local * fix: isolate realtime reader context and exercise real scanner merges --- .../table/log/HoodieMergedLogRecordScanner.java | 3 +- .../log/TestHoodieMergedLogRecordScanner.java | 113 +++++++++++++++++++++ .../realtime/RealtimeCompactedRecordReader.java | 3 +- 3 files changed, 117 insertions(+), 2 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index 14cf2c7e5fa4..20fb76d12210 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -100,6 +100,8 @@ public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordScanner private long totalTimeTakenToReadAndMergeBlocks; private final String[] orderingFields; private final DeleteContext deleteContext; + // Merging registers schema IDs, so the context and its local cache must belong to this scanner. + private final RecordContext recordContext = new AvroRecordContext(); @SuppressWarnings("unchecked") protected HoodieMergedLogRecordScanner(HoodieStorage storage, String basePath, List<String> logFilePaths, HoodieSchema readerSchema, @@ -255,7 +257,6 @@ public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordScanner HoodieRecord<T> prevRecord = records.get(key); if (prevRecord != null) { // Merge and store the combined record - RecordContext recordContext = AvroRecordContext.getFieldAccessorInstance(); BufferedRecord<T> prevBufferedRecord = BufferedRecords.fromHoodieRecord(prevRecord, readerSchema, recordContext, this.getPayloadProps(), orderingFields, deleteContext); BufferedRecord<T> newBufferedRecord = BufferedRecords.fromHoodieRecord(newRecord, readerSchema, diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordScanner.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordScanner.java new file mode 100644 index 000000000000..c8363307fe46 --- /dev/null +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordScanner.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.table.log; + +import org.apache.hudi.common.engine.RecordContext; +import org.apache.hudi.common.model.HoodieAvroRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.read.BufferedRecord; + +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.IndexedRecord; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestHoodieMergedLogRecordScanner { + @TempDir + Path tempDir; + + @Test + void testSchemaCacheIsOwnedByEachScanner() throws Exception { + HoodieSchema firstSchema = schema("first"); + HoodieSchema secondSchema = schema("second"); + List<RecordContext<IndexedRecord>> contexts = new ArrayList<>(); + HoodieRecordMerger merger = mock(HoodieRecordMerger.class); + when(merger.getRecordType()).thenReturn(HoodieRecord.HoodieRecordType.AVRO); + when(merger.merge(any(), any(), any(), any())).thenAnswer(invocation -> { + BufferedRecord<IndexedRecord> older = invocation.getArgument(0); + BufferedRecord<IndexedRecord> newer = invocation.getArgument(1); + RecordContext<IndexedRecord> context = invocation.getArgument(2); + contexts.add(context); + assertEquals(older.getRecord().getSchema(), context.getSchemaFromBufferRecord(older).toAvroSchema()); + assertEquals(newer.getRecord().getSchema(), context.getSchemaFromBufferRecord(newer).toAvroSchema()); + return new HoodiePreCombineAvroRecordMerger().merge(older, newer, context, invocation.getArgument(3)); + }); + + try (HoodieMergedLogRecordScanner first = scanner(firstSchema, merger); + HoodieMergedLogRecordScanner second = scanner(secondSchema, merger)) { + first.processNextRecord(record(firstSchema, "first-insert")); + first.processNextRecord(record(firstSchema, "first-update")); + second.processNextRecord(record(secondSchema, "second-insert")); + second.processNextRecord(record(secondSchema, "second-update")); + first.processNextRecord(record(firstSchema, "first-update-again")); + + assertEquals(3, contexts.size()); + RecordContext<IndexedRecord> firstContext = contexts.get(0); + RecordContext<IndexedRecord> secondContext = contexts.get(1); + assertNotSame(firstContext, secondContext, "Scanners must not share their schema cache"); + assertSame(firstContext, contexts.get(2), "A scanner must reuse its context across merges"); + assertEquals("first-update-again", ((IndexedRecord) first.getRecords().get("key").getData()).get(0).toString()); + assertEquals("second-update", ((IndexedRecord) second.getRecords().get("key").getData()).get(0).toString()); + } + } + + private HoodieMergedLogRecordScanner scanner(HoodieSchema schema, HoodieRecordMerger merger) { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + when(metaClient.getTableConfig()).thenReturn(new HoodieTableConfig()); + return HoodieMergedLogRecordScanner.newBuilder() + .withTableMetaClient(metaClient) + .withLogFilePaths(Collections.emptyList()) + .withReaderSchema(schema) + .withMaxMemorySizeInBytes(1024L * 1024) + .withSpillableMapBasePath(tempDir.toString()) + .withRecordMerger(merger) + .withForceFullScan(false) + .build(); + } + + private static HoodieSchema schema(String name) { + return HoodieSchema.parse("{\"type\":\"record\",\"name\":\"" + name + "\",\"fields\":[{\"name\":\"value\",\"type\":\"string\"}]}"); + } + + private static HoodieAvroRecord<OverwriteWithLatestAvroPayload> record(HoodieSchema schema, String value) { + GenericData.Record data = new GenericData.Record(schema.toAvroSchema()); + data.put("value", value); + return new HoodieAvroRecord<>(new HoodieKey("key", "partition"), new OverwriteWithLatestAvroPayload(data, 0L)); + } +} diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java index dced64dc3b84..635a2993c8ad 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java @@ -65,6 +65,8 @@ public class RealtimeCompactedRecordReader extends AbstractRealtimeRecordReader protected final RecordReader<NullWritable, ArrayWritable> parquetReader; private final Map<String, HoodieRecord> deltaRecordMap; + // Schema registration during merging must stay local to this reader. + private final RecordContext<IndexedRecord> recordContext = new AvroRecordContext(); private final HoodieRecordMerger merger = new HoodieAvroRecordMerger(); private final Set<String> deltaRecordKeys; private final HoodieMergedLogRecordScanner mergedLogRecordScanner; @@ -202,7 +204,6 @@ public class RealtimeCompactedRecordReader extends AbstractRealtimeRecordReader // so to be compatible with hive and presto, we should rewrite oldRecord before we call combineAndGetUpdateValue, // once presto on hudi have its own mor reader, we can remove the rewrite logical. GenericRecord genericRecord = HiveAvroSerializer.rewriteRecordIgnoreResultCheck(oldRecord, getLogScannerReaderSchema()); - RecordContext<IndexedRecord> recordContext = AvroRecordContext.getFieldAccessorInstance(); BufferedRecord record = BufferedRecords.fromEngineRecord(genericRecord, HoodieAvroSchemaCache.intern(genericRecord.getSchema()), recordContext, orderingFields, newRecord.getRecordKey(), false); BufferedRecord newBufferedRecord = BufferedRecords.fromHoodieRecord(newRecord, HoodieAvroSchemaCache.intern(getLogScannerReaderSchema().toAvroSchema()), recordContext, payloadProps, orderingFields, deleteContext);
