This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 408718220c76 fix: keep Avro merge record contexts instance-local
(#20734)
408718220c76 is described below
commit 408718220c76bff329fe992126320b63782d3a64
Author: Danny Chan <[email protected]>
AuthorDate: Fri Nov 20 05:11:21 2026 +0000
fix: keep Avro merge record contexts instance-local (#20734)
* fix(common): keep log scanner record contexts instance-local
* fix: isolate realtime reader context and exercise real scanner merges
---
.../table/log/HoodieMergedLogRecordScanner.java | 3 +-
.../log/TestHoodieMergedLogRecordScanner.java | 113 +++++++++++++++++++++
.../realtime/RealtimeCompactedRecordReader.java | 3 +-
3 files changed, 117 insertions(+), 2 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
index 14cf2c7e5fa4..20fb76d12210 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
@@ -100,6 +100,8 @@ public class HoodieMergedLogRecordScanner extends
AbstractHoodieLogRecordScanner
private long totalTimeTakenToReadAndMergeBlocks;
private final String[] orderingFields;
private final DeleteContext deleteContext;
+ // Merging registers schema IDs, so the context and its local cache must
belong to this scanner.
+ private final RecordContext recordContext = new AvroRecordContext();
@SuppressWarnings("unchecked")
protected HoodieMergedLogRecordScanner(HoodieStorage storage, String
basePath, List<String> logFilePaths, HoodieSchema readerSchema,
@@ -255,7 +257,6 @@ public class HoodieMergedLogRecordScanner extends
AbstractHoodieLogRecordScanner
HoodieRecord<T> prevRecord = records.get(key);
if (prevRecord != null) {
// Merge and store the combined record
- RecordContext recordContext =
AvroRecordContext.getFieldAccessorInstance();
BufferedRecord<T> prevBufferedRecord =
BufferedRecords.fromHoodieRecord(prevRecord, readerSchema,
recordContext, this.getPayloadProps(), orderingFields,
deleteContext);
BufferedRecord<T> newBufferedRecord =
BufferedRecords.fromHoodieRecord(newRecord, readerSchema,
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordScanner.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordScanner.java
new file mode 100644
index 000000000000..c8363307fe46
--- /dev/null
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordScanner.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.common.table.log;
+
+import org.apache.hudi.common.engine.RecordContext;
+import org.apache.hudi.common.model.HoodieAvroRecord;
+import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordMerger;
+import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.read.BufferedRecord;
+
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.IndexedRecord;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class TestHoodieMergedLogRecordScanner {
+ @TempDir
+ Path tempDir;
+
+ @Test
+ void testSchemaCacheIsOwnedByEachScanner() throws Exception {
+ HoodieSchema firstSchema = schema("first");
+ HoodieSchema secondSchema = schema("second");
+ List<RecordContext<IndexedRecord>> contexts = new ArrayList<>();
+ HoodieRecordMerger merger = mock(HoodieRecordMerger.class);
+
when(merger.getRecordType()).thenReturn(HoodieRecord.HoodieRecordType.AVRO);
+ when(merger.merge(any(), any(), any(), any())).thenAnswer(invocation -> {
+ BufferedRecord<IndexedRecord> older = invocation.getArgument(0);
+ BufferedRecord<IndexedRecord> newer = invocation.getArgument(1);
+ RecordContext<IndexedRecord> context = invocation.getArgument(2);
+ contexts.add(context);
+ assertEquals(older.getRecord().getSchema(),
context.getSchemaFromBufferRecord(older).toAvroSchema());
+ assertEquals(newer.getRecord().getSchema(),
context.getSchemaFromBufferRecord(newer).toAvroSchema());
+ return new HoodiePreCombineAvroRecordMerger().merge(older, newer,
context, invocation.getArgument(3));
+ });
+
+ try (HoodieMergedLogRecordScanner first = scanner(firstSchema, merger);
+ HoodieMergedLogRecordScanner second = scanner(secondSchema, merger)) {
+ first.processNextRecord(record(firstSchema, "first-insert"));
+ first.processNextRecord(record(firstSchema, "first-update"));
+ second.processNextRecord(record(secondSchema, "second-insert"));
+ second.processNextRecord(record(secondSchema, "second-update"));
+ first.processNextRecord(record(firstSchema, "first-update-again"));
+
+ assertEquals(3, contexts.size());
+ RecordContext<IndexedRecord> firstContext = contexts.get(0);
+ RecordContext<IndexedRecord> secondContext = contexts.get(1);
+ assertNotSame(firstContext, secondContext, "Scanners must not share
their schema cache");
+ assertSame(firstContext, contexts.get(2), "A scanner must reuse its
context across merges");
+ assertEquals("first-update-again", ((IndexedRecord)
first.getRecords().get("key").getData()).get(0).toString());
+ assertEquals("second-update", ((IndexedRecord)
second.getRecords().get("key").getData()).get(0).toString());
+ }
+ }
+
+ private HoodieMergedLogRecordScanner scanner(HoodieSchema schema,
HoodieRecordMerger merger) {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ when(metaClient.getTableConfig()).thenReturn(new HoodieTableConfig());
+ return HoodieMergedLogRecordScanner.newBuilder()
+ .withTableMetaClient(metaClient)
+ .withLogFilePaths(Collections.emptyList())
+ .withReaderSchema(schema)
+ .withMaxMemorySizeInBytes(1024L * 1024)
+ .withSpillableMapBasePath(tempDir.toString())
+ .withRecordMerger(merger)
+ .withForceFullScan(false)
+ .build();
+ }
+
+ private static HoodieSchema schema(String name) {
+ return HoodieSchema.parse("{\"type\":\"record\",\"name\":\"" + name +
"\",\"fields\":[{\"name\":\"value\",\"type\":\"string\"}]}");
+ }
+
+ private static HoodieAvroRecord<OverwriteWithLatestAvroPayload>
record(HoodieSchema schema, String value) {
+ GenericData.Record data = new GenericData.Record(schema.toAvroSchema());
+ data.put("value", value);
+ return new HoodieAvroRecord<>(new HoodieKey("key", "partition"), new
OverwriteWithLatestAvroPayload(data, 0L));
+ }
+}
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
index dced64dc3b84..635a2993c8ad 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
@@ -65,6 +65,8 @@ public class RealtimeCompactedRecordReader extends
AbstractRealtimeRecordReader
protected final RecordReader<NullWritable, ArrayWritable> parquetReader;
private final Map<String, HoodieRecord> deltaRecordMap;
+ // Schema registration during merging must stay local to this reader.
+ private final RecordContext<IndexedRecord> recordContext = new
AvroRecordContext();
private final HoodieRecordMerger merger = new HoodieAvroRecordMerger();
private final Set<String> deltaRecordKeys;
private final HoodieMergedLogRecordScanner mergedLogRecordScanner;
@@ -202,7 +204,6 @@ public class RealtimeCompactedRecordReader extends
AbstractRealtimeRecordReader
// so to be compatible with hive and presto, we should rewrite oldRecord
before we call combineAndGetUpdateValue,
// once presto on hudi have its own mor reader, we can remove the rewrite
logical.
GenericRecord genericRecord =
HiveAvroSerializer.rewriteRecordIgnoreResultCheck(oldRecord,
getLogScannerReaderSchema());
- RecordContext<IndexedRecord> recordContext =
AvroRecordContext.getFieldAccessorInstance();
BufferedRecord record = BufferedRecords.fromEngineRecord(genericRecord,
HoodieAvroSchemaCache.intern(genericRecord.getSchema()), recordContext,
orderingFields, newRecord.getRecordKey(), false);
BufferedRecord newBufferedRecord =
BufferedRecords.fromHoodieRecord(newRecord,
HoodieAvroSchemaCache.intern(getLogScannerReaderSchema().toAvroSchema()),
recordContext, payloadProps, orderingFields, deleteContext);