This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 709767094682c1c67813b491ad193512863cf538 Author: voonhous <[email protected]> AuthorDate: Sun Jun 14 13:09:02 2026 +0800 perf(metadata): Avoid per-record enum-array clone and string parse when materializing MDT records (#18997) * perf(metadata): Avoid per-record enum-array clone and string parse when materializing MDT records Two per-record costs on the metadata-table read path: - MetadataPartitionType.get(int) iterated values(), which clones the enum constant array on every call; it runs once per record materialized from the metadata table (RLI/SI/col-stats lookups, MDT log merges). Cache values() once and iterate the cached array; the linear scan and IllegalArgumentException for unknown types are unchanged. - RECORD_INDEX.constructMetadataPayload decoded the numeric record-index fields via Long.parseLong(x.toString()) / Integer.parseInt(...) even though they are long/int in HoodieMetadata.avsc. Read them directly with ((Number) x).longValue() / .intValue(), removing five String allocations and parses per materialized RLI record. String fields keep toString(). Behavior-preserving; reconstructed records are identical. Adds an avro write/read round-trip test over both fileId encodings. Closes #18996 * review: address nit (cherry picked from commit 097dd4a4c8df98414fcfd57375fb315ce844ae95) --- .../hudi/metadata/MetadataPartitionType.java | 20 +++++++++++++------- .../hudi/metadata/TestHoodieTableMetadataUtil.java | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java index 04bd9bdab264..1e12d05b1c22 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java @@ -176,14 +176,16 @@ public enum MetadataPartitionType { if (recordIndexRecord.hasField(RECORD_INDEX_FIELD_POSITION)) { recordIndexPosition = recordIndexRecord.get(RECORD_INDEX_FIELD_POSITION); } + // Numeric RLI fields are long/int per HoodieMetadata.avsc, so read them directly instead of + // round-tripping through String (toString + parse) for every materialized record. payload.recordIndexMetadata = new HoodieRecordIndexInfo(recordIndexRecord.get(RECORD_INDEX_FIELD_PARTITION).toString(), - Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_HIGH_BITS).toString()), - Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_LOW_BITS).toString()), - Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILE_INDEX).toString()), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_HIGH_BITS)).longValue(), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_LOW_BITS)).longValue(), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILE_INDEX)).intValue(), recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID).toString(), - Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_INSTANT_TIME).toString()), - Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_ENCODING).toString()), - recordIndexPosition != null ? Long.parseLong(recordIndexPosition.toString()) : null); + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_INSTANT_TIME)).longValue(), + ((Number) recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_ENCODING)).intValue(), + recordIndexPosition != null ? ((Number) recordIndexPosition).longValue() : null); } }, EXPRESSION_INDEX(PARTITION_NAME_EXPRESSION_INDEX_PREFIX, "expr-index-", -1) { @@ -427,11 +429,15 @@ public enum MetadataPartitionType { && partitionType != COLUMN_STATS; } + // Cache values() once; it clones the constant array on every call, and get(int) runs once per + // record materialized from the metadata table (RLI/SI/col-stats lookups, MDT log merges). + private static final MetadataPartitionType[] VALUES = values(); + /** * Get the metadata partition type for the given record type. */ public static MetadataPartitionType get(int type) { - for (MetadataPartitionType partitionType : values()) { + for (MetadataPartitionType partitionType : VALUES) { if (partitionType.getRecordType() == type) { return partitionType; } diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java index 6cab9b9067c3..cd35f382dedb 100644 --- a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java @@ -18,6 +18,8 @@ package org.apache.hudi.metadata; +import org.apache.hudi.avro.HoodieAvroUtils; +import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.common.function.SerializableBiFunction; import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieIndexMetadata; @@ -33,6 +35,7 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; import org.apache.hudi.common.util.Option; +import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -380,6 +383,25 @@ class TestHoodieTableMetadataUtil { assertEquals(fromString.getData(), fromMillis.getData()); } + @Test + void testRecordIndexPayloadRoundTripsThroughAvro() throws Exception { + // both fileId encodings populate the numeric RLI fields; they must survive the avro read path + // (constructMetadataPayload now reads the long/int fields directly instead of via toString+parse) + assertRecordIndexRoundTrips("49b8b3c8-9e5d-4731-9d51-a2d8e9b5c7f3-0", 0); + assertRecordIndexRoundTrips("some-raw-file-id", 1); + } + + private static void assertRecordIndexRoundTrips(String fileId, int fileIdEncoding) throws Exception { + HoodieRecord<HoodieMetadataPayload> written = + HoodieMetadataPayload.createRecordIndexUpdate("rk1", "p1", fileId, "20260610153045678", fileIdEncoding); + // serialize to avro bytes and back so the read path sees a GenericRecord with boxed Long/Integer fields + byte[] bytes = HoodieAvroUtils.avroToBytes(written.getData().getInsertValue(null).get()); + GenericRecord deserialized = HoodieAvroUtils.bytesToAvro(bytes, HoodieMetadataRecord.getClassSchema()); + HoodieMetadataPayload readBack = new HoodieMetadataPayload(Option.of(deserialized)); + assertEquals(written.getData().recordIndexMetadata, readBack.recordIndexMetadata, + "RLI metadata must survive the avro read path for fileId encoding " + fileIdEncoding); + } + @Test void testGetLocationFromRecordIndexInfoFormatsInstantConsistently() { long instantMillis1 = HoodieMetadataPayload.parseRecordIndexInstantTime("20260610153045678");
