(hudi) branch master updated: [HUDI-7877] Add record position to record index metadata payload (#11467)

[yihua]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new eb0725d1ef5 [HUDI-7877] Add record position to record index metadata 
payload (#11467)
eb0725d1ef5 is described below

commit eb0725d1ef5bbcc7884f442778523cfb76f8203b
Author: Lokesh Jain <lj...@apache.org>
AuthorDate: Thu Jun 27 01:50:50 2024 +0530

    [HUDI-7877] Add record position to record index metadata payload (#11467)
---
 hudi-common/src/main/avro/HoodieMetadata.avsc                 |  9 +++++++++
 .../java/org/apache/hudi/metadata/HoodieMetadataPayload.java  | 11 ++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/hudi-common/src/main/avro/HoodieMetadata.avsc 
b/hudi-common/src/main/avro/HoodieMetadata.avsc
index 4eefc3c37c8..e7a5a1e145d 100644
--- a/hudi-common/src/main/avro/HoodieMetadata.avsc
+++ b/hudi-common/src/main/avro/HoodieMetadata.avsc
@@ -427,6 +427,15 @@
                             "type": "int",
                             "default": 0,
                             "doc": "Represents fileId encoding. Possible 
values are 0 and 1. O represents UUID based fileID, and 1 represents raw string 
format of the fileId. \nWhen the encoding is 0, reader can deduce fileID from 
fileIdLowBits, fileIdHighBits and fileIndex."
+                        },
+                        {
+                            "name": "position",
+                            "type": [
+                                "null",
+                                "long"
+                            ],
+                            "default": null,
+                            "doc": "Represents position of record within a 
file group for easier access. It will be used for index lookup."
                         }
                     ]
                 }
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java 
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
index b04f943f1d9..7b9359d5dbb 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
@@ -156,6 +156,7 @@ public class HoodieMetadataPayload implements 
HoodieRecordPayload<HoodieMetadata
   public static final String RECORD_INDEX_FIELD_FILEID_ENCODING = 
"fileIdEncoding";
   public static final int RECORD_INDEX_FIELD_FILEID_ENCODING_UUID = 0;
   public static final int RECORD_INDEX_FIELD_FILEID_ENCODING_RAW_STRING = 1;
+  public static final String RECORD_INDEX_FIELD_POSITION = "position";
 
   /**
    * FileIndex value saved in record index record when the fileId has no index 
(old format of base filename)
@@ -261,13 +262,15 @@ public class HoodieMetadataPayload implements 
HoodieRecordPayload<HoodieMetadata
         }
       } else if (type == METADATA_TYPE_RECORD_INDEX) {
         GenericRecord recordIndexRecord = getNestedFieldValue(record, 
SCHEMA_FIELD_ID_RECORD_INDEX);
+        Object recordIndexPosition = 
recordIndexRecord.get(RECORD_INDEX_FIELD_POSITION);
         recordIndexMetadata = new 
HoodieRecordIndexInfo(recordIndexRecord.get(RECORD_INDEX_FIELD_PARTITION).toString(),
             
Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_HIGH_BITS).toString()),
             
Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_LOW_BITS).toString()),
             
Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILE_INDEX).toString()),
             recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID).toString(),
             
Long.parseLong(recordIndexRecord.get(RECORD_INDEX_FIELD_INSTANT_TIME).toString()),
-            
Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_ENCODING).toString()));
+            
Integer.parseInt(recordIndexRecord.get(RECORD_INDEX_FIELD_FILEID_ENCODING).toString()),
+            recordIndexPosition != null ? 
Long.parseLong(recordIndexPosition.toString()) : null);
       } else if (type == METADATA_TYPE_SECONDARY_INDEX) {
         GenericRecord secondaryIndexRecord = getNestedFieldValue(record, 
SCHEMA_FIELD_ID_SECONDARY_INDEX);
         checkState(secondaryIndexRecord != null, "Valid SecondaryIndexMetadata 
record expected for type: " + METADATA_TYPE_SECONDARY_INDEX);
@@ -797,7 +800,8 @@ public class HoodieMetadataPayload implements 
HoodieRecordPayload<HoodieMetadata
               fileIndex,
               "",
               instantTimeMillis,
-              0));
+              0,
+              null));
       return new HoodieAvroRecord<>(key, payload);
     } else {
       HoodieMetadataPayload payload = new HoodieMetadataPayload(recordKey,
@@ -808,7 +812,8 @@ public class HoodieMetadataPayload implements 
HoodieRecordPayload<HoodieMetadata
               -1,
               fileId,
               instantTimeMillis,
-              1));
+              1,
+              null));
       return new HoodieAvroRecord<>(key, payload);
     }
   }