This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit ff5c31ef78138a33695305279eafb5de34d4e9f7 Author: Vova Kolmakov <[email protected]> AuthorDate: Mon Apr 15 17:48:32 2024 +0700 [HUDI-7619] Removed code duplicates in HoodieTableMetadataUtil (#11022) Co-authored-by: Vova Kolmakov <[email protected]> --- .../hudi/metadata/HoodieTableMetadataUtil.java | 97 +++++++++------------- 1 file changed, 41 insertions(+), 56 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index b25d6741b83..cc070a2b531 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -1749,26 +1749,7 @@ public class HoodieTableMetadataUtil { final String instantTime = baseFile.getCommitTime(); HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO) .getFileReader(config, configuration.get(), dataFilePath); - ClosableIterator<String> recordKeyIterator = reader.getRecordKeyIterator(); - - return new ClosableIterator<HoodieRecord>() { - @Override - public void close() { - recordKeyIterator.close(); - } - - @Override - public boolean hasNext() { - return recordKeyIterator.hasNext(); - } - - @Override - public HoodieRecord next() { - return forDelete - ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); - } - }; + return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime); }); } @@ -1816,24 +1797,7 @@ public class HoodieTableMetadataUtil { .withTableMetaClient(metaClient) .build(); ClosableIterator<String> recordKeyIterator = ClosableIterator.wrap(mergedLogRecordScanner.getRecords().keySet().iterator()); - return new ClosableIterator<HoodieRecord>() { - @Override - public void close() { - recordKeyIterator.close(); - } - - @Override - public boolean hasNext() { - return recordKeyIterator.hasNext(); - } - - @Override - public HoodieRecord next() { - return forDelete - ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileSlice.getFileId(), fileSlice.getBaseInstantTime(), 0); - } - }; + return getHoodieRecordIterator(recordKeyIterator, forDelete, partition, fileSlice.getFileId(), fileSlice.getBaseInstantTime()); } final HoodieBaseFile baseFile = fileSlice.getBaseFile().get(); final String filename = baseFile.getFileName(); @@ -1844,26 +1808,47 @@ public class HoodieTableMetadataUtil { HoodieConfig hoodieConfig = getReaderConfigs(configuration.get()); HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO) .getFileReader(hoodieConfig, configuration.get(), dataFilePath); - ClosableIterator<String> recordKeyIterator = reader.getRecordKeyIterator(); + return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime); + }); + } - return new ClosableIterator<HoodieRecord>() { - @Override - public void close() { - recordKeyIterator.close(); - } + public static Schema getProjectedSchemaForFunctionalIndex(HoodieFunctionalIndexDefinition indexDefinition, HoodieTableMetaClient metaClient) throws Exception { + TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient); + Schema tableSchema = schemaResolver.getTableAvroSchema(); + return addMetadataFields(getSchemaForFields(tableSchema, indexDefinition.getSourceFields())); + } - @Override - public boolean hasNext() { - return recordKeyIterator.hasNext(); - } + private static Path filePath(String basePath, String partition, String filename) { + if (partition.isEmpty()) { + return new Path(basePath, filename); + } else { + return new Path(basePath, partition + StoragePath.SEPARATOR + filename); + } + } - @Override - public HoodieRecord next() { - return forDelete - ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); - } - }; - }); + private static ClosableIterator<HoodieRecord> getHoodieRecordIterator(ClosableIterator<String> recordKeyIterator, + boolean forDelete, + String partition, + String fileId, + String instantTime + ) { + return new ClosableIterator<HoodieRecord>() { + @Override + public void close() { + recordKeyIterator.close(); + } + + @Override + public boolean hasNext() { + return recordKeyIterator.hasNext(); + } + + @Override + public HoodieRecord next() { + return forDelete + ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) + : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); + } + }; } }
