This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8a1b6f3845c611dda6a8d5480e383a072bf2759c Author: Vova Kolmakov <[email protected]> AuthorDate: Tue May 14 16:01:09 2024 -0700 [HUDI-7619] Removed code duplicates in HoodieTableMetadataUtil (#11022) Co-authored-by: Vova Kolmakov <[email protected]> --- .../hudi/metadata/HoodieTableMetadataUtil.java | 92 +++++++++------------- 1 file changed, 36 insertions(+), 56 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java index b25d6741b83..503e3351d8c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java @@ -73,6 +73,7 @@ import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.io.storage.HoodieFileReader; import org.apache.hudi.io.storage.HoodieFileReaderFactory; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.util.Lazy; import org.apache.avro.AvroTypeException; @@ -1749,26 +1750,7 @@ public class HoodieTableMetadataUtil { final String instantTime = baseFile.getCommitTime(); HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO) .getFileReader(config, configuration.get(), dataFilePath); - ClosableIterator<String> recordKeyIterator = reader.getRecordKeyIterator(); - - return new ClosableIterator<HoodieRecord>() { - @Override - public void close() { - recordKeyIterator.close(); - } - - @Override - public boolean hasNext() { - return recordKeyIterator.hasNext(); - } - - @Override - public HoodieRecord next() { - return forDelete - ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); - } - }; + return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime); }); } @@ -1816,24 +1798,7 @@ public class HoodieTableMetadataUtil { .withTableMetaClient(metaClient) .build(); ClosableIterator<String> recordKeyIterator = ClosableIterator.wrap(mergedLogRecordScanner.getRecords().keySet().iterator()); - return new ClosableIterator<HoodieRecord>() { - @Override - public void close() { - recordKeyIterator.close(); - } - - @Override - public boolean hasNext() { - return recordKeyIterator.hasNext(); - } - - @Override - public HoodieRecord next() { - return forDelete - ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileSlice.getFileId(), fileSlice.getBaseInstantTime(), 0); - } - }; + return getHoodieRecordIterator(recordKeyIterator, forDelete, partition, fileSlice.getFileId(), fileSlice.getBaseInstantTime()); } final HoodieBaseFile baseFile = fileSlice.getBaseFile().get(); final String filename = baseFile.getFileName(); @@ -1844,26 +1809,41 @@ public class HoodieTableMetadataUtil { HoodieConfig hoodieConfig = getReaderConfigs(configuration.get()); HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO) .getFileReader(hoodieConfig, configuration.get(), dataFilePath); - ClosableIterator<String> recordKeyIterator = reader.getRecordKeyIterator(); + return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime); + }); + } - return new ClosableIterator<HoodieRecord>() { - @Override - public void close() { - recordKeyIterator.close(); - } + private static Path filePath(String basePath, String partition, String filename) { + if (partition.isEmpty()) { + return new Path(basePath, filename); + } else { + return new Path(basePath, partition + StoragePath.SEPARATOR + filename); + } + } - @Override - public boolean hasNext() { - return recordKeyIterator.hasNext(); - } + private static ClosableIterator<HoodieRecord> getHoodieRecordIterator(ClosableIterator<String> recordKeyIterator, + boolean forDelete, + String partition, + String fileId, + String instantTime + ) { + return new ClosableIterator<HoodieRecord>() { + @Override + public void close() { + recordKeyIterator.close(); + } - @Override - public HoodieRecord next() { - return forDelete - ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) - : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); - } - }; - }); + @Override + public boolean hasNext() { + return recordKeyIterator.hasNext(); + } + + @Override + public HoodieRecord next() { + return forDelete + ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next()) + : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0); + } + }; } }
