lokeshj1703 commented on code in PR #13286:
URL: https://github.com/apache/hudi/pull/13286#discussion_r2112352787
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -1094,6 +1116,137 @@ public void buildMetadataPartitions(HoodieEngineContext
engineContext, List<Hood
initializeFromFilesystem(instantTime, partitionTypes, Option.empty());
}
+ public void startCommit(String instantTime) {
Review Comment:
Javadoc already exists in the interface. Added validation.
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -669,37 +685,38 @@ private Pair<Integer, HoodieData<HoodieRecord>>
initializeSecondaryIndexPartitio
return Pair.of(fileGroupCount, records);
}
- private Lazy<List<Pair<String, FileSlice>>>
getLazyLatestMergedPartitionFileSliceList() {
- return Lazy.lazily(() -> {
- String latestInstant =
dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
- .map(HoodieInstant::requestedTime).orElse(SOLO_COMMIT_TIMESTAMP);
- try (HoodieTableFileSystemView fsView = getMetadataView()) {
- // Collect the list of latest file slices present in each partition
- List<String> partitions = metadata.getAllPartitionPaths();
- fsView.loadAllPartitions();
- List<Pair<String, FileSlice>> partitionFileSlicePairs = new
ArrayList<>();
- partitions.forEach(partition ->
fsView.getLatestMergedFileSlicesBeforeOrOn(partition, latestInstant)
- .forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition,
fs))));
- return partitionFileSlicePairs;
- } catch (IOException e) {
- throw new HoodieIOException("Cannot get the latest merged file
slices", e);
- }
- });
+ private List<Pair<String, FileSlice>> getPartitionFileSlicePairs() throws
IOException {
+ String latestInstant =
dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
+ .map(HoodieInstant::requestedTime).orElse(SOLO_COMMIT_TIMESTAMP);
+ try (HoodieTableFileSystemView fsView = getMetadataView()) {
+ // Collect the list of latest file slices present in each partition
+ List<String> partitions = metadata.getAllPartitionPaths();
+ fsView.loadAllPartitions();
+ List<Pair<String, FileSlice>> partitionFileSlicePairs = new
ArrayList<>();
+ partitions.forEach(partition ->
fsView.getLatestMergedFileSlicesBeforeOrOn(partition, latestInstant)
+ .forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs))));
+ return partitionFileSlicePairs;
+ }
}
- private Pair<Integer, HoodieData<HoodieRecord>>
initializeRecordIndexPartition(
- Lazy<List<Pair<String, FileSlice>>>
lazyLatestMergedPartitionFileSliceList) {
+ private Pair<Integer, HoodieData<HoodieRecord>>
initializeRecordIndexPartition() throws IOException {
+ final HoodieTableFileSystemView fsView = getMetadataView();
Review Comment:
Retained it
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -417,10 +438,9 @@ private boolean initializeFromFilesystem(String
dataTableInstantTime, List<Metad
}
}
- Lazy<List<Pair<String, FileSlice>>> lazyLatestMergedPartitionFileSliceList
= getLazyLatestMergedPartitionFileSliceList();
Review Comment:
Using the same now
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -388,17 +409,17 @@ private boolean initializeFromFilesystem(String
dataTableInstantTime, List<Metad
// Get a complete list of files and partitions from the file system or
from already initialized FILES partition of MDT
List<DirectoryInfo> partitionInfoList;
if (filesPartitionAvailable) {
- partitionInfoList = listAllPartitionsFromMDT(dataTableInstantTime,
pendingDataInstants);
+ partitionInfoList = listAllPartitionsFromMDT(initializationTime,
pendingDataInstants);
} else {
// if auto initialization is enabled, then we need to list all
partitions from the file system
if (dataWriteConfig.getMetadataConfig().shouldAutoInitialize()) {
- partitionInfoList =
listAllPartitionsFromFilesystem(dataTableInstantTime, pendingDataInstants);
+ partitionInfoList =
listAllPartitionsFromFilesystem(initializationTime, pendingDataInstants);
} else {
// if auto initialization is disabled, we can return an empty list
partitionInfoList = Collections.emptyList();
}
}
- Map<String, Map<String, Long>> partitionIdToAllFilesMap =
partitionInfoList.stream()
+ Map<String, Map<String, Long>> partitionToFilesMap =
partitionInfoList.stream()
Review Comment:
Addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
