codope commented on code in PR #11146:
URL: https://github.com/apache/hudi/pull/11146#discussion_r1619691280
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -525,21 +534,66 @@ private Pair<Integer, HoodieData<HoodieRecord>>
initializeFunctionalIndexPartiti
}
private Set<String> getFunctionalIndexPartitionsToInit() {
- Set<String> functionalIndexPartitions =
dataMetaClient.getFunctionalIndexMetadata().get().getIndexDefinitions().keySet();
+ Set<String> functionalIndexPartitions =
dataMetaClient.getIndexMetadata().get().getIndexDefinitions().keySet();
Set<String> completedMetadataPartitions =
dataMetaClient.getTableConfig().getMetadataPartitions();
functionalIndexPartitions.removeAll(completedMetadataPartitions);
return functionalIndexPartitions;
}
- private HoodieFunctionalIndexDefinition getFunctionalIndexDefinition(String
indexName) {
- Option<HoodieFunctionalIndexMetadata> functionalIndexMetadata =
dataMetaClient.getFunctionalIndexMetadata();
+ private HoodieIndexDefinition getFunctionalIndexDefinition(String indexName)
{
+ Option<HoodieIndexMetadata> functionalIndexMetadata =
dataMetaClient.getIndexMetadata();
if (functionalIndexMetadata.isPresent()) {
return
functionalIndexMetadata.get().getIndexDefinitions().get(indexName);
} else {
throw new HoodieIndexException("Functional Index definition is not
present");
}
}
+ private Set<String> getSecondaryIndexPartitionsToInit() {
+ Set<String> secondaryIndexPartitions =
dataMetaClient.getIndexMetadata().get().getIndexDefinitions().values().stream()
+ .map(HoodieIndexDefinition::getIndexName)
+ .filter(indexName ->
indexName.startsWith(HoodieTableMetadataUtil.PARTITION_NAME_SECONDARY_INDEX_PREFIX))
+ .collect(Collectors.toSet());
+ Set<String> completedMetadataPartitions =
dataMetaClient.getTableConfig().getMetadataPartitions();
+ secondaryIndexPartitions.removeAll(completedMetadataPartitions);
+ return secondaryIndexPartitions;
+ }
+
+ private Pair<Integer, HoodieData<HoodieRecord>>
initializeSecondaryIndexPartition(String indexName) throws IOException {
+ HoodieIndexDefinition indexDefinition =
getFunctionalIndexDefinition(indexName);
+ ValidationUtils.checkState(indexDefinition != null, "Secondary Index
definition is not present for index " + indexName);
+ List<Pair<String, FileSlice>> partitionFileSlicePairs =
getPartitionFileSlicePairs();
+
+ // Reuse record index parallelism config to build secondary index
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
