danny0405 commented on code in PR #13229:
URL: https://github.com/apache/hudi/pull/13229#discussion_r2101940734
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java:
##########
@@ -515,22 +524,54 @@ public boolean purgePendingClustering(String
clusteringInstant) {
}
/**
- * Delete expired partition by config.
- *
- * @param instantTime Instant Time for the action
- * @return HoodieWriteMetadata
+ * The API changes the input write metadata from type T to O.
*/
- public HoodieWriteMetadata<T> managePartitionTTL(String instantTime) {
- HoodieTable<?, I, ?, T> table = createTable(config,
context.getStorageConf());
- return table.managePartitionTTL(context, instantTime);
+ protected abstract HoodieWriteMetadata<O>
convertToOutputMetadata(HoodieWriteMetadata<T> writeMetadata);
+
+ /**
+ * Check if any validators are configured and run those validations. If any
of the validations fail, throws HoodieValidationException.
+ */
+ protected void runPrecommitValidationForClustering(HoodieWriteMetadata<O>
writeMetadata, HoodieTable table, String instantTime) {
+ if (StringUtils.isNullOrEmpty(config.getPreCommitValidators())) {
+ return;
+ }
+ throw new HoodieIOException("Precommit validation not implemented for all
engines yet");
}
- protected abstract HoodieWriteMetadata<O>
convertToOutputMetadata(HoodieWriteMetadata<T> writeMetadata);
+ private void commitClustering(HoodieWriteMetadata<O>
clusteringWriteMetadata, HoodieTable table, String clusteringCommitTime) {
+ // triggering the dag for the first time for clustering
+ List<HoodieWriteStat> writeStats =
triggerWritesAndFetchWriteStats(clusteringWriteMetadata);
+ clusteringWriteMetadata.setWriteStats(writeStats);
+ // Fetch Replace commit metadata and update HoodieWriteStats annd
Partition to Replace FileIds
+ HoodieReplaceCommitMetadata replaceCommitMetadata =
(HoodieReplaceCommitMetadata) clusteringWriteMetadata.getCommitMetadata().get();
+ for (HoodieWriteStat writeStat: writeStats) {
+ replaceCommitMetadata.addWriteStat(writeStat.getPartitionPath(),
writeStat);
+ }
+ HoodieClusteringPlan clusteringPlan =
ClusteringUtils.getPendingClusteringPlan(table.getMetaClient(),
clusteringCommitTime);
+ Map<String, List<String>> partitionToReplaceFileIds =
CommonClientUtils.getPartitionToReplacedFileIds(clusteringPlan,
clusteringWriteMetadata, config);
+
clusteringWriteMetadata.setPartitionToReplaceFileIds(partitionToReplaceFileIds);
+
replaceCommitMetadata.setPartitionToReplaceFileIds(partitionToReplaceFileIds);
+
+
clusteringWriteMetadata.setCommitMetadata(Option.of(replaceCommitMetadata));
Review Comment:
No need to set because it should be the same obj.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
