danny0405 commented on code in PR #13229:
URL: https://github.com/apache/hudi/pull/13229#discussion_r2084350839
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java:
##########
@@ -318,25 +320,39 @@ protected HoodieWriteMetadata<O> compact(HoodieTable<?,
I, ?, T> table, String c
}
compactionTimer = metrics.getCompactionCtx();
HoodieWriteMetadata<T> writeMetadata = table.compact(context,
compactionInstantTime);
- HoodieWriteMetadata<O> compactionMetadata =
convertToOutputMetadata(writeMetadata);
- if (shouldComplete && compactionMetadata.getCommitMetadata().isPresent()) {
- completeCompaction(compactionMetadata.getCommitMetadata().get(), table,
compactionInstantTime);
+ HoodieWriteMetadata<O> compactionWriteMetadata =
convertToOutputMetadata(writeMetadata);
+ if (shouldComplete) {
+ commitCompaction(compactionInstantTime, compactionWriteMetadata,
Option.of(table));
}
- return compactionMetadata;
+ return compactionWriteMetadata;
}
- /**
- * Commit a compaction operation. Allow passing additional meta-data to be
stored in commit instant file.
- *
- * @param compactionInstantTime Compaction Instant Time
- * @param metadata All the metadata that gets stored along with
a commit
- * @param extraMetadata Extra Metadata to be stored
- */
- public void commitCompaction(String compactionInstantTime,
HoodieCommitMetadata metadata, Option<Map<String, String>> extraMetadata) {
- extraMetadata.ifPresent(m -> m.forEach(metadata::addMetadata));
- completeCompaction(metadata, createTable(config,
context.getStorageConf()), compactionInstantTime);
+ public void commitCompaction(String compactionInstantTime,
HoodieWriteMetadata<O> compactionWriteMetadata, Option<HoodieTable> tableOpt) {
+ // dereferencing the write dag for compaction for the first time.
+ List<HoodieWriteStat> writeStats =
triggerWritesAndFetchWriteStats(compactionWriteMetadata);
+ // Fetch commit metadata from HoodieWriteMetadata and update
HoodieWriteStat
+ stitchHoodieWriteStats(compactionWriteMetadata, writeStats);
+ metrics.emitCompactionCompleted();
+ LOG.info("Compaction completed. Instant time: {}.", compactionInstantTime);
+
+ HoodieTable table = tableOpt.orElseGet(() -> createTable(config,
context.getStorageConf()));
+ completeCompaction(compactionWriteMetadata.getCommitMetadata().get(),
table, compactionInstantTime);
}
+ private HoodieWriteMetadata stitchHoodieWriteStats(HoodieWriteMetadata<O>
writeMetadata, List<HoodieWriteStat> writeStats) {
+ // Fetch commit metadata from HoodieWriteMetadata and update
HoodieWriteStat
+ HoodieCommitMetadata commitMetadata =
writeMetadata.getCommitMetadata().get();
+ commitMetadata.setCompacted(true);
+ for (HoodieWriteStat stat : writeStats) {
+ commitMetadata.addWriteStat(stat.getPartitionPath(), stat);
+ }
+ writeMetadata.setCommitted(true);
+ writeMetadata.setCommitMetadata(Option.of(commitMetadata));
+ return writeMetadata;
+ }
+
+ protected abstract List<HoodieWriteStat>
triggerWritesAndFetchWriteStats(HoodieWriteMetadata<O> writeMetadata);
Review Comment:
Add doc for this method.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
