n3nash commented on a change in pull request #1318: [HUDI-571] CLI - add option
to print additional commit metadata
URL: https://github.com/apache/incubator-hudi/pull/1318#discussion_r378435736
##########
File path:
hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
##########
@@ -100,8 +100,58 @@ private String printCommits(HoodieDefaultTimeline
timeline,
return HoodiePrintHelper.print(header, fieldNameToConverterMap,
sortByField, descending, limit, headerOnly, rows);
}
+ private String printCommitsWithMetadata(HoodieDefaultTimeline timeline,
+ final Integer limit, final String sortByField,
+ final boolean descending,
+ final boolean headerOnly) throws IOException {
+ final List<Comparable[]> rows = new ArrayList<>();
+
+ final List<HoodieInstant> commits =
timeline.getCommitsTimeline().filterCompletedInstants()
+ .getInstants().collect(Collectors.toList());
+ // timeline can be read from multiple files. So sort is needed instead of
reversing the collection
+ Collections.sort(commits, HoodieInstant.COMPARATOR.reversed());
+
+ for (int i = 0; i < commits.size(); i++) {
+ final HoodieInstant commit = commits.get(i);
+ final HoodieCommitMetadata commitMetadata =
HoodieCommitMetadata.fromBytes(
+ timeline.getInstantDetails(commit).get(),
+ HoodieCommitMetadata.class);
+
+ for (Map.Entry<String, List<HoodieWriteStat>> partitionWriteStat :
+ commitMetadata.getPartitionToWriteStats().entrySet()) {
+ for (HoodieWriteStat hoodieWriteStat : partitionWriteStat.getValue()) {
+ rows.add(new Comparable[]{ commit.getAction(),
commit.getTimestamp(), hoodieWriteStat.getPartitionPath(),
+ hoodieWriteStat.getFileId(),
hoodieWriteStat.getPrevCommit(), hoodieWriteStat.getNumWrites(),
+ hoodieWriteStat.getNumInserts(),
hoodieWriteStat.getNumDeletes(),
+ hoodieWriteStat.getNumUpdateWrites(),
hoodieWriteStat.getTotalWriteErrors(),
+ hoodieWriteStat.getTotalLogBlocks(),
hoodieWriteStat.getTotalCorruptLogBlock(),
+ hoodieWriteStat.getTotalRollbackBlocks(),
hoodieWriteStat.getTotalLogRecords(),
+ hoodieWriteStat.getTotalUpdatedRecordsCompacted(),
hoodieWriteStat.getTotalWriteBytes()
+ });
+ }
+ }
+ }
+
+ final Map<String, Function<Object, String>> fieldNameToConverterMap = new
HashMap<>();
+ fieldNameToConverterMap.put("Total Bytes Written", entry -> {
+ return
NumericUtils.humanReadableByteCount((Double.valueOf(entry.toString())));
+ });
+
+ TableHeader header = new
TableHeader().addTableHeaderField("action").addTableHeaderField("instant")
+
.addTableHeaderField("partition").addTableHeaderField("file_id").addTableHeaderField("prev_instant")
+
.addTableHeaderField("num_writes").addTableHeaderField("num_inserts").addTableHeaderField("num_deletes")
Review comment:
Do we add names in snake_case in other header commands ?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
