voonhous commented on code in PR #18816:
URL: https://github.com/apache/hudi/pull/18816#discussion_r3888876892
##########
hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java:
##########
@@ -156,94 +268,67 @@ public String showArchivedCommits(
allStats.addAll(readCommits);
}
}
- TableHeader header = new
TableHeader().addTableHeaderField("action").addTableHeaderField("instant")
-
.addTableHeaderField("partition").addTableHeaderField("file_id").addTableHeaderField("prev_instant")
-
.addTableHeaderField("num_writes").addTableHeaderField("num_inserts").addTableHeaderField("num_deletes")
-
.addTableHeaderField("num_update_writes").addTableHeaderField("total_log_files")
-
.addTableHeaderField("total_log_blocks").addTableHeaderField("total_corrupt_log_blocks")
-
.addTableHeaderField("total_rollback_blocks").addTableHeaderField("total_log_records")
-
.addTableHeaderField("total_updated_records_compacted").addTableHeaderField("total_write_bytes")
- .addTableHeaderField("total_write_errors");
-
- return HoodiePrintHelper.print(header, new HashMap<>(), sortByField,
descending, limit, headerOnly, allStats);
+ return allStats;
}
- @ShellMethod(key = "show archived commits", value = "Read commits from
archived files and show details")
- public String showCommits(
- @ShellOption(value = {"--skipMetadata"}, help = "Skip displaying commit
metadata",
- defaultValue = "true") boolean skipMetadata,
- @ShellOption(value = {"--limit"}, help = "Limit commits", defaultValue =
"10") final Integer limit,
- @ShellOption(value = {"--sortBy"}, help = "Sorting Field", defaultValue
= "") final String sortByField,
- @ShellOption(value = {"--desc"}, help = "Ordering", defaultValue =
"false") final boolean descending,
- @ShellOption(value = {"--headeronly"}, help = "Print Header Only",
- defaultValue = "false") final boolean headerOnly)
- throws IOException {
-
- System.out.println("===============> Showing only " + limit + " archived
commits <===============");
- HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
- StoragePath archivePath =
- new StoragePath(metaClient.getArchivePath(), ".commits_.archive*");
- HoodieStorage storage = metaClient.getStorage();
- List<StoragePathInfo> pathInfoList = storage.globEntries(archivePath);
- List<Comparable[]> allCommits = new ArrayList<>();
- for (StoragePathInfo pathInfo : pathInfoList) {
- // read the archived file
- try (HoodieLogFormat.Reader reader =
HoodieLogFormat.newReader(metaClient,
- new HoodieLogFile(pathInfo.getPath()),
HoodieSchema.fromAvroSchema(HoodieArchivedMetaEntry.getClassSchema()))) {
- List<IndexedRecord> readRecords = new ArrayList<>();
- // read the avro blocks
- while (reader.hasNext()) {
- HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
- try (ClosableIterator<HoodieRecord<IndexedRecord>> recordItr =
blk.getRecordIterator(HoodieRecordType.AVRO)) {
- recordItr.forEachRemaining(r -> readRecords.add(r.getData()));
- }
- }
- List<Comparable[]> readCommits = readRecords.stream().map(r ->
(GenericRecord) r)
- .map(r -> readCommit(r,
skipMetadata)).collect(Collectors.toList());
- allCommits.addAll(readCommits);
- }
- }
-
- TableHeader header = new
TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("CommitType");
-
+ private Comparable[] readArchivedCommit(HoodieArchivedTimeline
archivedTimeline, HoodieInstant instant, boolean skipMetadata) {
+ List<Comparable> commitDetails = new ArrayList<>();
+ commitDetails.add(instant.requestedTime());
+ commitDetails.add(instant.getAction());
if (!skipMetadata) {
- header = header.addTableHeaderField("CommitDetails");
- }
-
- return HoodiePrintHelper.print(header, new HashMap<>(), sortByField,
descending, limit, headerOnly, allCommits);
- }
-
- private Comparable[] commitDetail(GenericRecord record, String metadataName,
boolean skipMetadata) {
- List<Object> commitDetails = new ArrayList<>();
- commitDetails.add(record.get("commitTime"));
- commitDetails.add(record.get("actionType").toString());
- if (!skipMetadata) {
-
commitDetails.add(Option.ofNullable(record.get(metadataName)).orElse("{}").toString());
+ commitDetails.add(readArchivedMetadataString(archivedTimeline, instant));
}
return commitDetails.toArray(new Comparable[commitDetails.size()]);
}
- private Comparable[] readCommit(GenericRecord record, boolean skipMetadata) {
- String actionType = record.get("actionType").toString();
- switch (actionType) {
- case HoodieTimeline.CLEAN_ACTION:
- return commitDetail(record, "hoodieCleanMetadata", skipMetadata);
- case HoodieTimeline.COMMIT_ACTION:
- case HoodieTimeline.DELTA_COMMIT_ACTION:
- return commitDetail(record, "hoodieCommitMetadata", skipMetadata);
- case HoodieTimeline.ROLLBACK_ACTION:
- return commitDetail(record, "hoodieRollbackMetadata", skipMetadata);
- case HoodieTimeline.SAVEPOINT_ACTION:
- return commitDetail(record, "hoodieSavePointMetadata", skipMetadata);
- case HoodieTimeline.COMPACTION_ACTION:
- return commitDetail(record, "hoodieCompactionMetadata", skipMetadata);
- case HoodieTimeline.REPLACE_COMMIT_ACTION:
- case HoodieTimeline.CLUSTERING_ACTION:
- return commitDetail(record, "hoodieReplaceCommitMetadata",
skipMetadata);
- default: {
- throw new HoodieException("Unexpected action type: " + actionType);
+ private String readArchivedMetadataString(HoodieArchivedTimeline
archivedTimeline, HoodieInstant instant) {
+ Option<byte[]> details = archivedTimeline.getInstantDetails(instant);
+ if (!details.isPresent() || details.get().length == 0) {
+ // instants can be archived with no metadata, e.g. from an empty
completed
+ // meta file that a writer failure left behind
+ return "{}";
+ }
+ try {
+ switch (instant.getAction()) {
+ case HoodieTimeline.CLEAN_ACTION:
+ return archivedTimeline.readCleanMetadata(instant).toString();
+ case HoodieTimeline.COMMIT_ACTION:
+ case HoodieTimeline.DELTA_COMMIT_ACTION:
+ return
sortPartitions(archivedTimeline.readCommitMetadataToAvro(instant)).toString();
+ case HoodieTimeline.ROLLBACK_ACTION:
+ return archivedTimeline.readRollbackMetadata(instant).toString();
+ case HoodieTimeline.SAVEPOINT_ACTION:
+ return archivedTimeline.readSavepointMetadata(instant).toString();
+ case HoodieTimeline.COMPACTION_ACTION:
Review Comment:
Done in b85941ea: the four arms are removed.
##########
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java:
##########
@@ -227,19 +222,29 @@ public void testOverwriteHoodieProperties() throws
IOException {
.collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e ->
String.valueOf(e.getValue())));
expected.putIfAbsent(TABLE_CHECKSUM.key(),
String.valueOf(generateChecksum(tableConfig.getProps())));
expected.putIfAbsent(DROP_PARTITION_COLUMNS.key(),
String.valueOf(DROP_PARTITION_COLUMNS.defaultValue()));
+
+ // Add properties that are now present in Hudi 1.x by default
+ if (result.containsKey(HoodieTableConfig.TIMELINE_PATH.key())) {
+ expected.putIfAbsent(HoodieTableConfig.TIMELINE_PATH.key(),
result.get(HoodieTableConfig.TIMELINE_PATH.key()));
Review Comment:
Done in 059d0af2: constants for the three keys, and the exact rendered table
over the old/new key union.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
