voonhous commented on code in PR #18816:
URL: https://github.com/apache/hudi/pull/18816#discussion_r3956832989
##########
hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java:
##########
@@ -107,9 +123,182 @@ public String showArchivedCommits(
throws IOException {
System.out.println("===============> Showing only " + limit + " archived
commits <===============");
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
- StoragePath archivePath = folder != null && !folder.isEmpty()
- ? new StoragePath(metaClient.getMetaPath(), folder)
- : new StoragePath(metaClient.getArchivePath(), ".commits_.archive*");
+ List<Comparable[]> allStats;
+ if (folder != null && !folder.isEmpty()) {
+ allStats = readCommitStatsFromLegacyArchive(metaClient, new
StoragePath(metaClient.getMetaPath(), folder));
+ } else if (isLegacyArchive(metaClient)) {
+ allStats = readCommitStatsFromLegacyArchive(
+ metaClient, new StoragePath(metaClient.getArchivePath(),
".commits_.archive*"));
+ } else {
+ allStats =
readCommitStatsFromArchivedTimeline(newArchivedTimeline(metaClient),
sortByField, limit);
+ }
+ TableHeader header = new
TableHeader().addTableHeaderField("action").addTableHeaderField("instant")
+
.addTableHeaderField("partition").addTableHeaderField("file_id").addTableHeaderField("prev_instant")
+
.addTableHeaderField("num_writes").addTableHeaderField("num_inserts").addTableHeaderField("num_deletes")
+
.addTableHeaderField("num_update_writes").addTableHeaderField("total_log_files")
+
.addTableHeaderField("total_log_blocks").addTableHeaderField("total_corrupt_log_blocks")
+
.addTableHeaderField("total_rollback_blocks").addTableHeaderField("total_log_records")
+
.addTableHeaderField("total_updated_records_compacted").addTableHeaderField("total_write_bytes")
+ .addTableHeaderField("total_write_errors");
+
+ return HoodiePrintHelper.print(header, new HashMap<>(), sortByField,
descending, limit, headerOnly, allStats);
+ }
+
+ @ShellMethod(key = "show archived commits", value = "Read commits from
archived files and show details")
+ public String showCommits(
+ @ShellOption(value = {"--skipMetadata"}, help = "Skip displaying commit
metadata",
+ defaultValue = "true") boolean skipMetadata,
+ @ShellOption(value = {"--limit"}, help = "Limit commits", defaultValue =
"10") final Integer limit,
+ @ShellOption(value = {"--sortBy"}, help = "Sorting Field", defaultValue
= "") final String sortByField,
+ @ShellOption(value = {"--desc"}, help = "Ordering", defaultValue =
"false") final boolean descending,
+ @ShellOption(value = {"--headeronly"}, help = "Print Header Only",
+ defaultValue = "false") final boolean headerOnly) {
+
+ System.out.println("===============> Showing only " + limit + " archived
commits <===============");
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
+ List<Comparable[]> allCommits = readArchivedCommits(
+ newArchivedTimeline(metaClient), skipMetadata,
isLegacyArchive(metaClient), sortByField, limit);
+
+ TableHeader header = new
TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("CommitType");
+
+ if (!skipMetadata) {
+ header = header.addTableHeaderField("CommitDetails");
+ }
+
+ return HoodiePrintHelper.print(header, new HashMap<>(), sortByField,
descending, limit, headerOnly, allCommits);
+ }
+
+ /**
+ * Renders the completed archived instants as rows, loading metadata only
for the rows that
+ * can reach the output.
+ * <p>
+ * Without a sort field the printer keeps the timeline order and cuts at the
limit, so only
+ * the leading instants can be shown, and an archive that has grown for
years holds far more
+ * payload than those few rows need. With a sort field, or no limit, every
row takes part
+ * and everything has to be loaded.
+ */
+ @VisibleForTesting
+ static List<Comparable[]> readArchivedCommits(HoodieArchivedTimeline
archivedTimeline, boolean skipMetadata,
+ boolean legacyArchive, String
sortByField, int limit) {
+ List<HoodieInstant> completed = archivedTimeline.getInstants().stream()
+ .filter(HoodieInstant::isCompleted)
+ .collect(Collectors.toList());
+ List<HoodieInstant> shown = boundedByLimit(completed, sortByField, limit);
+ if (!skipMetadata) {
+ loadInstantDetails(archivedTimeline, shown, shown.size() <
completed.size());
+ }
+ return shown.stream()
+ .map(instant -> readArchivedCommit(archivedTimeline, instant,
skipMetadata, legacyArchive))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Returns the leading instants that the printer can render, or all of them
when a sort field
+ * or a non-positive limit makes every row a candidate.
+ */
+ private static List<HoodieInstant> boundedByLimit(List<HoodieInstant>
instants, String sortByField, int limit) {
+ if (!sortByField.isEmpty() || limit <= 0 || limit >= instants.size()) {
+ return instants;
+ }
+ return instants.subList(0, limit);
+ }
+
+ /**
+ * Loads the details of the given instants, through the closed time range
they span when they
+ * are a strict subset of the archive, so that the archive files outside the
range are never
Review Comment:
Right. `ArchivedTimelineV1.loadCompletedInstantDetailsInMemory(startTs,
endTs)` passes a null log-file filter, and
`ArchivedTimelineLoaderV1.loadInstants` globs `.commits_.archive*` and opens
every one of them, applying `filter.isInRange` per record after the block is
read. Only `ArchivedTimelineLoaderV2` prunes up front, through
`LSMTimeline.isFileInRange` in `getFilteredFiles`.
Narrowed the sentence: the payload bound holds on both layouts, the file
bound only on the LSM one, and the legacy layout now says outright that it
reads every archive file either way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
