This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 22666dc01fca feat: add CLI command to show inflight instants older
than specified duration (#17511)
22666dc01fca is described below
commit 22666dc01fca746ae4c566806beba6e0f262e152
Author: Surya Prasanna <[email protected]>
AuthorDate: Wed Dec 10 15:09:00 2025 -0800
feat: add CLI command to show inflight instants older than specified
duration (#17511)
Co-authored-by: Shawn Chang <[email protected]>
---
.../apache/hudi/cli/commands/CommitsCommand.java | 33 ++++++++++
.../hudi/cli/commands/TestCommitsCommand.java | 70 ++++++++++++++++++++++
.../common/testutils/HoodieTestDataGenerator.java | 2 +-
3 files changed, 104 insertions(+), 1 deletion(-)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
index fb7acc90b6e1..fa177eca9952 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.InstantComparator;
import org.apache.hudi.common.util.NumericUtils;
@@ -38,8 +39,10 @@ import org.springframework.shell.standard.ShellMethod;
import org.springframework.shell.standard.ShellOption;
import java.io.IOException;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -360,6 +363,36 @@ public class CommitsCommand {
limit, headerOnly, rows, exportTableName);
}
+ @ShellMethod(key = "commits show_infights", value = "Show inflight instants
that are left longer than a certain duration")
+ public String showInflightCommits(
+ @ShellOption(value = {"--lookbackInMins"}, help = "Only show inflight
commits that started before the specified lookback duration (in minutes).",
defaultValue = "0") final Long durationInMins) {
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
+
+ // Fetch inflight commits.
+ long goBackMs = Duration.ofMinutes(durationInMins).getSeconds() * 1000;
+ String oldestAllowedTimestamp = HoodieInstantTimeGenerator
+ .formatDate(new Date(System.currentTimeMillis() - goBackMs));
+
+ List<HoodieInstant> inflightInstants = metaClient
+ .reloadActiveTimeline()
+ .getWriteTimeline()
+ .filterInflightsAndRequested()
+ .findInstantsBefore(oldestAllowedTimestamp)
+ .getInstants();
+
+ // Create a table out of inflight commits.
+ List<String[]> data = new ArrayList<>();
+ inflightInstants.forEach(instant ->
+ data.add(new String[]{instant.requestedTime(), instant.getAction(),
instant.getState().name()}));
+ String[] header = new String[]{HoodieTableHeaderFields.HEADER_COMMIT_TIME,
+ HoodieTableHeaderFields.HEADER_ACTION,
HoodieTableHeaderFields.HEADER_STATE};
+ if (data.isEmpty()) {
+ return "No inflight instants are found.";
+ } else {
+ return HoodiePrintHelper.print(header, data.toArray(new String[0][]));
+ }
+ }
+
@ShellMethod(key = "commits compare", value = "Compare commits with another
Hoodie table")
public String compareCommits(@ShellOption(value = {"--path"}, help = "Path
of the table to compare to") final String path) {
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
index fb4c7804f676..7a0fd534309b 100644
---
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
@@ -33,8 +33,11 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.InstantFileNameGenerator;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import
org.apache.hudi.common.table.timeline.versioning.v1.InstantFileNameGeneratorV1;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
@@ -49,6 +52,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.table.HoodieSparkTable;
+import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -62,6 +66,7 @@ import org.springframework.shell.Shell;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -72,6 +77,7 @@ import java.util.stream.Collectors;
import static
org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR;
import static
org.apache.hudi.common.testutils.HoodieTestUtils.createCompactionCommitInMetadataTable;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
@@ -569,4 +575,68 @@ public class TestCommitsCommand extends
CLIFunctionalTestHarness {
String expected = String.format("Load sync state between %s and %s",
tableName1, tableName2);
assertEquals(expected, result.toString());
}
+
+ @Test
+ public void testInflightCommand() throws Exception {
+ generateData();
+ // Generate instant times using HoodieInstantTimeGenerator
+ String oldInstantTime1 = HoodieInstantTimeGenerator.formatDate(
+ new Date(System.currentTimeMillis() - 1000 * 60 * 60)); // 60 mins ago
+ String oldInstantTime2 = HoodieInstantTimeGenerator.formatDate(
+ new Date(System.currentTimeMillis() - 1000 * 60 * 45)); // 45 mins ago
+ String oldInstantTime3 = HoodieInstantTimeGenerator.formatDate(
+ new Date(System.currentTimeMillis() - 1000 * 60 * 5)); // 5 mins ago
+
+ // Reload meta client to pick up new instants
+ metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
+
+ // Create inflight commits
+ InstantFileNameGenerator v1InstantNameGenerator = new
InstantFileNameGeneratorV1();
+ List<String> fileNames = Arrays.asList(
+ v1InstantNameGenerator.makeInflightCommitFileName(oldInstantTime1),
+ v1InstantNameGenerator.makeCommitFileName(
+ oldInstantTime2 + "_" +
InProcessTimeGenerator.createNewInstantTime()),
+ v1InstantNameGenerator.makeInflightCommitFileName(oldInstantTime3));
+ fileNames.forEach(name -> {
+ try {
+ Path filePath = new Path(basePath() + "/" +
HoodieTableMetaClient.METAFOLDER_NAME + "/" + name);
+ HoodieTestDataGenerator.createEmptyFile(basePath(), filePath,
storageConf());
+ } catch (IOException ignored) {
+ // Exception ignored.
+ }
+ });
+
+ // Reload meta client to pick up new instants
+ metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
+
+ Object lookupBackInZeroMinsResult = shell.evaluate(() -> "commits
show_infights --lookbackInMins 0");
+
assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackInZeroMinsResult));
+
+ // All three instants should be shown when duration is 0
+ String output = lookupBackInZeroMinsResult.toString();
+ assertTrue(output.contains(oldInstantTime1));
+ assertFalse(output.contains(oldInstantTime2));
+ assertTrue(output.contains(oldInstantTime3));
+
+ // Only one instants should be shown when duration is 15 since 2nd commit
is a completed commit.
+ Object lookupBackIn15MinsResult = shell.evaluate(() -> "commits
show_infights --lookbackInMins 15");
+ assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackIn15MinsResult));
+ output = lookupBackIn15MinsResult.toString();
+ assertTrue(output.contains(oldInstantTime1));
+ assertFalse(output.contains(oldInstantTime2));
+
+ // Only one instant should be shown when duration is 50
+ Object lookupBackIn50MinsResult = shell.evaluate(() -> "commits
show_infights --lookbackInMins 50");
+ assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackIn50MinsResult));
+ output = lookupBackIn50MinsResult.toString();
+ assertTrue(output.contains(oldInstantTime1));
+ assertFalse(output.contains(oldInstantTime2));
+
+ // No instants should be shown when duration is > 60
+ Object lookupBackIn70MinsResult = shell.evaluate(() -> "commits
show_infights --lookbackInMins 70");
+ assertTrue(ShellEvaluationResultUtil.isSuccess(lookupBackIn70MinsResult));
+ output = lookupBackIn70MinsResult.toString();
+ assertTrue(output.contains(oldInstantTime1));
+ assertFalse(output.contains(oldInstantTime2));
+ }
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
index acc602f08db0..c7162d54d337 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
@@ -940,7 +940,7 @@ Generate random record using TRIP_ENCODED_DECIMAL_SCHEMA
createEmptyFile(basePath, commitFile, configuration);
}
- private static void createEmptyFile(String basePath, Path filePath,
StorageConfiguration<?> configuration) throws IOException {
+ public static void createEmptyFile(String basePath, Path filePath,
StorageConfiguration<?> configuration) throws IOException {
HoodieStorage storage = HoodieStorageUtils.getStorage(basePath,
configuration);
OutputStream os = storage.create(new StoragePath(filePath.toUri()), true);
os.close();
