This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 45017036ecf [HUDI-1593] Add support for "show restores" and "show
restore" commands in hudi-cli (#7868)
45017036ecf is described below
commit 45017036ecfa7ca70b7b9e5cc2435abb87f79e3e
Author: Pramod Biligiri <[email protected]>
AuthorDate: Fri Feb 17 07:56:22 2023 +0530
[HUDI-1593] Add support for "show restores" and "show restore" commands in
hudi-cli (#7868)
This adds two commands to the hudi-cli: "show restores" and "show restore
--instant INSTANT_VALUE".
---
.../apache/hudi/cli/HoodieTableHeaderFields.java | 6 +
.../apache/hudi/cli/commands/RestoresCommand.java | 172 +++++++++++++++++
.../hudi/cli/commands/TestRestoresCommand.java | 207 +++++++++++++++++++++
3 files changed, 385 insertions(+)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
index e6016e4cc1c..20829251ee2 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
@@ -101,6 +101,12 @@ public class HoodieTableHeaderFields {
public static final String HEADER_DELETED_FILE = "Deleted File";
public static final String HEADER_SUCCEEDED = "Succeeded";
+ /**
+ * Fields of Restore.
+ */
+ public static final String HEADER_RESTORE_INSTANT = "Restored " +
HEADER_INSTANT;
+ public static final String HEADER_RESTORE_STATE = "Restore State";
+
/**
* Fields of Stats.
*/
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
new file mode 100644
index 00000000000..fb6c4b7a66c
--- /dev/null
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.avro.model.HoodieInstantInfo;
+import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieRestorePlan;
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.HoodiePrintHelper;
+import org.apache.hudi.cli.HoodieTableHeaderFields;
+import org.apache.hudi.cli.TableHeader;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.util.Option;
+import org.springframework.shell.standard.ShellComponent;
+import org.springframework.shell.standard.ShellMethod;
+import org.springframework.shell.standard.ShellOption;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static
org.apache.hudi.common.table.timeline.HoodieTimeline.RESTORE_ACTION;
+
+/**
+ * CLI command to display info about restore actions.
+ */
+@ShellComponent
+public class RestoresCommand {
+
+ @ShellMethod(key = "show restores", value = "List all restore instants")
+ public String showRestores(
+ @ShellOption(value = {"--limit"}, help = "Limit #rows to be
displayed", defaultValue = "10") Integer limit,
+ @ShellOption(value = {"--sortBy"}, help = "Sorting Field",
defaultValue = "") final String sortByField,
+ @ShellOption(value = {"--desc"}, help = "Ordering", defaultValue =
"false") final boolean descending,
+ @ShellOption(value = {"--headeronly"}, help = "Print Header Only",
+ defaultValue = "false") final boolean headerOnly,
+ @ShellOption(value = {"--includeInflights"}, help = "Also list
restores that are in flight",
+ defaultValue = "false") final boolean includeInflights) {
+
+ HoodieActiveTimeline activeTimeline =
HoodieCLI.getTableMetaClient().getActiveTimeline();
+ List<HoodieInstant> restoreInstants = getRestoreInstants(activeTimeline,
includeInflights);
+
+ final List<Comparable[]> outputRows = new ArrayList<>();
+ for (HoodieInstant restoreInstant : restoreInstants) {
+ populateOutputFromRestoreInstant(restoreInstant, outputRows,
activeTimeline);
+ }
+
+ TableHeader header = createResultHeader();
+ return HoodiePrintHelper.print(header, new HashMap<>(), sortByField,
descending, limit, headerOnly, outputRows);
+ }
+
+ @ShellMethod(key = "show restore", value = "Show details of a restore
instant")
+ public String showRestore(
+ @ShellOption(value = {"--instant"}, help = "Restore instant") String
restoreInstant,
+ @ShellOption(value = {"--limit"}, help = "Limit #rows to be
displayed", defaultValue = "10") Integer limit,
+ @ShellOption(value = {"--sortBy"}, help = "Sorting Field",
defaultValue = "") final String sortByField,
+ @ShellOption(value = {"--desc"}, help = "Ordering", defaultValue =
"false") final boolean descending,
+ @ShellOption(value = {"--headeronly"}, help = "Print Header Only",
+ defaultValue = "false") final boolean headerOnly) {
+
+ HoodieActiveTimeline activeTimeline =
HoodieCLI.getTableMetaClient().getActiveTimeline();
+ List<HoodieInstant> matchingInstants =
activeTimeline.filterCompletedInstants().filter(completed ->
+ completed.getTimestamp().equals(restoreInstant)).getInstants();
+ if (matchingInstants.isEmpty()) {
+ matchingInstants = activeTimeline.filterInflights().filter(inflight ->
+ inflight.getTimestamp().equals(restoreInstant)).getInstants();
+ }
+
+ // Assuming a single exact match is found in either completed or inflight
instants
+ HoodieInstant instant = matchingInstants.get(0);
+ List<Comparable[]> outputRows = new ArrayList<>();
+ populateOutputFromRestoreInstant(instant, outputRows, activeTimeline);
+
+ TableHeader header = createResultHeader();
+ return HoodiePrintHelper.print(header, new HashMap<>(), sortByField,
descending, limit, headerOnly, outputRows);
+ }
+
+ private void addDetailsOfCompletedRestore(HoodieActiveTimeline
activeTimeline, List<Comparable[]> rows,
+ HoodieInstant restoreInstant)
throws IOException {
+ HoodieRestoreMetadata instantMetadata;
+ Option<byte[]> instantDetails =
activeTimeline.getInstantDetails(restoreInstant);
+ instantMetadata = TimelineMetadataUtils
+ .deserializeAvroMetadata(instantDetails.get(),
HoodieRestoreMetadata.class);
+
+ for (String rolledbackInstant : instantMetadata.getInstantsToRollback()) {
+ Comparable[] row = createDataRow(instantMetadata.getStartRestoreTime(),
rolledbackInstant,
+ instantMetadata.getTimeTakenInMillis(),
restoreInstant.getState());
+ rows.add(row);
+ }
+ }
+
+ private void addDetailsOfInflightRestore(HoodieActiveTimeline
activeTimeline, List<Comparable[]> rows,
+ HoodieInstant restoreInstant)
throws IOException {
+ HoodieRestorePlan restorePlan = getRestorePlan(activeTimeline,
restoreInstant);
+ for (HoodieInstantInfo instantToRollback :
restorePlan.getInstantsToRollback()) {
+ Comparable[] dataRow = createDataRow(restoreInstant.getTimestamp(),
instantToRollback.getCommitTime(), "",
+ restoreInstant.getState());
+ rows.add(dataRow);
+ }
+ }
+
+ private HoodieRestorePlan getRestorePlan(HoodieActiveTimeline
activeTimeline, HoodieInstant restoreInstant) throws IOException {
+ HoodieInstant instantKey = new
HoodieInstant(HoodieInstant.State.REQUESTED, RESTORE_ACTION,
+ restoreInstant.getTimestamp());
+ Option<byte[]> instantDetails =
activeTimeline.getInstantDetails(instantKey);
+ HoodieRestorePlan restorePlan = TimelineMetadataUtils
+ .deserializeAvroMetadata(instantDetails.get(),
HoodieRestorePlan.class);
+ return restorePlan;
+ }
+
+ private List<HoodieInstant> getRestoreInstants(HoodieActiveTimeline
activeTimeline, boolean includeInFlight) {
+ List<HoodieInstant> restores = new ArrayList<>();
+
restores.addAll(activeTimeline.getRestoreTimeline().filterCompletedInstants().getInstants());
+
+ if (includeInFlight) {
+
restores.addAll(activeTimeline.getRestoreTimeline().filterInflights().getInstants());
+ }
+
+ return restores;
+ }
+
+ private TableHeader createResultHeader() {
+ return new TableHeader()
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT)
+
.addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_INSTANT)
+
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_STATE);
+ }
+
+ private void populateOutputFromRestoreInstant(HoodieInstant restoreInstant,
List<Comparable[]> outputRows,
+ HoodieActiveTimeline
activeTimeline) {
+ try {
+ if (restoreInstant.isInflight() || restoreInstant.isRequested()) {
+ addDetailsOfInflightRestore(activeTimeline, outputRows,
restoreInstant);
+ } else if (restoreInstant.isCompleted()) {
+ addDetailsOfCompletedRestore(activeTimeline, outputRows,
restoreInstant);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private Comparable[] createDataRow(Comparable restoreInstantTimestamp,
Comparable rolledbackInstantTimestamp,
+ Comparable timeInterval, Comparable
state) {
+ Comparable[] row = new Comparable[4];
+ row[0] = restoreInstantTimestamp;
+ row[1] = rolledbackInstantTimestamp;
+ row[2] = timeInterval;
+ row[3] = state;
+ return row;
+ }
+
+}
diff --git
a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
new file mode 100644
index 00000000000..aa75ff29b8b
--- /dev/null
+++
b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieSavepointMetadata;
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.HoodiePrintHelper;
+import org.apache.hudi.cli.HoodieTableHeaderFields;
+import org.apache.hudi.cli.TableHeader;
+import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
+import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil;
+import org.apache.hudi.client.BaseHoodieWriteClient;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.shell.Shell;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@Tag("functional")
+@SpringBootTest(properties = {"spring.shell.interactive.enabled=false",
"spring.shell.command.script.enabled=false"})
+public class TestRestoresCommand extends CLIFunctionalTestHarness {
+
+ @Autowired
+ private Shell shell;
+
+ @BeforeEach
+ public void init() throws Exception {
+ String tableName = tableName();
+ String tablePath = tablePath(tableName);
+ new TableCommand().createTable(
+ tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(),
+ "", TimelineLayoutVersion.VERSION_1,
"org.apache.hudi.common.model.HoodieAvroPayload");
+ HoodieTableMetaClient metaClient =
HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
+ //Create some commits files and base files
+ Map<String, String> partitionAndFileId = new HashMap<String, String>() {
+ {
+ put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+ put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+ put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
+ }
+ };
+
+ HoodieWriteConfig config =
HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withMetadataConfig(
+ // Column Stats Index is disabled, since these tests
construct tables which are
+ // not valid (empty commit metadata, etc)
+ HoodieMetadataConfig.newBuilder()
+ .withMetadataIndexColumnStats(false)
+ .build()
+ )
+ .withRollbackUsingMarkers(false)
+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .build();
+
+ HoodieTestTable hoodieTestTable = HoodieMetadataTestTable.of(metaClient,
SparkHoodieBackedTableMetadataWriter.create(
+ metaClient.getHadoopConf(), config, context))
+ .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+ .addCommit("100")
+ .withBaseFilesInPartitions(partitionAndFileId)
+ .addCommit("101");
+
+
hoodieTestTable.addCommit("102").withBaseFilesInPartitions(partitionAndFileId);
+ HoodieSavepointMetadata savepointMetadata2 =
hoodieTestTable.doSavepoint("102");
+ hoodieTestTable.addSavepoint("102", savepointMetadata2);
+
+
hoodieTestTable.addCommit("103").withBaseFilesInPartitions(partitionAndFileId);
+
+ BaseHoodieWriteClient client = new SparkRDDWriteClient(context(), config);
+ client.rollback("103");
+ client.restoreToSavepoint("102");
+
+
hoodieTestTable.addCommit("105").withBaseFilesInPartitions(partitionAndFileId);
+ HoodieSavepointMetadata savepointMetadata =
hoodieTestTable.doSavepoint("105");
+ hoodieTestTable.addSavepoint("105", savepointMetadata);
+
+
hoodieTestTable.addCommit("106").withBaseFilesInPartitions(partitionAndFileId);
+ client.rollback("106");
+ client.restoreToSavepoint("105");
+ client.close();
+ }
+
+ @Test
+ public void testShowRestores() {
+ Object result = shell.evaluate(() -> "show restores");
+ assertTrue(ShellEvaluationResultUtil.isSuccess(result));
+
+ // get restored instants
+ HoodieActiveTimeline activeTimeline =
HoodieCLI.getTableMetaClient().getActiveTimeline();
+ Stream<HoodieInstant> restores =
activeTimeline.getRestoreTimeline().filterCompletedInstants().getInstantsAsStream();
+
+ List<Comparable[]> rows = new ArrayList<>();
+ restores.sorted().forEach(instant -> {
+ try {
+ HoodieRestoreMetadata metadata = TimelineMetadataUtils
+
.deserializeAvroMetadata(activeTimeline.getInstantDetails(instant).get(),
HoodieRestoreMetadata.class);
+ metadata.getInstantsToRollback().forEach(c -> {
+ Comparable[] row = new Comparable[4];
+ row[0] = metadata.getStartRestoreTime();
+ row[1] = c;
+ row[2] = metadata.getTimeTakenInMillis();
+ row[3] = HoodieInstant.State.COMPLETED.toString();
+ rows.add(row);
+ });
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ });
+
+ TableHeader header = new TableHeader()
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT)
+
.addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_INSTANT)
+
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_STATE);
+ String expected = HoodiePrintHelper.print(header, new HashMap<>(), "",
false,
+ -1, false, rows);
+ expected = removeNonWordAndStripSpace(expected);
+ String got = removeNonWordAndStripSpace(result.toString());
+ assertEquals(expected, got);
+ }
+
+ @Test
+ public void testShowRestore() throws IOException {
+ // get instant
+ HoodieActiveTimeline activeTimeline =
HoodieCLI.getTableMetaClient().getActiveTimeline();
+ Stream<HoodieInstant> restores =
activeTimeline.getRestoreTimeline().filterCompletedInstants().getInstantsAsStream();
+ HoodieInstant instant = restores.findFirst().orElse(null);
+ assertNotNull(instant, "The instant can not be null.");
+
+ Object result = shell.evaluate(() -> "show restore --instant " +
instant.getTimestamp());
+ assertTrue(ShellEvaluationResultUtil.isSuccess(result));
+
+ // get metadata of instant
+ HoodieRestoreMetadata instantMetadata =
TimelineMetadataUtils.deserializeAvroMetadata(
+ activeTimeline.getInstantDetails(instant).get(),
HoodieRestoreMetadata.class);
+
+ // generate expected result
+ TableHeader header = new TableHeader()
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT)
+
.addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_INSTANT)
+
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
+ .addTableHeaderField(HoodieTableHeaderFields.HEADER_RESTORE_STATE);
+
+ List<Comparable[]> rows = new ArrayList<>();
+ instantMetadata.getInstantsToRollback().forEach((String rolledbackInstant)
-> {
+ Comparable[] row = new Comparable[4];
+ row[0] = instantMetadata.getStartRestoreTime();
+ row[1] = rolledbackInstant;
+ row[2] = instantMetadata.getTimeTakenInMillis();
+ row[3] = HoodieInstant.State.COMPLETED.toString();
+ rows.add(row);
+ });
+ String expected = HoodiePrintHelper.print(header, new HashMap<>(), "",
false, -1,
+ false, rows);
+ expected = removeNonWordAndStripSpace(expected);
+ String got = removeNonWordAndStripSpace(result.toString());
+ assertEquals(expected, got);
+ }
+
+}
