wombatu-kun commented on code in PR #19877:
URL: https://github.com/apache/hudi/pull/19877#discussion_r3976026423
##########
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java:
##########
@@ -146,6 +172,209 @@ public void testCompactionShow() throws IOException {
assertNotNull(result);
}
+ /**
+ * Test case of the compaction validation entry point of {@link SparkMain},
which the
+ * 'compaction validate' command reaches through a spark-submit of its own.
+ */
+ @Test
+ public void testSparkMainCompactValidate() throws Exception {
+ createPendingCompactions();
+ String outputPath = outputPath("validate");
+
+ SparkMain.doCompactValidate(jsc(), tablePath, PENDING_COMPACTION_INSTANT,
outputPath, 2);
+
+ List<ValidationOpResult> results = readOperationResults(outputPath);
+ assertEquals(operationsOf(PENDING_COMPACTION_INSTANT).size(),
results.size());
+ assertTrue(results.stream().allMatch(ValidationOpResult::isSuccess),
results.toString());
+ assertEquals(fileIdsOf(PENDING_COMPACTION_INSTANT),
+ results.stream().map(result ->
result.getOperation().getFileId()).collect(Collectors.toSet()));
+ }
+
+ @Test
+ public void testSparkMainCompactValidateReportsMissingLogFile() throws
Exception {
+ createPendingCompactions();
+ HoodieCompactionOperation broken =
operationsOf(PENDING_COMPACTION_INSTANT).get(0);
+ // a log file the plan reads is gone, so that operation can no longer be
compacted
+ Files.delete(Paths.get(tablePath, broken.getPartitionPath(),
broken.getDeltaFilePaths().get(0)));
+ String outputPath = outputPath("validate-broken");
+
+ SparkMain.doCompactValidate(jsc(), tablePath, PENDING_COMPACTION_INSTANT,
outputPath, 2);
+
+ List<ValidationOpResult> results = readOperationResults(outputPath);
+ assertEquals(operationsOf(PENDING_COMPACTION_INSTANT).size(),
results.size());
+ List<ValidationOpResult> failed = results.stream().filter(result ->
!result.isSuccess()).collect(Collectors.toList());
+ assertEquals(1, failed.size(), results.toString());
+ assertEquals(broken.getFileId(), failed.get(0).getOperation().getFileId());
+ assertTrue(failed.get(0).getException().isPresent());
+ }
+
+ /**
+ * Repair runs the plan validation and returns an empty result: the log file
renaming it was
+ * written for is gone from the admin client, which leaves the plan
untouched and never reads
+ * the dry run flag, so there is only one arm to exercise. See
+ * https://github.com/apache/hudi/issues/19881.
+ */
+ @Test
+ public void testSparkMainCompactRepair() throws Exception {
+ createPendingCompactions();
+ Set<String> fileIdsBefore = fileIdsOf(PENDING_COMPACTION_INSTANT);
+ String outputPath = outputPath("repair");
+
+ SparkMain.doCompactRepair(jsc(), tablePath, PENDING_COMPACTION_INSTANT,
outputPath, 2, false);
+
+ assertTrue(readOperationResults(outputPath).isEmpty());
+
assertTrue(pendingCompactionInstants().contains(PENDING_COMPACTION_INSTANT));
+ assertEquals(fileIdsBefore, fileIdsOf(PENDING_COMPACTION_INSTANT));
+ }
+
+ /**
+ * Unscheduling a plan takes the requested compaction instant off the
timeline, unless this is a
+ * dry run. The other pending plans are left alone either way. Skip
validation is held at false:
+ * the admin client takes the flag but never reads it, so toggling it
repeats the same run.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testSparkMainCompactUnschedulePlan(boolean dryRun) throws
Exception {
+ createPendingCompactions();
+ Set<String> pendingBefore = pendingCompactionInstants();
+ String outputPath = outputPath("unschedule-" + dryRun);
+
+ SparkMain.doCompactUnschedule(jsc(), tablePath,
PENDING_COMPACTION_INSTANT, outputPath, 2, false, dryRun);
+
+ assertTrue(readOperationResults(outputPath).isEmpty());
+ Set<String> pendingAfter = pendingCompactionInstants();
+ if (dryRun) {
+ assertEquals(pendingBefore, pendingAfter);
+ } else {
+ assertFalse(pendingAfter.contains(PENDING_COMPACTION_INSTANT),
pendingAfter.toString());
+ pendingBefore.remove(PENDING_COMPACTION_INSTANT);
+ assertEquals(pendingBefore, pendingAfter);
+ }
+ }
+
+ /**
+ * Unscheduling a single file group rewrites the plan without it, unless
this is a dry run. Skip
+ * validation is held at false: the admin client takes the flag but never
reads it, so toggling
+ * it repeats the same run.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testSparkMainCompactUnscheduleFile(boolean dryRun) throws
Exception {
+ Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>>
pendingOperations = createPendingCompactions();
+ HoodieFileGroupId unscheduled = pendingOperations.entrySet().stream()
+ .filter(entry ->
entry.getValue().getKey().equals(PENDING_COMPACTION_INSTANT))
+ .map(Map.Entry::getKey).findFirst().get();
+ String outputPath = outputPath("unschedule-file-" + dryRun);
+
+ SparkMain.doCompactUnscheduleFile(jsc(), tablePath,
unscheduled.getFileId(), unscheduled.getPartitionPath(),
+ outputPath, 2, false, dryRun);
+
+ assertTrue(readOperationResults(outputPath).isEmpty());
+ // the plan itself stays pending either way, only its operations change.
Only the target file
+ // group is asserted: the admin client currently drops the other
operations of the same
+ // partition as well (https://github.com/apache/hudi/issues/19881); assert
they survive once fixed.
+
assertTrue(pendingCompactionInstants().contains(PENDING_COMPACTION_INSTANT));
+ if (dryRun) {
+
assertTrue(fileIdsOf(PENDING_COMPACTION_INSTANT).contains(unscheduled.getFileId()));
+ } else {
+
assertFalse(fileIdsOf(PENDING_COMPACTION_INSTANT).contains(unscheduled.getFileId()));
Review Comment:
Both operations of this plan share a partition, so the admin client drops
both and this passes on an empty set - and it will still pass once #19881 is
fixed, so nothing prompts the follow-up the comment above asks for. Could it
assert the whole surviving set instead, empty today, so the test fails the day
that lands?
##########
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java:
##########
@@ -146,6 +172,209 @@ public void testCompactionShow() throws IOException {
assertNotNull(result);
}
+ /**
+ * Test case of the compaction validation entry point of {@link SparkMain},
which the
+ * 'compaction validate' command reaches through a spark-submit of its own.
+ */
+ @Test
+ public void testSparkMainCompactValidate() throws Exception {
+ createPendingCompactions();
+ String outputPath = outputPath("validate");
+
+ SparkMain.doCompactValidate(jsc(), tablePath, PENDING_COMPACTION_INSTANT,
outputPath, 2);
+
+ List<ValidationOpResult> results = readOperationResults(outputPath);
+ assertEquals(operationsOf(PENDING_COMPACTION_INSTANT).size(),
results.size());
+ assertTrue(results.stream().allMatch(ValidationOpResult::isSuccess),
results.toString());
+ assertEquals(fileIdsOf(PENDING_COMPACTION_INSTANT),
+ results.stream().map(result ->
result.getOperation().getFileId()).collect(Collectors.toSet()));
+ }
+
+ @Test
+ public void testSparkMainCompactValidateReportsMissingLogFile() throws
Exception {
+ createPendingCompactions();
+ HoodieCompactionOperation broken =
operationsOf(PENDING_COMPACTION_INSTANT).get(0);
+ // a log file the plan reads is gone, so that operation can no longer be
compacted
+ Files.delete(Paths.get(tablePath, broken.getPartitionPath(),
broken.getDeltaFilePaths().get(0)));
+ String outputPath = outputPath("validate-broken");
+
+ SparkMain.doCompactValidate(jsc(), tablePath, PENDING_COMPACTION_INSTANT,
outputPath, 2);
+
+ List<ValidationOpResult> results = readOperationResults(outputPath);
+ assertEquals(operationsOf(PENDING_COMPACTION_INSTANT).size(),
results.size());
+ List<ValidationOpResult> failed = results.stream().filter(result ->
!result.isSuccess()).collect(Collectors.toList());
+ assertEquals(1, failed.size(), results.toString());
+ assertEquals(broken.getFileId(), failed.get(0).getOperation().getFileId());
+ assertTrue(failed.get(0).getException().isPresent());
+ }
+
+ /**
+ * Repair runs the plan validation and returns an empty result: the log file
renaming it was
+ * written for is gone from the admin client, which leaves the plan
untouched and never reads
+ * the dry run flag, so there is only one arm to exercise. See
+ * https://github.com/apache/hudi/issues/19881.
+ */
+ @Test
+ public void testSparkMainCompactRepair() throws Exception {
+ createPendingCompactions();
+ Set<String> fileIdsBefore = fileIdsOf(PENDING_COMPACTION_INSTANT);
+ String outputPath = outputPath("repair");
+
+ SparkMain.doCompactRepair(jsc(), tablePath, PENDING_COMPACTION_INSTANT,
outputPath, 2, false);
+
+ assertTrue(readOperationResults(outputPath).isEmpty());
+
assertTrue(pendingCompactionInstants().contains(PENDING_COMPACTION_INSTANT));
+ assertEquals(fileIdsBefore, fileIdsOf(PENDING_COMPACTION_INSTANT));
+ }
+
+ /**
+ * Unscheduling a plan takes the requested compaction instant off the
timeline, unless this is a
+ * dry run. The other pending plans are left alone either way. Skip
validation is held at false:
+ * the admin client takes the flag but never reads it, so toggling it
repeats the same run.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testSparkMainCompactUnschedulePlan(boolean dryRun) throws
Exception {
+ createPendingCompactions();
+ Set<String> pendingBefore = pendingCompactionInstants();
+ String outputPath = outputPath("unschedule-" + dryRun);
+
+ SparkMain.doCompactUnschedule(jsc(), tablePath,
PENDING_COMPACTION_INSTANT, outputPath, 2, false, dryRun);
Review Comment:
The summary still advertises unschedule coverage across the `skipValidation`
and `dryRun` combinations, which this round narrowed to `dryRun` alone. Could
the body be refreshed along with its counts, which now come to 26 executions
across seven tagged classes rather than 30 across eight?
##########
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTimelineCommand.java:
##########
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.avro.model.HoodieInstantInfo;
+import org.apache.hudi.avro.model.HoodieRollbackPlan;
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
+import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataWriter;
+import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.shell.Shell;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test cases for {@link TimelineCommand}.
+ */
+@Tag("functional")
+@SpringBootTest(properties = {"spring.shell.interactive.enabled=false",
"spring.shell.command.script.enabled=false"})
+public class TestTimelineCommand extends CLIFunctionalTestHarness {
+
+ // Column offsets of the data table part of the rendered timeline, row
number included.
+ private static final int COL_INSTANT = 1;
+ private static final int COL_ACTION = 2;
+ private static final int COL_STATE = 3;
+ private static final int COL_REQUESTED_TIME = 4;
+ private static final int COL_INFLIGHT_TIME = 5;
+ private static final int COL_COMPLETED_TIME = 6;
+ // Column offsets of the metadata table part, only rendered with
--with-metadata-table.
+ private static final int COL_MT_ACTION = 7;
+ private static final int COL_MT_STATE = 8;
+
+ // The commit left in the requested state, and the rollback scheduled
against it.
+ private static final String REQUESTED_COMMIT = "103";
+ private static final String PENDING_ROLLBACK_INSTANT = "104";
+ private static final String ROLLED_BACK_COMMIT = "102";
+
+ private static final String DATE_NO_SECONDS = "\\d{2}-\\d{2} \\d{2}:\\d{2}";
+ private static final String DATE_WITH_SECONDS = "\\d{2}-\\d{2}
\\d{2}:\\d{2}:\\d{2}";
+
+ @Autowired
+ private Shell shell;
+
+ private String tablePath;
+ private HoodieTableMetaClient metaClient;
+ private String rollbackInstantTime;
+
+ /**
+ * Builds a table whose active timeline holds two completed commits, a
completed rollback of a
+ * third commit, one commit left in the requested state and a rollback
scheduled against that
+ * commit, with the metadata table enabled so that the metadata table
timeline is populated too.
+ */
+ @BeforeEach
+ public void init() throws Exception {
+ HoodieCLI.conf = storageConf();
+ String tableName = tableName();
+ tablePath = tablePath(tableName);
+
+ new TableCommand().createTable(
+ tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
+ "", HoodieTableVersion.current().versionCode(),
"org.apache.hudi.common.model.HoodieAvroPayload");
+ metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
+
+ Map<String, String> partitionAndFileId = new HashMap<>();
+ partitionAndFileId.put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+ partitionAndFileId.put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+
+ HoodieWriteConfig config =
HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withMetadataConfig(
+ // Column Stats Index is disabled, since this table is built with
empty commit metadata
+
HoodieMetadataConfig.newBuilder().withMetadataIndexColumnStats(false).build())
+ .withRollbackUsingMarkers(false)
+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .build();
+
+ try (HoodieTableMetadataWriter metadataWriter =
SparkHoodieBackedTableMetadataWriter.create(
+ metaClient.getStorageConf(), config, context)) {
+ HoodieTestTable testTable = HoodieMetadataTestTable.of(metaClient,
metadataWriter, Option.of(context))
+ .withPartitionMetaFiles(DEFAULT_FIRST_PARTITION_PATH,
DEFAULT_SECOND_PARTITION_PATH)
+
.addCommit("100").withBaseFilesInPartitions(partitionAndFileId).getLeft()
+
.addCommit("101").withBaseFilesInPartitions(partitionAndFileId).getLeft()
+ .addInflightCommit(ROLLED_BACK_COMMIT);
+ testTable.withBaseFilesInPartitions(partitionAndFileId);
+
+ try (SparkRDDWriteClient client = new SparkRDDWriteClient(context(),
config)) {
+ client.rollback(ROLLED_BACK_COMMIT);
+ }
+ // left behind on the timeline so that the incomplete timeline is not
empty
+ testTable.addRequestedCommit(REQUESTED_COMMIT);
+
+ // A rollback that is scheduled but has not run yet. Unlike the
completed one above it leaves
+ // the commit it targets on the timeline, which is the only way an
instant is rendered as
Review Comment:
`getRolledBackInstantInfo` fills the map from its completed arm too, so a
finished rollback annotates its target whenever that target survives, which is
exactly what the MOR fixture in `TestRollbacksCommand` leaves behind. Could
this say the pending rollback is what makes the annotation reachable on a COW
table, rather than the only way?
##########
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTimelineCommand.java:
##########
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.avro.model.HoodieInstantInfo;
+import org.apache.hudi.avro.model.HoodieRollbackPlan;
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
+import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
+import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataWriter;
+import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.shell.Shell;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test cases for {@link TimelineCommand}.
+ */
+@Tag("functional")
+@SpringBootTest(properties = {"spring.shell.interactive.enabled=false",
"spring.shell.command.script.enabled=false"})
+public class TestTimelineCommand extends CLIFunctionalTestHarness {
+
+ // Column offsets of the data table part of the rendered timeline, row
number included.
+ private static final int COL_INSTANT = 1;
+ private static final int COL_ACTION = 2;
+ private static final int COL_STATE = 3;
+ private static final int COL_REQUESTED_TIME = 4;
+ private static final int COL_INFLIGHT_TIME = 5;
+ private static final int COL_COMPLETED_TIME = 6;
+ // Column offsets of the metadata table part, only rendered with
--with-metadata-table.
+ private static final int COL_MT_ACTION = 7;
+ private static final int COL_MT_STATE = 8;
+
+ // The commit left in the requested state, and the rollback scheduled
against it.
+ private static final String REQUESTED_COMMIT = "103";
+ private static final String PENDING_ROLLBACK_INSTANT = "104";
+ private static final String ROLLED_BACK_COMMIT = "102";
+
+ private static final String DATE_NO_SECONDS = "\\d{2}-\\d{2} \\d{2}:\\d{2}";
+ private static final String DATE_WITH_SECONDS = "\\d{2}-\\d{2}
\\d{2}:\\d{2}:\\d{2}";
+
+ @Autowired
+ private Shell shell;
+
+ private String tablePath;
+ private HoodieTableMetaClient metaClient;
+ private String rollbackInstantTime;
+
+ /**
+ * Builds a table whose active timeline holds two completed commits, a
completed rollback of a
+ * third commit, one commit left in the requested state and a rollback
scheduled against that
+ * commit, with the metadata table enabled so that the metadata table
timeline is populated too.
+ */
+ @BeforeEach
+ public void init() throws Exception {
+ HoodieCLI.conf = storageConf();
+ String tableName = tableName();
+ tablePath = tablePath(tableName);
+
+ new TableCommand().createTable(
Review Comment:
`createTable` probes for an existing table first, and on a fresh path that
probe spends a flat five seconds in the retry loop of
`ConfigUtils.fetchConfigs` before it throws, which is roughly 110 seconds of
the 150 this PR adds across the new fixtures. `TestMetadataCommand` here
already sidesteps it with `newTableBuilder().initTable(...)` plus
`connect(...)`; worth the same in the new fixtures, follow-up rather than a
blocker.
##########
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestExportCommand.java:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
+import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator;
+import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.exception.HoodieException;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.shell.Shell;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test cases for {@link ExportCommand}.
+ */
+@Tag("functional")
+@SpringBootTest(properties = {"spring.shell.interactive.enabled=false",
"spring.shell.command.script.enabled=false"})
+public class TestExportCommand extends CLIFunctionalTestHarness {
+
+ private static final String[] COMMIT_TIMES = new String[] {"101", "102",
"103"};
+
+ @Autowired
+ private Shell shell;
+
+ private String tablePath;
+ private Path exportFolder;
+
+ @BeforeEach
+ public void init() throws Exception {
+ HoodieCLI.conf = storageConf();
+ String tableName = tableName();
+ tablePath = tablePath(tableName);
+ exportFolder = Files.createDirectories(Paths.get(basePath(),
"exported-instants"));
+
+ new TableCommand().createTable(
+ tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
+ "", HoodieTableVersion.current().versionCode(),
"org.apache.hudi.common.model.HoodieAvroPayload");
+ for (String commitTime : COMMIT_TIMES) {
+
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath,
commitTime, storageConf());
+ }
+ HoodieCLI.refreshTableMetadata();
+ }
+
+ /**
+ * Exports the whole timeline. The instant count is passed as the limit and
the ordering is
+ * descending on purpose: that is the one shape in which the export does not
walk the archived
+ * timeline, whose reader cannot open the LSM timeline history directory of
a table of version
+ * eight or above. The limit is not honoured for active instants either.
Both are tracked in
+ * https://github.com/apache/hudi/issues/19879; once fixed, this test can
drop the workaround.
+ */
+ @Test
+ public void testExportInstants() throws Exception {
+ Object result = shell.evaluate(
+ () -> "export instants --desc true --limit " + COMMIT_TIMES.length + "
--localFolder " + exportFolder);
+ assertTrue(ShellEvaluationResultUtil.isSuccess(result),
String.valueOf(result));
+ assertEquals("Exported " + COMMIT_TIMES.length + " Instants to " +
exportFolder, result.toString());
+
+ // one file per completed instant, named after the instant file it was
read from
+ assertEquals(instantFileNames(COMMIT_TIMES), exportedFiles());
+ for (String fileName : exportedFiles()) {
+ // commit metadata is already json on the timeline and is copied over as
is
+ String content = new
String(Files.readAllBytes(exportFolder.resolve(fileName)));
+ assertTrue(content.contains("partitionToWriteStats"), content);
Review Comment:
On a table of the current version the instant file is an Avro object
container, so `partitionToWriteStats` is present in its embedded schema header
whatever the commit holds and this assertion cannot fail. Could it decode the
bytes through the table's `CommitMetadataSerDe` and assert on the partition
path the fixture actually wrote, with the comment corrected to match?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
