voonhous commented on code in PR #19877: URL: https://github.com/apache/hudi/pull/19877#discussion_r3976340713
########## hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestExportCommand.java: ########## @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.cli.commands; + +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.functional.CLIFunctionalTestHarness; +import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; +import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.exception.HoodieException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.shell.Shell; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test cases for {@link ExportCommand}. + */ +@Tag("functional") +@SpringBootTest(properties = {"spring.shell.interactive.enabled=false", "spring.shell.command.script.enabled=false"}) +public class TestExportCommand extends CLIFunctionalTestHarness { + + private static final String[] COMMIT_TIMES = new String[] {"101", "102", "103"}; + + @Autowired + private Shell shell; + + private String tablePath; + private Path exportFolder; + + @BeforeEach + public void init() throws Exception { + HoodieCLI.conf = storageConf(); + String tableName = tableName(); + tablePath = tablePath(tableName); + exportFolder = Files.createDirectories(Paths.get(basePath(), "exported-instants")); + + new TableCommand().createTable( + tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(), + "", HoodieTableVersion.current().versionCode(), "org.apache.hudi.common.model.HoodieAvroPayload"); + for (String commitTime : COMMIT_TIMES) { + HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, commitTime, storageConf()); + } + HoodieCLI.refreshTableMetadata(); + } + + /** + * Exports the whole timeline. The instant count is passed as the limit and the ordering is + * descending on purpose: that is the one shape in which the export does not walk the archived + * timeline, whose reader cannot open the LSM timeline history directory of a table of version + * eight or above. The limit is not honoured for active instants either. Both are tracked in + * https://github.com/apache/hudi/issues/19879; once fixed, this test can drop the workaround. + */ + @Test + public void testExportInstants() throws Exception { + Object result = shell.evaluate( + () -> "export instants --desc true --limit " + COMMIT_TIMES.length + " --localFolder " + exportFolder); + assertTrue(ShellEvaluationResultUtil.isSuccess(result), String.valueOf(result)); + assertEquals("Exported " + COMMIT_TIMES.length + " Instants to " + exportFolder, result.toString()); + + // one file per completed instant, named after the instant file it was read from + assertEquals(instantFileNames(COMMIT_TIMES), exportedFiles()); + for (String fileName : exportedFiles()) { + // commit metadata is already json on the timeline and is copied over as is + String content = new String(Files.readAllBytes(exportFolder.resolve(fileName))); + assertTrue(content.contains("partitionToWriteStats"), content); Review Comment: Right, the schema header made that unfalsifiable. The test now decodes the exported bytes through the table's `CommitMetadataSerDe` and asserts the partition paths and write-stat count the fixture wrote; comment corrected. Done in a3c0e32b9cba. ########## hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTimelineCommand.java: ########## @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.cli.commands; + +import org.apache.hudi.avro.model.HoodieInstantInfo; +import org.apache.hudi.avro.model.HoodieRollbackPlan; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.functional.CLIFunctionalTestHarness; +import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil; +import org.apache.hudi.client.SparkRDDWriteClient; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieMetadataTestTable; +import org.apache.hudi.common.testutils.HoodieTestTable; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataWriter; +import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.shell.Shell; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test cases for {@link TimelineCommand}. + */ +@Tag("functional") +@SpringBootTest(properties = {"spring.shell.interactive.enabled=false", "spring.shell.command.script.enabled=false"}) +public class TestTimelineCommand extends CLIFunctionalTestHarness { + + // Column offsets of the data table part of the rendered timeline, row number included. + private static final int COL_INSTANT = 1; + private static final int COL_ACTION = 2; + private static final int COL_STATE = 3; + private static final int COL_REQUESTED_TIME = 4; + private static final int COL_INFLIGHT_TIME = 5; + private static final int COL_COMPLETED_TIME = 6; + // Column offsets of the metadata table part, only rendered with --with-metadata-table. + private static final int COL_MT_ACTION = 7; + private static final int COL_MT_STATE = 8; + + // The commit left in the requested state, and the rollback scheduled against it. + private static final String REQUESTED_COMMIT = "103"; + private static final String PENDING_ROLLBACK_INSTANT = "104"; + private static final String ROLLED_BACK_COMMIT = "102"; + + private static final String DATE_NO_SECONDS = "\\d{2}-\\d{2} \\d{2}:\\d{2}"; + private static final String DATE_WITH_SECONDS = "\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}"; + + @Autowired + private Shell shell; + + private String tablePath; + private HoodieTableMetaClient metaClient; + private String rollbackInstantTime; + + /** + * Builds a table whose active timeline holds two completed commits, a completed rollback of a + * third commit, one commit left in the requested state and a rollback scheduled against that + * commit, with the metadata table enabled so that the metadata table timeline is populated too. + */ + @BeforeEach + public void init() throws Exception { + HoodieCLI.conf = storageConf(); + String tableName = tableName(); + tablePath = tablePath(tableName); + + new TableCommand().createTable( Review Comment: Confirmed and taken now rather than as a follow-up: the new fixtures use `newTableBuilder().initTable(...)` plus `connect(...)`. The four new fixtures went from 155 s to 54 s in one run (Compaction 85 to 42, Timeline 48 to 7, Clustering 12 to 4, Export 10 to 0.2), about 5 s per fresh table as you estimated; the body now says about 50 s added. Done in a3c0e32b9cba. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
