voonhous commented on code in PR #19877: URL: https://github.com/apache/hudi/pull/19877#discussion_r3976342331
########## hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTimelineCommand.java: ########## @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.cli.commands; + +import org.apache.hudi.avro.model.HoodieInstantInfo; +import org.apache.hudi.avro.model.HoodieRollbackPlan; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.functional.CLIFunctionalTestHarness; +import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil; +import org.apache.hudi.client.SparkRDDWriteClient; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieMetadataTestTable; +import org.apache.hudi.common.testutils.HoodieTestTable; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataWriter; +import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.shell.Shell; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test cases for {@link TimelineCommand}. + */ +@Tag("functional") +@SpringBootTest(properties = {"spring.shell.interactive.enabled=false", "spring.shell.command.script.enabled=false"}) +public class TestTimelineCommand extends CLIFunctionalTestHarness { + + // Column offsets of the data table part of the rendered timeline, row number included. + private static final int COL_INSTANT = 1; + private static final int COL_ACTION = 2; + private static final int COL_STATE = 3; + private static final int COL_REQUESTED_TIME = 4; + private static final int COL_INFLIGHT_TIME = 5; + private static final int COL_COMPLETED_TIME = 6; + // Column offsets of the metadata table part, only rendered with --with-metadata-table. + private static final int COL_MT_ACTION = 7; + private static final int COL_MT_STATE = 8; + + // The commit left in the requested state, and the rollback scheduled against it. + private static final String REQUESTED_COMMIT = "103"; + private static final String PENDING_ROLLBACK_INSTANT = "104"; + private static final String ROLLED_BACK_COMMIT = "102"; + + private static final String DATE_NO_SECONDS = "\\d{2}-\\d{2} \\d{2}:\\d{2}"; + private static final String DATE_WITH_SECONDS = "\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}"; + + @Autowired + private Shell shell; + + private String tablePath; + private HoodieTableMetaClient metaClient; + private String rollbackInstantTime; + + /** + * Builds a table whose active timeline holds two completed commits, a completed rollback of a + * third commit, one commit left in the requested state and a rollback scheduled against that + * commit, with the metadata table enabled so that the metadata table timeline is populated too. + */ + @BeforeEach + public void init() throws Exception { + HoodieCLI.conf = storageConf(); + String tableName = tableName(); + tablePath = tablePath(tableName); + + new TableCommand().createTable( + tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(), + "", HoodieTableVersion.current().versionCode(), "org.apache.hudi.common.model.HoodieAvroPayload"); + metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); + + Map<String, String> partitionAndFileId = new HashMap<>(); + partitionAndFileId.put(DEFAULT_FIRST_PARTITION_PATH, "file-1"); + partitionAndFileId.put(DEFAULT_SECOND_PARTITION_PATH, "file-2"); + + HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath) + .withMetadataConfig( + // Column Stats Index is disabled, since this table is built with empty commit metadata + HoodieMetadataConfig.newBuilder().withMetadataIndexColumnStats(false).build()) + .withRollbackUsingMarkers(false) + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()) + .build(); + + try (HoodieTableMetadataWriter metadataWriter = SparkHoodieBackedTableMetadataWriter.create( + metaClient.getStorageConf(), config, context)) { + HoodieTestTable testTable = HoodieMetadataTestTable.of(metaClient, metadataWriter, Option.of(context)) + .withPartitionMetaFiles(DEFAULT_FIRST_PARTITION_PATH, DEFAULT_SECOND_PARTITION_PATH) + .addCommit("100").withBaseFilesInPartitions(partitionAndFileId).getLeft() + .addCommit("101").withBaseFilesInPartitions(partitionAndFileId).getLeft() + .addInflightCommit(ROLLED_BACK_COMMIT); + testTable.withBaseFilesInPartitions(partitionAndFileId); + + try (SparkRDDWriteClient client = new SparkRDDWriteClient(context(), config)) { + client.rollback(ROLLED_BACK_COMMIT); + } + // left behind on the timeline so that the incomplete timeline is not empty + testTable.addRequestedCommit(REQUESTED_COMMIT); + + // A rollback that is scheduled but has not run yet. Unlike the completed one above it leaves + // the commit it targets on the timeline, which is the only way an instant is rendered as Review Comment: Reworded: the pending rollback is what makes the annotation reachable on this COW table, where the completed rollback deleted its target. Done in a3c0e32b9cba. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
