This is an automated email from the ASF dual-hosted git repository. codope pushed a commit to branch release-0.12.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9f284a0125e8aedc68b19c07ce57789db33ff584 Author: Sivabalan Narayanan <[email protected]> AuthorDate: Tue Aug 2 18:20:38 2022 -0400 [HUDI-4501] Throwing exception when restore is attempted with hoodie.arhive.beyond.savepoint is enabled (#6239) --- .../apache/hudi/client/BaseHoodieWriteClient.java | 3 ++ .../TestHoodieClientOnCopyOnWriteStorage.java | 57 ++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index ab14f2216a..4cf58f1c30 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -61,6 +61,7 @@ import org.apache.hudi.common.util.CommitUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieClusteringConfig; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; @@ -721,6 +722,8 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K, HoodieTable<T, I, K, O> table = initTable(WriteOperationType.UNKNOWN, Option.empty(), initialMetadataTableIfNecessary); SavepointHelpers.validateSavepointPresence(table, savepointTime); + ValidationUtils.checkArgument(!config.shouldArchiveBeyondSavepoint(), "Restore is not supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + + " is enabled"); restoreToInstant(savepointTime, initialMetadataTableIfNecessary); SavepointHelpers.validateSavepointRestore(table, savepointTime); } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 8ba459b772..28108b793a 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -71,6 +71,7 @@ import org.apache.hudi.common.util.MarkerUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieClusteringConfig; @@ -675,6 +676,62 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { }).collect(); } + @Test + public void testRestoreWithSavepointBeyondArchival() throws Exception { + HoodieWriteConfig config = getConfigBuilder().withRollbackUsingMarkers(true).build(); + HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(EAGER) + .withRollbackUsingMarkers(true) + .withArchivalConfig(HoodieArchivalConfig.newBuilder().withArchiveBeyondSavepoint(true).build()) + .withProps(config.getProps()).withTimelineLayoutVersion( + VERSION_0).build(); + + HoodieTableMetaClient.withPropertyBuilder() + .fromMetaClient(metaClient) + .setTimelineLayoutVersion(VERSION_0) + .setPopulateMetaFields(config.populateMetaFields()) + .initTable(metaClient.getHadoopConf(), metaClient.getBasePath()); + + SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig); + + // Write 1 (only inserts) + String newCommitTime = "001"; + String initCommitTime = "000"; + int numRecords = 200; + insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, SparkRDDWriteClient::insert, + false, true, numRecords, config.populateMetaFields()); + + // Write 2 (updates) + String prevCommitTime = newCommitTime; + newCommitTime = "004"; + numRecords = 100; + String commitTimeBetweenPrevAndNew = "002"; + updateBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, + Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, SparkRDDWriteClient::upsert, false, true, + numRecords, 200, 2, config.populateMetaFields()); + + // Delete 1 + prevCommitTime = newCommitTime; + newCommitTime = "005"; + numRecords = 50; + + deleteBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, + initCommitTime, numRecords, SparkRDDWriteClient::delete, false, true, + 0, 150, config.populateMetaFields()); + + HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion( + TimelineLayoutVersion.CURR_VERSION) + .withArchivalConfig(HoodieArchivalConfig.newBuilder().withArchiveBeyondSavepoint(true).build()).build(); + client = getHoodieWriteClient(newConfig); + + client.savepoint("004", "user1", "comment1"); + + // verify that restore fails when "hoodie.archive.beyond.savepoint" is enabled. + SparkRDDWriteClient finalClient = client; + assertThrows(IllegalArgumentException.class, () -> { + finalClient.restoreToSavepoint("004"); + }, "Restore should not be supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled"); + } + /** * Test Insert API for HoodieConcatHandle. */
