This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch release-1.1.2 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 38087e2aeebc2ccca3ae4caf2f2919cb93728f62 Author: danny0405 <[email protected]> AuthorDate: Sat Oct 10 16:04:08 2026 +0800 test(metadata): backport clustering retry test fixes from master Backport the Java-client test changes from #19120 and #19434: use monotonic instant times and simulate the failed clustering commit before any reader can cache its completed state. --- .../hudi/client/TestJavaHoodieBackedMetadata.java | 35 +++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index d439189a0a63..9df7d4b8ae32 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -1818,7 +1818,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { HoodieJavaWriteClient client = getHoodieWriteClient(config); // Write 1 (Bulk insert) - String newCommitTime = "0000001"; + String newCommitTime = WriteClientTestUtils.createNewInstantTime(); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 20); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); List<WriteStatus> writeStatuses = client.insert(records, newCommitTime); @@ -1827,7 +1827,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { validateMetadata(client); // Write 2 (inserts) - newCommitTime = "0000002"; + newCommitTime = WriteClientTestUtils.createNewInstantTime(); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); records = dataGen.generateInserts(newCommitTime, 20); writeStatuses = client.insert(records, newCommitTime); @@ -1843,7 +1843,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName()) .build(); - HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER) + HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.LAZY) .withClusteringConfig(clusteringConfig) .withRollbackUsingMarkers(false) .build(); @@ -1859,17 +1859,38 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { partitionFiles.getValue().stream().forEach(file -> replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file)))); + // manually remove clustering completed instant from .hoodie folder to mimic clustering + // succeeded in metadata table, but failed before committing to data table. This must happen + // before any subsequent metadata-table read so that no reader ever sees the first attempt's + // instant as committed, matching the crash scenario this test simulates. + FileCreateUtilsLegacy.deleteReplaceCommit(basePath, clusteringCommitTime); + + // preconditions: clustering is pending on the data table while its deltacommit is completed + // in the metadata table. + metaClient.reloadActiveTimeline(); + assertFalse(metaClient.getActiveTimeline().filterCompletedInstants().containsInstant(clusteringCommitTime)); + assertTrue(metaClient.getActiveTimeline().filterPendingClusteringTimeline().containsInstant(clusteringCommitTime)); + HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient( + metaClient.getStorageConf(), getMetadataTableBasePath(basePath)); + assertTrue(metadataMetaClient.getActiveTimeline().getDeltaCommitTimeline() + .filterCompletedInstants().containsInstant(clusteringCommitTime)); + // trigger new write to mimic other writes succeeding before re-attempt. - newCommitTime = "0000003"; + newCommitTime = WriteClientTestUtils.createNewInstantTime(); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); records = dataGen.generateInserts(newCommitTime, 20); writeStatuses = client.insert(records, newCommitTime); client.commit(newCommitTime, writeStatuses); assertNoWriteErrors(writeStatuses); - validateMetadata(client); + // ignore the first clustering attempt's uncommitted files when comparing listings. + validateMetadata(client, Option.of(clusteringCommitTime)); + + // the new write must leave the pending clustering and its metadata-table deltacommit intact. + metaClient.reloadActiveTimeline(); + assertTrue(metaClient.getActiveTimeline().filterPendingClusteringTimeline().containsInstant(clusteringCommitTime)); + assertTrue(metadataMetaClient.reloadActiveTimeline().getDeltaCommitTimeline() + .filterCompletedInstants().containsInstant(clusteringCommitTime)); - // manually remove clustering completed instant from .hoodie folder and to mimic succeeded clustering in metadata table, but failed in data table. - FileCreateUtilsLegacy.deleteReplaceCommit(basePath, clusteringCommitTime); HoodieWriteMetadata<List<WriteStatus>> updatedClusterMetadata = newClient.cluster(clusteringCommitTime, true); metaClient.reloadActiveTimeline();
