This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 1fa3f2ed6a0 [MINOR] Fix flakiness in
TestHoodieFileSystemViews.testFileSystemViewConsistency (#13375)
1fa3f2ed6a0 is described below
commit 1fa3f2ed6a0fcc687abf747a14483e4981dfb859
Author: Lokesh Jain <[email protected]>
AuthorDate: Fri Jun 6 21:47:48 2025 +0530
[MINOR] Fix flakiness in
TestHoodieFileSystemViews.testFileSystemViewConsistency (#13375)
---
.../functional/TestHoodieFileSystemViews.java | 53 ++++++++++++++++------
1 file changed, 39 insertions(+), 14 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
index 6c104ad110f..fab5e7c1588 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
+++
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
@@ -39,7 +39,6 @@ import
org.apache.hudi.common.table.view.FileSystemViewManager;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieArchivalConfig;
@@ -69,6 +68,7 @@ import java.util.stream.Collectors;
import static
org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
import static
org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION;
+import static
org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN;
import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
@@ -145,7 +145,7 @@ public class TestHoodieFileSystemViews extends
HoodieClientTestBase {
HoodieTableMetadata.create(context, metaClient.getStorage(),
config.getMetadataConfig(), config.getBasePath()))
.getFileSystemView(basePath);
- assertFileSystemViews(config, enableMdt, storageType, Option.empty());
+ assertFileSystemViews(config, enableMdt, storageType);
for (int i = 3; i < 10; i++) {
String commitTime = client.createNewInstantTime();
upsertRecords(client, commitTime, 50);
@@ -157,15 +157,29 @@ public class TestHoodieFileSystemViews extends
HoodieClientTestBase {
String commitTime = client.createNewInstantTime();
upsertRecords(client, commitTime, 50);
}
-
- // mimic failed write for last completed operation and retry few more
operations.
- HoodieInstant lastInstant =
metaClient.reloadActiveTimeline().getWriteTimeline().lastInstant().get();
- HoodieCommitMetadata commitMetadata =
metaClient.getActiveTimeline().readCommitMetadata(lastInstant);
- StoragePath instantPath = HoodieTestUtils
- .getCompleteInstantPath(metaClient.getStorage(),
- metaClient.getTimelinePath(),
- lastInstant.requestedTime(), lastInstant.getAction(),
HoodieTableVersion.fromVersionCode(writeVersion));
- metaClient.getStorage().deleteFile(instantPath);
+
+ HoodieCommitMetadata commitMetadata = null;
+ boolean rollbackExecuted = false;
+ for (int i = 23; i < 26; i++) {
+ // We want to rollback deltacommit and commit instant here. With table
version 6 and 8,
+ // the last instant can be compaction in MDT table at different
instants due to change in how the
+ // MDT tables are initialised. Therefore we need to take a range of
commits and see at what instant rollback
+ // is feasible
+ String commitTime = client.createNewInstantTime();
+ upsertRecords(client, commitTime, 50);
+ HoodieInstant lastInstant =
metaClient.reloadActiveTimeline().getWriteTimeline().lastInstant().get();
+ if (!rollbackExecuted) {
+ // rollback needs to be performed for delta commit or commit instant
+ if (isRollbackPossible(enableMdt, lastInstant)) {
+ // rollback last completed deltacommit or commit operation and
retry few more operations.
+ commitMetadata =
metaClient.getActiveTimeline().readCommitMetadata(lastInstant);
+ client.rollback(lastInstant.requestedTime());
+ rollbackExecuted = true;
+ }
+ }
+ }
+ // validate rollback was executed
+ assertTrue(rollbackExecuted);
expectedFileSystemView.sync();
actualFileSystemView.sync();
@@ -174,17 +188,28 @@ public class TestHoodieFileSystemViews extends
HoodieClientTestBase {
metaClient.getTableConfig().getTableVersion().greaterThanOrEquals(HoodieTableVersion.EIGHT)
? Option.empty() : Option.of(commitMetadata));
// add few more updates
- for (int i = 23; i < 28; i++) {
+ for (int i = 26; i < 28; i++) {
String commitTime = client.createNewInstantTime();
upsertRecords(client, commitTime, 50);
}
actualFileSystemView.close();
expectedFileSystemView.close();
}
- assertFileSystemViews(config, enableMdt, storageType, Option.empty());
+ assertFileSystemViews(config, enableMdt, storageType);
+ }
+
+ private boolean isRollbackPossible(boolean enableMdt, HoodieInstant
lastInstant) {
+ boolean shouldRollback =
lastInstant.getAction().equals(DELTA_COMMIT_ACTION) ||
lastInstant.getAction().equals(COMMIT_ACTION);
+ if (shouldRollback && enableMdt) {
+ // commit lesser than the last compaction instant in MDT can not be
rolled back
+ HoodieTableMetaClient mdtMetaClient =
HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() +
"/metadata").setConf(metaClient.getStorageConf()).build();
+ String mdtLastCompactionTime =
mdtMetaClient.reloadActiveTimeline().filter(instant ->
instant.getAction().equals(COMMIT_ACTION)).lastInstant().get().requestedTime();
+ shouldRollback = LESSER_THAN.test(mdtLastCompactionTime,
lastInstant.requestedTime());
+ }
+ return shouldRollback;
}
- private void assertFileSystemViews(HoodieWriteConfig writeConfig, boolean
enableMdt, FileSystemViewStorageType baseStorageType,
Option<HoodieCommitMetadata> commitMetadataOpt) {
+ private void assertFileSystemViews(HoodieWriteConfig writeConfig, boolean
enableMdt, FileSystemViewStorageType baseStorageType) {
metaClient = HoodieTableMetaClient.reload(metaClient);
// base line file system view is in-memory for any combination.
HoodieTableFileSystemView expectedFileSystemView =
FileSystemViewManager.createInMemoryFileSystemView(context, metaClient,
