This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e1f70fda589 [HUDI-8112] Fix
TestHoodieActiveTimeline#testTimelineGetOperations test logic error (#11814)
e1f70fda589 is described below
commit e1f70fda5890cff44b3e44eb362c5a411f4174f3
Author: usberkeley <[email protected]>
AuthorDate: Fri Aug 23 08:49:37 2024 +0800
[HUDI-8112] Fix TestHoodieActiveTimeline#testTimelineGetOperations test
logic error (#11814)
---
.../hudi/common/table/timeline/HoodieTimeline.java | 2 +-
.../table/timeline/TestHoodieActiveTimeline.java | 34 ++++++++++++----------
2 files changed, 20 insertions(+), 16 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
index a0ccdd43467..2ca793c1926 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
@@ -65,7 +65,7 @@ public interface HoodieTimeline extends Serializable {
String SCHEMA_COMMIT_ACTION = "schemacommit";
String[] VALID_ACTIONS_IN_TIMELINE = {COMMIT_ACTION, DELTA_COMMIT_ACTION,
CLEAN_ACTION, SAVEPOINT_ACTION, RESTORE_ACTION, ROLLBACK_ACTION,
- COMPACTION_ACTION, REPLACE_COMMIT_ACTION, CLUSTERING_ACTION,
INDEXING_ACTION};
+ COMPACTION_ACTION, LOG_COMPACTION_ACTION, REPLACE_COMMIT_ACTION,
CLUSTERING_ACTION, INDEXING_ACTION};
String COMMIT_EXTENSION = "." + COMMIT_ACTION;
String DELTA_COMMIT_EXTENSION = "." + DELTA_COMMIT_ACTION;
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
index 3f7f1c05b84..1e68f399cc4 100755
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
@@ -336,7 +336,7 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
@Test
public void testTimelineGetOperations() {
List<HoodieInstant> allInstants = getAllInstants();
- Supplier<Stream<HoodieInstant>> sup = allInstants::stream;
+ Supplier<Stream<HoodieInstant>> allInstantsSup = allInstants::stream;
timeline = new HoodieActiveTimeline(metaClient, true);
timeline.setInstants(allInstants);
@@ -346,8 +346,14 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
* @param actions The actions that should be present in the timeline being
checked
*/
BiConsumer<HoodieTimeline, Set<String>> checkTimeline = (HoodieTimeline
timeline, Set<String> actions) -> {
- sup.get().filter(i -> actions.contains(i.getAction())).forEach(i ->
assertTrue(timeline.containsInstant(i)));
- sup.get().filter(i -> !actions.contains(i.getAction())).forEach(i ->
assertFalse(timeline.containsInstant(i)));
+ List<HoodieInstant> expectedInstants = allInstantsSup.get().filter(i ->
actions.contains(i.getAction())).collect(Collectors.toList());
+ List<HoodieInstant> unexpectedInstants = allInstantsSup.get().filter(i
-> !actions.contains(i.getAction())).collect(Collectors.toList());
+
+ // The test set is the instant of all actions and states, so the
instants returned by the timeline get operation cannot be empty.
+ // At the same time, it helps to detect errors such as incomplete test
sets
+ assertFalse(expectedInstants.isEmpty());
+ expectedInstants.forEach(i -> assertTrue(timeline.containsInstant(i)));
+ unexpectedInstants.forEach(i ->
assertFalse(timeline.containsInstant(i)));
};
// Test that various types of getXXX operations from HoodieActiveTimeline
@@ -355,7 +361,8 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
checkTimeline.accept(timeline.getCommitsTimeline(),
CollectionUtils.createSet(
HoodieTimeline.COMMIT_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION,
HoodieTimeline.REPLACE_COMMIT_ACTION, HoodieTimeline.CLUSTERING_ACTION));
checkTimeline.accept(timeline.getWriteTimeline(),
CollectionUtils.createSet(
- HoodieTimeline.COMMIT_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION,
HoodieTimeline.COMPACTION_ACTION, HoodieTimeline.REPLACE_COMMIT_ACTION,
HoodieTimeline.CLUSTERING_ACTION));
+ HoodieTimeline.COMMIT_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION,
HoodieTimeline.COMPACTION_ACTION, HoodieTimeline.LOG_COMPACTION_ACTION,
+ HoodieTimeline.REPLACE_COMMIT_ACTION,
HoodieTimeline.CLUSTERING_ACTION));
checkTimeline.accept(timeline.getCommitAndReplaceTimeline(),
CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.REPLACE_COMMIT_ACTION, HoodieTimeline.CLUSTERING_ACTION));
checkTimeline.accept(timeline.getDeltaCommitTimeline(),
Collections.singleton(HoodieTimeline.DELTA_COMMIT_ACTION));
checkTimeline.accept(timeline.getCleanerTimeline(),
Collections.singleton(HoodieTimeline.CLEAN_ACTION));
@@ -363,14 +370,14 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
checkTimeline.accept(timeline.getRestoreTimeline(),
Collections.singleton(HoodieTimeline.RESTORE_ACTION));
checkTimeline.accept(timeline.getSavePointTimeline(),
Collections.singleton(HoodieTimeline.SAVEPOINT_ACTION));
checkTimeline.accept(timeline.getAllCommitsTimeline(),
CollectionUtils.createSet(
- HoodieTimeline.COMMIT_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION,
HoodieTimeline.CLEAN_ACTION, HoodieTimeline.COMPACTION_ACTION,
+ HoodieTimeline.COMMIT_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION,
HoodieTimeline.CLEAN_ACTION, HoodieTimeline.COMPACTION_ACTION,
HoodieTimeline.LOG_COMPACTION_ACTION,
HoodieTimeline.REPLACE_COMMIT_ACTION,
HoodieTimeline.CLUSTERING_ACTION, HoodieTimeline.SAVEPOINT_ACTION,
HoodieTimeline.ROLLBACK_ACTION, HoodieTimeline.INDEXING_ACTION));
// Get some random Instants
Random rand = new Random();
- Set<String> randomInstants = sup.get().filter(i -> rand.nextBoolean())
+ Set<String> randomActions = allInstantsSup.get().filter(i ->
rand.nextBoolean())
.map(HoodieInstant::getAction).collect(Collectors.toSet());
- checkTimeline.accept(timeline.getTimelineOfActions(randomInstants),
randomInstants);
+ checkTimeline.accept(timeline.getTimelineOfActions(randomActions),
randomActions);
}
@Test
@@ -738,21 +745,18 @@ public class TestHoodieActiveTimeline extends
HoodieCommonTestHarness {
// Following are not valid combinations of actions and state so we
should
// not be generating them.
if (state == State.REQUESTED) {
- if (action.equals(HoodieTimeline.SAVEPOINT_ACTION) ||
action.equals(HoodieTimeline.RESTORE_ACTION)
- || action.equals(HoodieTimeline.ROLLBACK_ACTION)) {
+ if (action.equals(HoodieTimeline.SAVEPOINT_ACTION) ||
action.equals(HoodieTimeline.RESTORE_ACTION)) {
continue;
}
}
- if (state == State.INFLIGHT &&
action.equals(HoodieTimeline.ROLLBACK_ACTION)) {
- continue;
- }
- if (state == State.COMPLETED &&
action.equals(HoodieTimeline.ROLLBACK_ACTION)) {
- continue;
- }
// Compaction complete is called commit complete
if (state == State.COMPLETED &&
action.equals(HoodieTimeline.COMPACTION_ACTION)) {
action = HoodieTimeline.COMMIT_ACTION;
}
+ // LogCompaction complete is called deltacommit complete
+ if (state == State.COMPLETED &&
action.equals(HoodieTimeline.LOG_COMPACTION_ACTION)) {
+ action = HoodieTimeline.DELTA_COMMIT_ACTION;
+ }
// Cluster complete is called replacecommit complete
if (state == State.COMPLETED &&
action.equals(HoodieTimeline.CLUSTERING_ACTION)) {
action = HoodieTimeline.REPLACE_COMMIT_ACTION;
