(hudi) branch master updated: [HUDI-7030] Update containsInstant without containsOrBeforeTimelineStarts to fix data lost (#9982)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new e731755b990 [HUDI-7030] Update containsInstant without 
containsOrBeforeTimelineStarts to fix data lost (#9982)
e731755b990 is described below

commit e731755b99057dc916378f1f7e95c73642ff96e8
Author: xoln ann <xoln_...@outlook.com>
AuthorDate: Wed Nov 8 14:39:32 2023 +0800

    [HUDI-7030] Update containsInstant without containsOrBeforeTimelineStarts 
to fix data lost (#9982)
---
 .../hudi/client/functional/TestHoodieIndex.java     | 21 +++++++++++++++++++++
 .../table/timeline/HoodieDefaultTimeline.java       |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git 
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieIndex.java
 
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieIndex.java
index 4518b909813..37199c783bb 100644
--- 
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieIndex.java
+++ 
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieIndex.java
@@ -553,6 +553,27 @@ public class TestHoodieIndex extends 
TestHoodieMetadataBase {
     assertFalse(timeline.empty());
     assertFalse(HoodieIndexUtils.checkIfValidCommit(timeline, 
instantTimestamp));
     assertFalse(HoodieIndexUtils.checkIfValidCommit(timeline, 
instantTimestampSec));
+
+    // Check the completed delta commit instant which is end with 
DEFAULT_MILLIS_EXT timestamp
+    // Timestamp not contain in inflight timeline, checkContainsInstant() 
should return false
+    // Timestamp contain in inflight timeline, checkContainsInstant() should 
return true
+    String checkInstantTimestampSec = instantTimestamp.substring(0, 
instantTimestamp.length() - 
HoodieInstantTimeGenerator.DEFAULT_MILLIS_EXT.length());
+    String checkInstantTimestamp = checkInstantTimestampSec + 
HoodieInstantTimeGenerator.DEFAULT_MILLIS_EXT;
+    Thread.sleep(2000); // sleep required so that new timestamp differs in the 
seconds rather than msec
+    String newTimestamp = writeClient.createNewInstantTime();
+    String newTimestampSec = newTimestamp.substring(0, newTimestamp.length() - 
HoodieInstantTimeGenerator.DEFAULT_MILLIS_EXT.length());
+    final HoodieInstant instant5 = new HoodieInstant(true, 
HoodieTimeline.DELTA_COMMIT_ACTION, newTimestamp);
+    timeline = new HoodieDefaultTimeline(Stream.of(instant5), 
metaClient.getActiveTimeline()::getInstantDetails);
+    assertFalse(timeline.empty());
+    assertFalse(timeline.containsInstant(checkInstantTimestamp));
+    assertFalse(timeline.containsInstant(checkInstantTimestampSec));
+
+    final HoodieInstant instant6 = new HoodieInstant(true, 
HoodieTimeline.DELTA_COMMIT_ACTION, newTimestampSec + 
HoodieInstantTimeGenerator.DEFAULT_MILLIS_EXT);
+    timeline = new HoodieDefaultTimeline(Stream.of(instant6), 
metaClient.getActiveTimeline()::getInstantDetails);
+    assertFalse(timeline.empty());
+    assertFalse(timeline.containsInstant(newTimestamp));
+    assertFalse(timeline.containsInstant(checkInstantTimestamp));
+    assertTrue(timeline.containsInstant(instant6.getTimestamp()));
   }
 
   @Test
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
index ec7c9633576..ecf7c938b01 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java
@@ -440,7 +440,7 @@ public class HoodieDefaultTimeline implements 
HoodieTimeline {
     // Check for older timestamp which have sec granularity and an extension 
of DEFAULT_MILLIS_EXT may have been added via Timeline operations
     if (ts.length() == 
HoodieInstantTimeGenerator.MILLIS_INSTANT_TIMESTAMP_FORMAT_LENGTH && 
ts.endsWith(HoodieInstantTimeGenerator.DEFAULT_MILLIS_EXT)) {
       final String actualOlderFormatTs = ts.substring(0, ts.length() - 
HoodieInstantTimeGenerator.DEFAULT_MILLIS_EXT.length());
-      return containsOrBeforeTimelineStarts(actualOlderFormatTs);
+      return containsInstant(actualOlderFormatTs);
     }
 
     return false;