This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4efc241d7a8 [MINOR] Avoid wrong way to get the latest completed
instant (#12590)
4efc241d7a8 is described below
commit 4efc241d7a89f7736b79ef0d16deab796e9e05f5
Author: TheR1sing3un <[email protected]>
AuthorDate: Wed Jan 8 15:41:34 2025 +0800
[MINOR] Avoid wrong way to get the latest completed instant (#12590)
1. avoid wrong way to get the latest completed instant
Signed-off-by: TheR1sing3un <[email protected]>
---
.../java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java | 6 +-----
.../org/apache/hudi/common/table/timeline/HoodieTimeline.java | 2 +-
.../apache/spark/sql/hudi/streaming/HoodieStreamSourceV1.scala | 6 +-----
.../main/java/org/apache/hudi/sync/datahub/DataHubSyncClient.java | 8 +-------
.../src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java | 7 +------
.../src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java | 4 +---
6 files changed, 6 insertions(+), 27 deletions(-)
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index 506023b22a7..829242c72db 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -896,11 +896,7 @@ public class AWSGlueCatalogSyncClient extends
HoodieSyncClient {
HoodieTimeline activeTimeline = getActiveTimeline();
Option<String> lastCommitSynced =
activeTimeline.lastInstant().map(HoodieInstant::requestedTime);
Option<String> lastCommitCompletionSynced = activeTimeline
- .getInstantsOrderedByCompletionTime()
- .skip(activeTimeline.countInstants() - 1)
- .findFirst()
- .map(i -> Option.of(i.getCompletionTime()))
- .orElse(Option.empty());
+ .getLatestCompletionTime();
if (lastCommitSynced.isPresent()) {
try {
HashMap<String, String> propertyMap = new HashMap<>();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
index 33ea149e148..cadf45574d4 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieTimeline.java
@@ -405,7 +405,7 @@ public interface HoodieTimeline extends Serializable {
Option<String> getLatestCompletionTime();
/**
- * Get the stream of instants in order by state transition timestamp of
actions.
+ * Get the stream of instants in order by completion timestamp of actions.
*/
Stream<HoodieInstant> getInstantsOrderedByCompletionTime();
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV1.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV1.scala
index a43b77551a8..82a7e804656 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV1.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV1.scala
@@ -107,11 +107,7 @@ class HoodieStreamSourceV1(sqlContext: SQLContext,
filteredTimeline match {
case activeInstants if !activeInstants.empty() =>
val timestamp = if (hollowCommitHandling == USE_TRANSITION_TIME) {
- activeInstants.getInstantsOrderedByCompletionTime
- .skip(activeInstants.countInstants() - 1)
- .findFirst()
- .get()
- .getCompletionTime
+ activeInstants.getLatestCompletionTime.get()
} else {
activeInstants.lastInstant().get().requestedTime()
}
diff --git
a/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/DataHubSyncClient.java
b/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/DataHubSyncClient.java
index 44ce04f59e6..f9bb2ea2710 100644
---
a/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/DataHubSyncClient.java
+++
b/hudi-sync/hudi-datahub-sync/src/main/java/org/apache/hudi/sync/datahub/DataHubSyncClient.java
@@ -99,13 +99,7 @@ public class DataHubSyncClient extends HoodieSyncClient {
}
protected Option<String> getLastCommitCompletionTime() {
- int countInstants = getActiveTimeline().countInstants();
- return getActiveTimeline()
- .getInstantsOrderedByCompletionTime()
- .skip(countInstants - 1)
- .findFirst()
- .map(HoodieInstant::getCompletionTime)
- .map(Option::of).orElseGet(Option::empty);
+ return getActiveTimeline().getLatestCompletionTime();
}
@Override
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
index bd00773e913..797b20aea69 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
@@ -404,12 +404,7 @@ public class HoodieHiveSyncClient extends HoodieSyncClient
{
// Set the last commit time and commit completion from the TBLproperties
HoodieTimeline activeTimeline = getActiveTimeline();
Option<String> lastCommitSynced =
activeTimeline.lastInstant().map(HoodieInstant::requestedTime);
- Option<String> lastCommitCompletionSynced = activeTimeline
- .getInstantsOrderedByCompletionTime()
- .skip(activeTimeline.countInstants() - 1)
- .findFirst()
- .map(i -> Option.of(i.getCompletionTime()))
- .orElse(Option.empty());
+ Option<String> lastCommitCompletionSynced =
activeTimeline.getLatestCompletionTime();
if (lastCommitSynced.isPresent()) {
try {
Table table = client.getTable(databaseName, tableName);
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index 9e420683e0c..11671b1e02b 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -1979,9 +1979,7 @@ public class TestHiveSyncTool {
}
private String getLastCommitCompletionTimeSynced() {
- return hiveClient.getActiveTimeline()
- .getInstantsOrderedByCompletionTime()
- .skip(hiveClient.getActiveTimeline().countInstants() -
1).findFirst().get().getCompletionTime();
+ return hiveClient.getActiveTimeline().getLatestCompletionTime().get();
}
private void reInitHiveSyncClient() {
