This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4f50fde3da3 [HUDI-8009] Optimize the code of
HoodieTable#getPendingCommitTimeline (#11665)
4f50fde3da3 is described below
commit 4f50fde3da345bea989b26ae1e7944d94e426dfd
Author: usberkeley <[email protected]>
AuthorDate: Wed Jul 24 08:55:56 2024 +0800
[HUDI-8009] Optimize the code of HoodieTable#getPendingCommitTimeline
(#11665)
---
.../hudi/index/bucket/ConsistentBucketIndexUtils.java | 2 +-
.../src/main/java/org/apache/hudi/table/HoodieTable.java | 4 ++--
.../action/rollback/BaseRollbackActionExecutor.java | 4 ++--
.../apache/hudi/table/upgrade/TestUpgradeDowngrade.java | 16 ++++++++--------
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 7473e0d6632..453f08a486a 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -270,7 +270,7 @@ public class ConsistentBucketIndexUtils {
private static boolean recommitMetadataFile(HoodieTable table, FileStatus
metaFile, String partition) {
Path partitionPath = new
Path(FSUtils.constructAbsolutePath(table.getMetaClient().getBasePath(),
partition).toUri());
String timestamp = getTimestampFromFile(metaFile.getPath().getName());
- if (table.getPendingCommitTimeline().containsInstant(timestamp)) {
+ if (table.getPendingCommitsTimeline().containsInstant(timestamp)) {
return false;
}
Option<HoodieConsistentHashingMetadata>
hoodieConsistentHashingMetadataOption = loadMetadataFromGivenFile(table,
metaFile);
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index ef14396bf9e..9e51ef99ae4 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -367,8 +367,8 @@ public abstract class HoodieTable<T, I, K, O> implements
Serializable {
/**
* Get only the inflights (no-completed) commit timeline.
*/
- public HoodieTimeline getPendingCommitTimeline() {
- return
metaClient.getCommitsTimeline().filterPendingExcludingMajorAndMinorCompaction();
+ public HoodieTimeline getPendingCommitsTimeline() {
+ return metaClient.getCommitsTimeline().filterInflightsAndRequested();
}
/**
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index 5e02f20c86a..e4094e86864 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -165,7 +165,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K,
O> extends BaseActionE
if (config.getFailedWritesCleanPolicy().isEager() &&
!HoodieTableMetadata.isMetadataTable(config.getBasePath())) {
final String instantTimeToRollback = instantToRollback.getTimestamp();
HoodieTimeline commitTimeline = table.getCompletedCommitsTimeline();
- HoodieTimeline inflightAndRequestedCommitTimeline =
table.getPendingCommitTimeline();
+ HoodieTimeline pendingCommitsTimeline =
table.getPendingCommitsTimeline();
// Check validity of completed commit timeline.
// Make sure only the last n commits are being rolled back
// If there is a commit in-between or after that is not rolled back,
then abort
@@ -184,7 +184,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K,
O> extends BaseActionE
}
}
- List<String> inflights =
inflightAndRequestedCommitTimeline.getInstantsAsStream()
+ List<String> inflights = pendingCommitsTimeline.getInstantsAsStream()
.filter(instant ->
!ClusteringUtils.isClusteringInstant(table.getActiveTimeline(), instant))
.map(HoodieInstant::getTimestamp)
.collect(Collectors.toList());
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index c993292a6d4..4e3d98f52bb 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -199,16 +199,16 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
HoodieTable table = this.getHoodieTable(metaClient, cfg);
prepForUpgradeFromZeroToOne(table);
- HoodieInstant commitInstant =
table.getPendingCommitTimeline().lastInstant().get();
+ HoodieInstant commitsInstant =
table.getPendingCommitsTimeline().lastInstant().get();
// delete one of the marker files in 2nd commit if need be.
WriteMarkers writeMarkers =
- WriteMarkersFactory.get(getConfig().getMarkersType(), table,
commitInstant.getTimestamp());
+ WriteMarkersFactory.get(getConfig().getMarkersType(), table,
commitsInstant.getTimestamp());
List<String> markerPaths = new
ArrayList<>(writeMarkers.allMarkerFilePaths());
if (deletePartialMarkerFiles) {
String toDeleteMarkerFile = markerPaths.get(0);
table.getStorage().deleteDirectory(new StoragePath(
- table.getMetaClient().getTempFolderPath() + "/" +
commitInstant.getTimestamp()
+ table.getMetaClient().getTempFolderPath() + "/" +
commitsInstant.getTimestamp()
+ "/" + toDeleteMarkerFile));
markerPaths.remove(toDeleteMarkerFile);
}
@@ -225,7 +225,7 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
.run(HoodieTableVersion.ONE, null);
// assert marker files
- assertMarkerFilesForUpgrade(table, commitInstant,
firstPartitionCommit2FileSlices, secondPartitionCommit2FileSlices);
+ assertMarkerFilesForUpgrade(table, commitsInstant,
firstPartitionCommit2FileSlices, secondPartitionCommit2FileSlices);
// verify hoodie.table.version got upgraded
metaClient = HoodieTableMetaClient.builder()
@@ -614,15 +614,15 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
twoUpsertCommitDataWithTwoPartitions(firstPartitionCommit2FileSlices,
secondPartitionCommit2FileSlices, cfg, client, false);
HoodieTable table = this.getHoodieTable(metaClient, cfg);
- HoodieInstant commitInstant =
table.getPendingCommitTimeline().lastInstant().get();
+ HoodieInstant commitsInstant =
table.getPendingCommitsTimeline().lastInstant().get();
// delete one of the marker files in 2nd commit if need be.
- WriteMarkers writeMarkers = WriteMarkersFactory.get(markerType, table,
commitInstant.getTimestamp());
+ WriteMarkers writeMarkers = WriteMarkersFactory.get(markerType, table,
commitsInstant.getTimestamp());
List<String> markerPaths = new
ArrayList<>(writeMarkers.allMarkerFilePaths());
if (deletePartialMarkerFiles) {
String toDeleteMarkerFile = markerPaths.get(0);
table.getStorage().deleteDirectory(new StoragePath(
- table.getMetaClient().getTempFolderPath() + "/" +
commitInstant.getTimestamp()
+ table.getMetaClient().getTempFolderPath() + "/" +
commitsInstant.getTimestamp()
+ "/" + toDeleteMarkerFile));
markerPaths.remove(toDeleteMarkerFile);
}
@@ -635,7 +635,7 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
if (fromVersion.versionCode() == HoodieTableVersion.TWO.versionCode()) {
// assert marker files
- assertMarkerFilesForDowngrade(table, commitInstant, toVersion ==
HoodieTableVersion.ONE);
+ assertMarkerFilesForDowngrade(table, commitsInstant, toVersion ==
HoodieTableVersion.ONE);
}
// verify hoodie.table.version got downgraded
