(paimon) branch master updated: [core] Fix that latestSnapshotOfUser might list snapshot directory to find the earliest (#7572)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 1e8a0f16fa [core] Fix that latestSnapshotOfUser might list snapshot 
directory to find the earliest (#7572)
1e8a0f16fa is described below

commit 1e8a0f16fa46d72b89e663fe8586d8583dae5005
Author: yuzelin <33053040+yuze...@users.noreply.github.com>
AuthorDate: Wed Apr 1 15:29:03 2026 +0800

    [core] Fix that latestSnapshotOfUser might list snapshot directory to find 
the earliest (#7572)
    
    Previously, latestSnapshotOfUser called earliestSnapshotId() to
    determine the loop bound before iterating snapshots.
    
    During checkpoint, if the snapshot pointed to by the EARLIEST hint is
    concurrently expired, the hint misses and earliestSnapshotId() falls
    back to findByListFiles(), which lists the entire snapshot directory.
    
    This method is called during every checkpoint via prepareCommit →
    createWriterCleanChecker → latestCommittedIdentifier →
    latestSnapshotOfUserFromFilesystem. For jobs with high parallelism and a
    large number of snapshots (e.g. 10000+), this might cause massive
    listStatus requests, easily triggering QPS limits on object storage
    (e.g. OSS QpsLimitExceeded).
    
    The fix removes the earliestSnapshotId() call and instead iterates
    backward from the latest snapshot, stopping when a FileNotFoundException
    is encountered (indicating an expired snapshot). Other exceptions are
    thrown directly.
---
 .../org/apache/paimon/utils/SnapshotManager.java   | 28 +++++-----------------
 .../apache/paimon/utils/SnapshotManagerTest.java   |  4 ++--
 2 files changed, 8 insertions(+), 24 deletions(-)

diff --git 
a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java 
b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java
index 414ef3f5b7..b6a818604e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java
@@ -600,31 +600,14 @@ public class SnapshotManager implements Serializable {
         if (latestId == null) {
             return Optional.empty();
         }
-        if (earliestId == null) {
-            earliestId =
-                    Preconditions.checkNotNull(
-                            earliestSnapshotId(),
-                            "Latest snapshot id is not null, but earliest 
snapshot id is null. "
-                                    + "This is unexpected.");
-        }
 
-        for (long id = latestId; id >= earliestId; id--) {
+        long searchEnd = earliestId != null ? earliestId : 
Snapshot.FIRST_SNAPSHOT_ID;
+        for (long id = latestId; id >= searchEnd; id--) {
             Snapshot snapshot;
             try {
-                snapshot = snapshot(id);
-            } catch (Exception e) {
-                long newEarliestId =
-                        Preconditions.checkNotNull(
-                                earliestSnapshotId(),
-                                "Latest snapshot id is not null, but earliest 
snapshot id is null. "
-                                        + "This is unexpected.");
-
-                // this is a valid snapshot, should throw exception
-                if (id >= newEarliestId) {
-                    throw e;
-                }
-
-                // this is an expired snapshot
+                snapshot = tryGetSnapshot(id);
+            } catch (FileNotFoundException e) {
+                // this snapshot has been expired, stop searching
                 LOG.warn(
                         "Snapshot #"
                                 + id
@@ -632,6 +615,7 @@ public class SnapshotManager implements Serializable {
                                 + user
                                 + ") is not found.");
                 break;
+                // other exceptions will be thrown
             }
 
             if (user.equals(snapshot.commitUser())) {
diff --git 
a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java 
b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
index 19ae865692..486f0b3fe5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
@@ -334,7 +334,7 @@ public class SnapshotManagerTest {
         SnapshotManager snapshotManager =
                 newSnapshotManager(localFileIO, new Path(tempDir.toString()));
         // create 100 snapshots using user "lastCommitUser"
-        for (long i = 0; i < 100; i++) {
+        for (long i = 1; i <= 100; i++) {
             Snapshot snapshot =
                     new Snapshot(
                             i,
@@ -375,7 +375,7 @@ public class SnapshotManagerTest {
         Thread.sleep(100);
 
         // expire snapshot
-        localFileIO.deleteQuietly(snapshotManager.snapshotPath(0));
+        localFileIO.deleteQuietly(snapshotManager.snapshotPath(1));
         thread.join();
 
         assertThat(exception.get()).isNull();