This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fbe7663d0 [core] Enforce snapshot expire will not expire all snapshots
(#1168)
fbe7663d0 is described below
commit fbe7663d057fc75e27004cf49caffdffe5ce8ce7
Author: 含风 <[email protected]>
AuthorDate: Wed May 17 15:08:28 2023 +0800
[core] Enforce snapshot expire will not expire all snapshots (#1168)
---
docs/content/maintenance/manage-snapshots.md | 4 ++--
docs/layouts/shortcodes/generated/core_configuration.html | 4 ++--
.../src/main/java/org/apache/paimon/CoreOptions.java | 6 ++++--
.../org/apache/paimon/operation/FileStoreExpireImpl.java | 13 +++++++++++--
4 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/docs/content/maintenance/manage-snapshots.md
b/docs/content/maintenance/manage-snapshots.md
index 56008bc90..7b6bd8818 100644
--- a/docs/content/maintenance/manage-snapshots.md
+++ b/docs/content/maintenance/manage-snapshots.md
@@ -59,14 +59,14 @@ Snapshot expiration is controlled by the following table
properties.
<td>No</td>
<td style="word-wrap: break-word;">10</td>
<td>Integer</td>
- <td>The minimum number of completed snapshots to retain.</td>
+ <td>The minimum number of completed snapshots to retain. Should be
greater than or equal to 1.</td>
</tr>
<tr>
<td><h5>snapshot.num-retained.max</h5></td>
<td>No</td>
<td style="word-wrap: break-word;">Integer.MAX_VALUE</td>
<td>Integer</td>
- <td>The maximum number of completed snapshots to retain.</td>
+ <td>The maximum number of completed snapshots to retain. Should be
greater than or equal to the minimum number.</td>
</tr>
</tbody>
</table>
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 9df8991c8..6c2ec9402 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -414,13 +414,13 @@ under the License.
<td><h5>snapshot.num-retained.max</h5></td>
<td style="word-wrap: break-word;">2147483647</td>
<td>Integer</td>
- <td>The maximum number of completed snapshots to retain.</td>
+ <td>The maximum number of completed snapshots to retain. Should be
greater than or equal to the minimum number.</td>
</tr>
<tr>
<td><h5>snapshot.num-retained.min</h5></td>
<td style="word-wrap: break-word;">10</td>
<td>Integer</td>
- <td>The minimum number of completed snapshots to retain.</td>
+ <td>The minimum number of completed snapshots to retain. Should be
greater than or equal to 1.</td>
</tr>
<tr>
<td><h5>snapshot.time-retained</h5></td>
diff --git a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
index 13325d536..5b983dbee 100644
--- a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
@@ -165,13 +165,15 @@ public class CoreOptions implements Serializable {
key("snapshot.num-retained.min")
.intType()
.defaultValue(10)
- .withDescription("The minimum number of completed
snapshots to retain.");
+ .withDescription(
+ "The minimum number of completed snapshots to
retain. Should be greater than or equal to 1.");
public static final ConfigOption<Integer> SNAPSHOT_NUM_RETAINED_MAX =
key("snapshot.num-retained.max")
.intType()
.defaultValue(Integer.MAX_VALUE)
- .withDescription("The maximum number of completed
snapshots to retain.");
+ .withDescription(
+ "The maximum number of completed snapshots to
retain. Should be greater than or equal to the minimum number.");
public static final ConfigOption<Duration> SNAPSHOT_TIME_RETAINED =
key("snapshot.time-retained")
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
index 468fbeb02..cd52589d0 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
@@ -21,6 +21,7 @@ package org.apache.paimon.operation;
import org.apache.paimon.Snapshot;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.consumer.ConsumerManager;
+import org.apache.paimon.utils.Preconditions;
import org.apache.paimon.utils.SnapshotManager;
import org.apache.paimon.utils.TagManager;
@@ -65,6 +66,12 @@ public class FileStoreExpireImpl implements FileStoreExpire {
SnapshotManager snapshotManager,
SnapshotDeletion snapshotDeletion,
TagManager tagManager) {
+ Preconditions.checkArgument(
+ numRetainedMin >= 1,
+ "The minimum number of completed snapshots to retain should be
>= 1.");
+ Preconditions.checkArgument(
+ numRetainedMax >= numRetainedMin,
+ "The maximum number of snapshots to retain should be >= the
minimum number.");
this.numRetainedMin = numRetainedMin;
this.numRetainedMax = numRetainedMax;
this.millisRetained = millisRetained;
@@ -96,7 +103,9 @@ public class FileStoreExpireImpl implements FileStoreExpire {
return;
}
- // find the earliest snapshot to retain
+ // locate the first snapshot between the numRetainedMax th and
(numRetainedMin+1) th latest
+ // snapshots to be retained. This snapshot needs to be preserved
because it
+ // doesn't fulfill the time threshold condition for expiration.
for (long id = Math.max(latestSnapshotId - numRetainedMax + 1,
earliest);
id <= latestSnapshotId - numRetainedMin;
id++) {
@@ -110,7 +119,7 @@ public class FileStoreExpireImpl implements FileStoreExpire
{
}
}
- // no snapshot can be retained, expire until there are only
numRetainedMin snapshots left
+ // by default, expire until there are only numRetainedMin snapshots
left
expireUntil(earliest, latestSnapshotId - numRetainedMin + 1);
}
