This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ac92d1d08 [core] Rename snapshots to tags in expiration classes
ac92d1d08 is described below
commit ac92d1d084921cc8d3d6310140f0a7e776c0af25
Author: Jingsong <[email protected]>
AuthorDate: Sun Aug 11 21:19:21 2024 +0800
[core] Rename snapshots to tags in expiration classes
---
.../apache/paimon/operation/FileDeletionBase.java | 29 +++++++++++-----------
.../apache/paimon/table/ExpireChangelogImpl.java | 6 ++---
.../apache/paimon/table/ExpireSnapshotsImpl.java | 6 ++---
3 files changed, 20 insertions(+), 21 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
index 978735bab..0ce94bb9b 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
@@ -77,12 +77,11 @@ public abstract class FileDeletionBase<T extends Snapshot> {
protected boolean changelogDecoupled;
- /** Used to record which snapshot is cached. */
- private long cachedSnapshotId = 0;
+ /** Used to record which tag is cached. */
+ private long cachedTag = 0;
- /** Used to cache data files used by current snapshot. */
- private final Map<BinaryRow, Map<Integer, Set<String>>>
cachedSnapshotDataFiles =
- new HashMap<>();
+ /** Used to cache data files used by current tag. */
+ private final Map<BinaryRow, Map<Integer, Set<String>>> cachedTagDataFiles
= new HashMap<>();
public FileDeletionBase(
FileIO fileIO,
@@ -328,18 +327,18 @@ public abstract class FileDeletionBase<T extends
Snapshot> {
cleanUnusedStatisticsManifests(snapshot, skippingSet);
}
- public Predicate<ManifestEntry> dataFileSkipper(
- List<Snapshot> skippingSnapshots, long expiringSnapshotId) throws
Exception {
- int index = SnapshotManager.findPreviousSnapshot(skippingSnapshots,
expiringSnapshotId);
- // refresh snapshot data files
+ public Predicate<ManifestEntry> createDataFileSkipperForTags(
+ List<Snapshot> taggedSnapshots, long expiringSnapshotId) throws
Exception {
+ int index = SnapshotManager.findPreviousSnapshot(taggedSnapshots,
expiringSnapshotId);
+ // refresh tag data files
if (index >= 0) {
- Snapshot previousSnapshot = skippingSnapshots.get(index);
- if (previousSnapshot.id() != cachedSnapshotId) {
- cachedSnapshotId = previousSnapshot.id();
- cachedSnapshotDataFiles.clear();
- addMergedDataFiles(cachedSnapshotDataFiles, previousSnapshot);
+ Snapshot previousTag = taggedSnapshots.get(index);
+ if (previousTag.id() != cachedTag) {
+ cachedTag = previousTag.id();
+ cachedTagDataFiles.clear();
+ addMergedDataFiles(cachedTagDataFiles, previousTag);
}
- return entry -> containsDataFile(cachedSnapshotDataFiles, entry);
+ return entry -> containsDataFile(cachedTagDataFiles, entry);
}
return entry -> false;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
index 72f655bba..246a85cbd 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
@@ -134,11 +134,11 @@ public class ExpireChangelogImpl implements
ExpireSnapshots {
LOG.debug("Changelog expire range is [" + earliestId + ", " +
endExclusiveId + ")");
}
- List<Snapshot> referencedSnapshots = tagManager.taggedSnapshots();
+ List<Snapshot> taggedSnapshots = tagManager.taggedSnapshots();
List<Snapshot> skippingSnapshots =
SnapshotManager.findOverlappedSnapshots(
- referencedSnapshots, earliestId, endExclusiveId);
+ taggedSnapshots, earliestId, endExclusiveId);
skippingSnapshots.add(snapshotManager.changelog(endExclusiveId));
Set<String> manifestSkippSet =
changelogDeletion.manifestSkippingSet(skippingSnapshots);
for (long id = earliestId; id < endExclusiveId; id++) {
@@ -148,7 +148,7 @@ public class ExpireChangelogImpl implements ExpireSnapshots
{
Changelog changelog = snapshotManager.longLivedChangelog(id);
Predicate<ManifestEntry> skipper;
try {
- skipper =
changelogDeletion.dataFileSkipper(referencedSnapshots, id);
+ skipper =
changelogDeletion.createDataFileSkipperForTags(taggedSnapshots, id);
} catch (Exception e) {
LOG.info(
String.format(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
index c49db6498..71abd90e6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
@@ -153,7 +153,7 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots
{
"Snapshot expire range is [" + beginInclusiveId + ", " +
endExclusiveId + ")");
}
- List<Snapshot> referencedSnapshots = tagManager.taggedSnapshots();
+ List<Snapshot> taggedSnapshots = tagManager.taggedSnapshots();
// delete merge tree files
// deleted merge tree files in a snapshot are not used by the next
snapshot, so the range of
@@ -166,7 +166,7 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots
{
// expire merge tree files and collect changed buckets
Predicate<ManifestEntry> skipper;
try {
- skipper =
snapshotDeletion.dataFileSkipper(referencedSnapshots, id);
+ skipper =
snapshotDeletion.createDataFileSkipperForTags(taggedSnapshots, id);
} catch (Exception e) {
LOG.info(
String.format(
@@ -199,7 +199,7 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots
{
// delete manifests and indexFiles
List<Snapshot> skippingSnapshots =
SnapshotManager.findOverlappedSnapshots(
- referencedSnapshots, beginInclusiveId, endExclusiveId);
+ taggedSnapshots, beginInclusiveId, endExclusiveId);
skippingSnapshots.add(snapshotManager.snapshot(endExclusiveId));
Set<String> skippingSet =
snapshotDeletion.manifestSkippingSet(skippingSnapshots);
for (long id = beginInclusiveId; id < endExclusiveId; id++) {
