This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new b1b1268ca [refactor] Remove useless class TagFileKeeper#DataFileInfo
(#1349)
b1b1268ca is described below
commit b1b1268cabaf57e940b5cc7d17fa809b8e8f3a13
Author: yuzelin <[email protected]>
AuthorDate: Fri Jun 9 18:49:24 2023 +0800
[refactor] Remove useless class TagFileKeeper#DataFileInfo (#1349)
---
.../apache/paimon/operation/SnapshotDeletion.java | 20 ++++++-------
.../org/apache/paimon/operation/TagFileKeeper.java | 33 ++++------------------
2 files changed, 15 insertions(+), 38 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
index 8b0a4c48f..7cc39e753 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
@@ -84,7 +84,7 @@ public class SnapshotDeletion {
public void deleteExpiredDataFiles(
String manifestListName,
Map<BinaryRow, Set<Integer>> deletionBuckets,
- Predicate<TagFileKeeper.DataFileInfo> dataFileSkipper) {
+ Predicate<ManifestEntry> dataFileSkipper) {
doDeleteExpiredDataFiles(
getManifestEntriesFromManifestList(manifestListName),
deletionBuckets,
@@ -206,11 +206,11 @@ public class SnapshotDeletion {
void doDeleteExpiredDataFiles(
Iterable<ManifestEntry> dataFileLog,
Map<BinaryRow, Set<Integer>> deletionBuckets,
- Predicate<TagFileKeeper.DataFileInfo> dataFileSkipper) {
+ Predicate<ManifestEntry> dataFileSkipper) {
// we cannot delete a data file directly when we meet a DELETE entry,
because that
// file might be upgraded
- // data file path -> (data file info, extra file paths)
- Map<Path, Pair<TagFileKeeper.DataFileInfo, List<Path>>>
dataFileToDelete = new HashMap<>();
+ // data file path -> (original manifest entry, extra file paths)
+ Map<Path, Pair<ManifestEntry, List<Path>>> dataFileToDelete = new
HashMap<>();
for (ManifestEntry entry : dataFileLog) {
Path bucketPath = pathFactory.bucketPath(entry.partition(),
entry.bucket());
Path dataFilePath = new Path(bucketPath, entry.file().fileName());
@@ -223,9 +223,7 @@ public class SnapshotDeletion {
for (String file : entry.file().extraFiles()) {
extraFiles.add(new Path(bucketPath, file));
}
- dataFileToDelete.put(
- dataFilePath,
- Pair.of(TagFileKeeper.DataFileInfo.of(entry),
extraFiles));
+ dataFileToDelete.put(dataFilePath, Pair.of(entry,
extraFiles));
break;
default:
throw new UnsupportedOperationException(
@@ -235,16 +233,16 @@ public class SnapshotDeletion {
dataFileToDelete.forEach(
(path, pair) -> {
- TagFileKeeper.DataFileInfo info = pair.getLeft();
+ ManifestEntry entry = pair.getLeft();
// check whether we should skip the data file
- if (!dataFileSkipper.test(info)) {
+ if (!dataFileSkipper.test(entry)) {
// delete data files
fileIO.deleteQuietly(path);
pair.getRight().forEach(fileIO::deleteQuietly);
// record changed buckets
deletionBuckets
- .computeIfAbsent(info.partition, p -> new
HashSet<>())
- .add(info.bucket);
+ .computeIfAbsent(entry.partition(), p -> new
HashSet<>())
+ .add(entry.bucket());
}
});
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/TagFileKeeper.java
b/paimon-core/src/main/java/org/apache/paimon/operation/TagFileKeeper.java
index 20e2d32aa..dd064fd7a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/TagFileKeeper.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/TagFileKeeper.java
@@ -60,12 +60,12 @@ public class TagFileKeeper {
taggedSnapshots = tagManager.taggedSnapshots();
}
- public Predicate<DataFileInfo> tagDataFileSkipper(long expiringSnapshotId)
{
+ public Predicate<ManifestEntry> tagDataFileSkipper(long
expiringSnapshotId) {
int index = findPreviousTag(expiringSnapshotId, taggedSnapshots);
if (index >= 0) {
tryRefresh(taggedSnapshots.get(index));
}
- return dataFileInfo -> index >= 0 && contains(dataFileInfo);
+ return entry -> index >= 0 && contains(entry);
}
public Set<String> collectManifestSkippingSet(long beginInclusive, long
endExclusive) {
@@ -114,12 +114,12 @@ public class TagFileKeeper {
}
}
- private boolean contains(DataFileInfo dataFileInfo) {
- Map<Integer, Set<String>> buckets =
cachedTagDataFiles.get(dataFileInfo.partition);
+ private boolean contains(ManifestEntry entry) {
+ Map<Integer, Set<String>> buckets =
cachedTagDataFiles.get(entry.partition());
if (buckets != null) {
- Set<String> fileNames = buckets.get(dataFileInfo.bucket);
+ Set<String> fileNames = buckets.get(entry.bucket());
if (fileNames != null) {
- return fileNames.contains(dataFileInfo.fileName);
+ return fileNames.contains(entry.file().fileName());
}
}
return false;
@@ -142,25 +142,4 @@ public class TagFileKeeper {
}
return -1;
}
-
- /** To accommodate information of a data file. */
- static class DataFileInfo {
-
- public final BinaryRow partition;
- public final int bucket;
- public final String fileName;
-
- DataFileInfo(BinaryRow partition, int bucket, String fileName) {
- this.partition = partition;
- this.bucket = bucket;
- this.fileName = fileName;
- }
-
- static DataFileInfo of(ManifestEntry manifestEntry) {
- return new DataFileInfo(
- manifestEntry.partition(),
- manifestEntry.bucket(),
- manifestEntry.file().fileName());
- }
- }
}
