This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3210a3e4b [core] Avoid building the LookupFile cache repeatedly when
the cache is nearly full. (#3859)
3210a3e4b is described below
commit 3210a3e4b4a9e233b794cd20f58485ee975229e6
Author: Aitozi <[email protected]>
AuthorDate: Wed Aug 7 17:45:25 2024 +0800
[core] Avoid building the LookupFile cache repeatedly when the cache is
nearly full. (#3859)
---
.../org/apache/paimon/mergetree/LookupFile.java | 29 +++++++++++++++++-----
.../org/apache/paimon/mergetree/LookupLevels.java | 16 +++++++++---
.../apache/paimon/mergetree/LookupLevelsTest.java | 3 ++-
3 files changed, 37 insertions(+), 11 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupFile.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupFile.java
index 0e05d85a4..097b18655 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupFile.java
@@ -29,9 +29,11 @@ import
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cach
import
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine;
import
org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import javax.annotation.Nullable;
-import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -42,13 +44,17 @@ import static
org.apache.paimon.utils.InternalRowPartitionComputer.toSimpleStrin
import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Lookup file for cache remote file to local. */
-public class LookupFile implements Closeable {
+public class LookupFile {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(LookupFile.class);
private final File localFile;
private final DataFileMeta remoteFile;
private final LookupStoreReader reader;
private final Runnable callback;
+ private long requestCount;
+ private long hitCount;
private boolean isClosed = false;
public LookupFile(
@@ -62,7 +68,12 @@ public class LookupFile implements Closeable {
@Nullable
public byte[] get(byte[] key) throws IOException {
checkArgument(!isClosed);
- return reader.lookup(key);
+ requestCount++;
+ byte[] res = reader.lookup(key);
+ if (res != null) {
+ hitCount++;
+ }
+ return res;
}
public DataFileMeta remoteFile() {
@@ -73,11 +84,17 @@ public class LookupFile implements Closeable {
return isClosed;
}
- @Override
- public void close() throws IOException {
+ public void close(RemovalCause cause) throws IOException {
reader.close();
isClosed = true;
callback.run();
+ LOG.info(
+ "Delete Lookup file {} due to {}. Access stats:
requestCount={}, hitCount={}, size={}KB",
+ localFile.getName(),
+ cause,
+ requestCount,
+ hitCount,
+ localFile.length() >> 10);
FileIOUtils.deleteFileOrDirectory(localFile);
}
@@ -101,7 +118,7 @@ public class LookupFile implements Closeable {
private static void removalCallback(String file, LookupFile lookupFile,
RemovalCause cause) {
if (lookupFile != null) {
try {
- lookupFile.close();
+ lookupFile.close(cause);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
index da1192a6e..f7c783875 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
@@ -128,13 +128,21 @@ public class LookupLevels<T> implements
Levels.DropFileCallback, Closeable {
private T lookup(InternalRow key, DataFileMeta file) throws IOException {
LookupFile lookupFile = lookupFileCache.getIfPresent(file.fileName());
- while (lookupFile == null || lookupFile.isClosed()) {
+ boolean newCreatedLookupFile = false;
+ if (lookupFile == null) {
lookupFile = createLookupFile(file);
- lookupFileCache.put(file.fileName(), lookupFile);
+ newCreatedLookupFile = true;
}
- byte[] keyBytes = keySerializer.serializeToBytes(key);
- byte[] valueBytes = lookupFile.get(keyBytes);
+ byte[] valueBytes;
+ try {
+ byte[] keyBytes = keySerializer.serializeToBytes(key);
+ valueBytes = lookupFile.get(keyBytes);
+ } finally {
+ if (newCreatedLookupFile) {
+ lookupFileCache.put(file.fileName(), lookupFile);
+ }
+ }
if (valueBytes == null) {
return null;
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
index f9b4bf727..76094a520 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
@@ -47,6 +47,7 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.BloomFilter;
import org.apache.paimon.utils.FileStorePathFactory;
+import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -169,7 +170,7 @@ public class LookupLevelsTest {
assertThat(lookupLevels.lookupFiles().estimatedSize()).isEqualTo(0);
}
- @Test
+ @RepeatedTest(value = 10)
public void testMaxDiskSize() throws IOException {
List<DataFileMeta> files = new ArrayList<>();
int fileNum = 10;
