This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit d9fda1ef09e84365d0e432c2472e17b5e105183a Author: YangJie <[email protected]> AuthorDate: Fri Sep 11 03:10:24 2026 -0400 [common] Version the memory-mode cache key in CachingFileIO (#9642) --- .../java/org/apache/paimon/utils/FileType.java | 14 +++++- .../apache/paimon/fs/cache/CachingFileIOTest.java | 57 ++++++++++++++++++++++ .../java/org/apache/paimon/utils/FileTypeTest.java | 32 ++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java index 46c767bb24..9b30c864c1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java @@ -114,8 +114,18 @@ public enum FileType { /** Returns {@code true} if the file is mutable and should not be cached. */ public static boolean isMutable(Path filePath) { - String name = filePath.getName(); - return "EARLIEST".equals(name) || "LATEST".equals(name); + String name = unwrapTempFileName(filePath.getName()); + // Files rewritten in place under a stable path: caching them by path keeps serving the + // pre-overwrite content (and a len+mtime key still collides when a rewrite lands at the + // same size within the same clock second). Hint files, consumer and service progress + // files, replaceable tags and _SUCCESS all go through overwriteFileUtf8. + return "EARLIEST".equals(name) + || "LATEST".equals(name) + || "_SUCCESS".equals(name) + || name.endsWith("_SUCCESS") + || name.startsWith(CONSUMER_PREFIX) + || name.startsWith(SERVICE_PREFIX) + || name.startsWith(TAG_PREFIX); } /** diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java index ded23667a9..05ddd7cad0 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java @@ -94,6 +94,63 @@ class CachingFileIOTest { MockFileIO.resetGlobalInputStreamCalls(); } + @Test + void testMemoryModeDoesNotCacheInPlaceOverwrittenFiles() throws IOException { + MockFileIO delegate = new MockFileIO(); + CachingFileIO cachingIO = + newCachingFileIO( + delegate, + new LocalMemoryCacheManager(Long.MAX_VALUE, 64), + EnumSet.of(FileType.META), + 64); + Path consumer = new Path("consumer-1"); + + // consumer-* is written in place by overwriteFileUtf8, so it is mutable and bypasses the + // cache: each read reaches the delegate and sees the current content. + delegate.addFile("consumer-1", "v1cc".getBytes()); + try (SeekableInputStream in = cachingIO.newInputStream(consumer)) { + assertThat(in).isNotInstanceOf(CachingSeekableInputStream.class); + byte[] buf = new byte[4]; + in.read(buf, 0, 4); + assertThat(new String(buf)).isEqualTo("v1cc"); + } + assertThat(delegate.newInputStreamCallCount("consumer-1")).isEqualTo(1); + + delegate.addFile("consumer-1", "v2cc".getBytes()); + try (SeekableInputStream in = cachingIO.newInputStream(consumer)) { + byte[] buf = new byte[4]; + in.read(buf, 0, 4); + assertThat(new String(buf)).isEqualTo("v2cc"); + } + // never cached: the overwrite is visible and the delegate was opened again + assertThat(delegate.newInputStreamCallCount("consumer-1")).isEqualTo(2); + } + + @Test + void testMemoryModeImmutableCacheHitsDoNotRestatDelegate() throws IOException { + MockFileIO delegate = new MockFileIO(); + CachingFileIO cachingIO = + newCachingFileIO( + delegate, + new LocalMemoryCacheManager(Long.MAX_VALUE, 64), + EnumSet.of(FileType.META), + 64); + Path snapshot = new Path("snapshot-1"); + delegate.addFile("snapshot-1", "0123456789abcdef".getBytes()); + + for (int i = 0; i < 3; i++) { + try (SeekableInputStream in = cachingIO.newInputStream(snapshot)) { + assertThat(readAll(in, 16)).isEqualTo("0123456789abcdef".getBytes()); + } + } + + // An immutable file keeps the path-only memory key: opened once, then served from cache. + // Its size is resolved lazily and remembered, so repeated hits do not re-stat the + // delegate the way moving getFileStatus onto every open would. + assertThat(delegate.newInputStreamCallCount("snapshot-1")).isEqualTo(1); + assertThat(delegate.getFileStatusCallCount("snapshot-1")).isEqualTo(1); + } + @Test void testCreateBlobPresignedUrlDelegates() throws IOException { FileIO delegate = mock(FileIO.class); diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java index d946f71bb7..d043d6bf97 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java @@ -31,6 +31,38 @@ public class FileTypeTest { private static final String TABLE_ROOT = "hdfs://cluster/warehouse/db.db/table"; + // ===== mutable files (bypass the cache) ===== + + @Test + public void testIsMutable() { + // Files overwritten in place under a stable path must bypass the cache. + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/snapshot/EARLIEST"))).isTrue(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/snapshot/LATEST"))).isTrue(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/consumer/consumer-myGroup"))) + .isTrue(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/service/service-primary-key-lookup"))) + .isTrue(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/tag/tag-myTag"))).isTrue(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/dt=2024-01-01/bucket-0/_SUCCESS"))) + .isTrue(); + // A temp rewrite of a mutable file is still mutable. + assertThat( + FileType.isMutable( + new Path( + TABLE_ROOT + + "/consumer/.consumer-myGroup." + + UUID.randomUUID() + + ".tmp"))) + .isTrue(); + + // Write-once files stay cacheable: a new version lands under a new name. + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/snapshot/snapshot-1"))).isFalse(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/schema/schema-0"))).isFalse(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/manifest/manifest-a1b2c3d4-0"))) + .isFalse(); + assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/bucket-0/data-abc.orc"))).isFalse(); + } + // ===== META files ===== @Test
