This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6c8141204d [core] Treat temp and Iceberg hint files as mutable in
FileType (#10342)
6c8141204d is described below
commit 6c8141204d65a90a8a9e93d4250ce6817d16bebc
Author: Dapeng Sun(孙大鹏) <[email protected]>
AuthorDate: Sat Oct 3 17:03:21 2026 +0800
[core] Treat temp and Iceberg hint files as mutable in FileType (#10342)
---
docs/docs/program-api/file-cache.mdx | 8 ++-
docs/generated/catalog_configuration.html | 2 +-
.../org/apache/paimon/options/CatalogOptions.java | 3 +-
.../java/org/apache/paimon/utils/FileType.java | 30 ++++++--
.../java/org/apache/paimon/utils/FileTypeTest.java | 81 ++++++++++++++++++++++
.../paimon/iceberg/IcebergCompatibilityTest.java | 47 +++++++++++++
6 files changed, 162 insertions(+), 9 deletions(-)
diff --git a/docs/docs/program-api/file-cache.mdx
b/docs/docs/program-api/file-cache.mdx
index 8c58a398bc..6d4b2fd24a 100644
--- a/docs/docs/program-api/file-cache.mdx
+++ b/docs/docs/program-api/file-cache.mdx
@@ -52,8 +52,10 @@ The cache classifies files by path. Set
`local-cache.whitelist` to select the fi
| DATA | data | Data files (ORC, Parquet, etc.) | No |
| FILE_INDEX | file-index | Data-file level bloom filter, bitmap | No |
-The default whitelist is `meta,global-index`. The mutable `LATEST` and
`EARLIEST` hint files bypass
-the cache even though they are classified as metadata.
+The default whitelist is `meta,global-index`, and `*` selects every file type.
Files rewritten in
+place under a stable path always bypass the cache, whatever their type: the
`LATEST` and `EARLIEST`
+hint files, tags, consumer and service files, `_SUCCESS` files,
Iceberg-compatible metadata
+(`v{N}.metadata.json`, `version-hint.text`, `retire-pending`), and temporary
files.
## Enable Cache
@@ -126,7 +128,7 @@ The Java snippet is a method-body fragment; declare or
handle the catalog except
| `local-cache.dir` | String | (none) | Directory for storing cached blocks on
disk. If not configured, memory cache is used. |
| `local-cache.max-size` | MemorySize | See below | Maximum cached block bytes
per cache manager. Least recently used blocks are evicted when the limit is
exceeded. |
| `local-cache.block-size` | MemorySize | 1 mb | Block size for caching. Files
are logically divided into fixed-size blocks and cached independently. |
-| `local-cache.whitelist` | String | meta,global-index | Comma-separated list
of file types to cache. Supported values: `meta`, `global-index`,
`bucket-index`, `data`, `file-index`. |
+| `local-cache.whitelist` | String | meta,global-index | Comma-separated list
of file types to cache. Supported values: `meta`, `global-index`,
`bucket-index`, `data`, `file-index`, or `*` for all of them. |
When `local-cache.max-size` is omitted, Java has no configured size limit.
PyPaimon uses a 256 MiB
memory limit or a 10 GiB disk limit. Set the option explicitly when you want
the same limit across
diff --git a/docs/generated/catalog_configuration.html
b/docs/generated/catalog_configuration.html
index 443fce9910..eea1dd9b28 100644
--- a/docs/generated/catalog_configuration.html
+++ b/docs/generated/catalog_configuration.html
@@ -150,7 +150,7 @@ under the License.
<td><h5>local-cache.whitelist</h5></td>
<td style="word-wrap: break-word;">"meta,global-index"</td>
<td>String</td>
- <td>Comma-separated list of file types to cache. Supported values:
meta, global-index, bucket-index, data, file-index.</td>
+ <td>Comma-separated list of file types to cache. Supported values:
meta, global-index, bucket-index, data, file-index, or * for all of them.</td>
</tr>
<tr>
<td><h5>lock-acquire-timeout</h5></td>
diff --git
a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
index fc923cb854..ebb46d1eeb 100644
--- a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
@@ -277,5 +277,6 @@ public class CatalogOptions {
.defaultValue("meta,global-index")
.withDescription(
"Comma-separated list of file types to cache. "
- + "Supported values: meta, global-index,
bucket-index, data, file-index.");
+ + "Supported values: meta, global-index,
bucket-index, data, file-index, "
+ + "or * for all of them.");
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
index 9b30c864c1..a668e90510 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
@@ -65,6 +65,14 @@ public enum FileType {
private static final String INDEX_PREFIX = "index-";
// keep in sync with ChangelogManager.CHANGELOG_PREFIX
private static final String CHANGELOG_PREFIX = "changelog-";
+ // keep in sync with ManifestSidecar.SUFFIX
+ private static final String MANIFEST_SIDECAR_SUFFIX = ".avro.sidecar";
+ // keep in sync with IcebergCommitCallback.VERSION_HINT_FILENAME
+ private static final String VERSION_HINT_FILENAME = "version-hint.text";
+ // keep in sync with IcebergCommitCallback.RETIRE_PENDING_FILENAME
+ private static final String RETIRE_PENDING_FILENAME = "retire-pending";
+ // keep in sync with IcebergPathFactory#toMetadataPath: v{N}.metadata.json
+ private static final String ICEBERG_METADATA_SUFFIX = ".metadata.json";
private static final String MANIFEST = "manifest";
private static final String CHANGELOG_DIR = "changelog";
@@ -84,6 +92,9 @@ public enum FileType {
for (String name : whitelist.split(",")) {
name = name.trim();
switch (name) {
+ case "*":
+ result.addAll(EnumSet.allOf(FileType.class));
+ break;
case "meta":
result.add(META);
break;
@@ -103,7 +114,8 @@ public enum FileType {
if (!name.isEmpty()) {
LOG.warn(
"Unknown local-cache.whitelist value '{}'. "
- + "Supported values: meta,
global-index, bucket-index, data, file-index.",
+ + "Supported values: meta,
global-index, bucket-index, data, file-index, "
+ + "or * for all of them.",
name);
}
break;
@@ -114,7 +126,11 @@ public enum FileType {
/** Returns {@code true} if the file is mutable and should not be cached.
*/
public static boolean isMutable(Path filePath) {
- String name = unwrapTempFileName(filePath.getName());
+ String name = filePath.getName();
+ // Temp files from Path.createTempPath() are renamed away and never
read back by path.
+ if (!unwrapTempFileName(name).equals(name)) {
+ return true;
+ }
// Files rewritten in place under a stable path: caching them by path
keeps serving the
// pre-overwrite content (and a len+mtime key still collides when a
rewrite lands at the
// same size within the same clock second). Hint files, consumer and
service progress
@@ -125,7 +141,12 @@ public enum FileType {
|| name.endsWith("_SUCCESS")
|| name.startsWith(CONSUMER_PREFIX)
|| name.startsWith(SERVICE_PREFIX)
- || name.startsWith(TAG_PREFIX);
+ || name.startsWith(TAG_PREFIX)
+ // Iceberg-compatible metadata rewritten in place by
IcebergCommitCallback,
+ // including v{N}.metadata.json when tags are created or
deleted
+ || VERSION_HINT_FILENAME.equals(name)
+ || RETIRE_PENDING_FILENAME.equals(name)
+ || name.endsWith(ICEBERG_METADATA_SUFFIX);
}
/**
@@ -160,7 +181,8 @@ public enum FileType {
}
// manifest, manifest-list, index-manifest: name contains "manifest"
- if (name.contains(MANIFEST)) {
+ // manifest sidecar: {manifest}.avro.sidecar, whose name need not
contain "manifest"
+ if (name.contains(MANIFEST) || name.endsWith(MANIFEST_SIDECAR_SUFFIX))
{
return META;
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
index d043d6bf97..da2a3a4e93 100644
--- a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
@@ -63,6 +63,48 @@ public class FileTypeTest {
assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/bucket-0/data-abc.orc"))).isFalse();
}
+ @Test
+ public void testTempFilesAreMutable() {
+ String uuid = UUID.randomUUID().toString();
+ Path snapshotTmp = new Path(TABLE_ROOT + "/snapshot/.snapshot-13." +
uuid + ".tmp");
+ assertThat(FileType.classify(snapshotTmp)).isEqualTo(FileType.META);
+ assertThat(FileType.isMutable(snapshotTmp)).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/snapshot/snapshot-13"))).isFalse();
+
+ Path dataTmp = new Path(TABLE_ROOT + "/bucket-0/.data-abc.orc." + uuid
+ ".tmp");
+ assertThat(FileType.classify(dataTmp)).isEqualTo(FileType.DATA);
+ assertThat(FileType.isMutable(dataTmp)).isTrue();
+
+ // Not the createTempPath() format, judged by its own name.
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/bucket-0/data-abc.tmp"))).isFalse();
+ }
+
+ @Test
+ public void testIcebergMetadataIsMutable() {
+ Path versionHint = new Path(TABLE_ROOT +
"/metadata/version-hint.text");
+ assertThat(FileType.isMutable(versionHint)).isTrue();
+ assertThat(FileType.classify(versionHint)).isEqualTo(FileType.DATA);
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/metadata/retire-pending"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/metadata/v3.metadata.json")))
+ .isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/metadata/snap-1-1-abc.avro")))
+ .isFalse();
+ }
+
+ @Test
+ public void testTagSuccessFile() {
+ Path success = new Path(TABLE_ROOT +
"/tag/tag-success-file/t1_SUCCESS");
+ assertThat(FileType.classify(success)).isEqualTo(FileType.META);
+ assertThat(FileType.isMutable(success)).isTrue();
+ }
+
+ @Test
+ public void testChangelogMetadataIsImmutable() {
+ Path changelog = new Path(TABLE_ROOT + "/changelog/changelog-5");
+ assertThat(FileType.classify(changelog)).isEqualTo(FileType.META);
+ assertThat(FileType.isMutable(changelog)).isFalse();
+ }
+
// ===== META files =====
@Test
@@ -117,6 +159,22 @@ public class FileTypeTest {
.isEqualTo(FileType.META);
}
+ @Test
+ public void testManifestSidecarFiles() {
+ Path sidecar = new Path(TABLE_ROOT +
"/manifest/manifest-a1b2c3d4-0.avro.sidecar");
+ assertThat(FileType.classify(sidecar)).isEqualTo(FileType.META);
+ assertThat(FileType.isMutable(sidecar)).isFalse();
+ // Sidecar names come from ManifestFileMeta#extraFiles and need not
contain "manifest".
+ assertThat(
+ FileType.classify(
+ new Path(TABLE_ROOT +
"/manifest/explicit-rewrite.avro.sidecar")))
+ .isEqualTo(FileType.META);
+ assertThat(
+ FileType.classify(
+ new Path(TABLE_ROOT +
"/manifest/index-a1b2c3d4.avro.sidecar")))
+ .isEqualTo(FileType.META);
+ }
+
// ===== BUCKET_INDEX files =====
@Test
@@ -188,6 +246,29 @@ public class FileTypeTest {
.isEqualTo(FileType.FILE_INDEX);
}
+ // ===== parseWhitelist() =====
+
+ @Test
+ public void testParseWhitelist() {
+ assertThat(FileType.parseWhitelist("meta,global-index"))
+ .containsExactlyInAnyOrder(FileType.META,
FileType.GLOBAL_INDEX);
+ assertThat(FileType.parseWhitelist(" data , unknown ,,
")).containsExactly(FileType.DATA);
+ assertThat(FileType.parseWhitelist("")).isEmpty();
+ }
+
+ @Test
+ public void testParseWhitelistWildcard() {
+ FileType[] all = {
+ FileType.META,
+ FileType.DATA,
+ FileType.BUCKET_INDEX,
+ FileType.GLOBAL_INDEX,
+ FileType.FILE_INDEX
+ };
+
assertThat(FileType.parseWhitelist("*")).containsExactlyInAnyOrder(all);
+ assertThat(FileType.parseWhitelist(" meta , *
")).containsExactlyInAnyOrder(all);
+ }
+
// ===== isIndex() =====
@Test
diff --git
a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
index 84a987ba55..8a2df8e51d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
@@ -20,6 +20,7 @@ package org.apache.paimon.iceberg;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.FileSystemCatalog;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BinaryRow;
@@ -44,6 +45,7 @@ import org.apache.paimon.iceberg.metadata.IcebergRef;
import org.apache.paimon.iceberg.metadata.IcebergSchema;
import org.apache.paimon.iceberg.metadata.IcebergSnapshot;
import org.apache.paimon.manifest.ManifestCommittable;
+import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.ExpireConfig;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
@@ -1910,6 +1912,51 @@ public class IcebergCompatibilityTest {
commit.close();
}
+ @Test
+ public void testTagsWithLocalCacheOfAllFileTypes() throws Exception {
+ // Tag changes rewrite v{N}.metadata.json in place, so the local cache
must not serve it.
+ Options catalogOptions = new Options();
+ catalogOptions.set(CatalogOptions.LOCAL_CACHE_ENABLED, true);
+ catalogOptions.set(CatalogOptions.LOCAL_CACHE_WHITELIST, "*");
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.BUCKET.key(), "1");
+ options.put(IcebergOptions.METADATA_ICEBERG_STORAGE.key(),
"table-location");
+ Schema schema =
+ Schema.newBuilder()
+ .column("k", DataTypes.INT())
+ .column("v", DataTypes.INT())
+ .primaryKey("k")
+ .options(options)
+ .build();
+
+ try (FileSystemCatalog catalog =
+ new FileSystemCatalog(
+ LocalFileIO.create(),
+ new Path(tempDir.toString()),
+ CatalogContext.create(catalogOptions))) {
+ catalog.createDatabase("mydb", false);
+ Identifier identifier = Identifier.create("mydb", "t");
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = (FileStoreTable)
catalog.getTable(identifier);
+
+ String commitUser = UUID.randomUUID().toString();
+ try (TableWriteImpl<?> write = table.newWrite(commitUser);
+ TableCommitImpl commit = table.newCommit(commitUser)) {
+ write.write(GenericRow.of(1, 10));
+ commit.commit(1, write.prepareCommit(false, 1));
+ }
+
+ table.createTag("t1", 1);
+ table.createTag("t2", 1);
+ table.createTag("t3", 1);
+ table.deleteTag("t2");
+
+ Path metadata = new Path(table.location(),
"metadata/v1.metadata.json");
+ assertThat(IcebergMetadata.fromPath(LocalFileIO.create(),
metadata).refs().keySet())
+ .containsExactlyInAnyOrder("t1", "t3");
+ }
+ }
+
// Create snapshots and Iceberg metadata
// Delete Iceberg metadata
// Create tag - this should not create any metadata file
