This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 21f60b80 test(file_index): verify Java table read compatibility (#876)
21f60b80 is described below
commit 21f60b80bc02c4227905b4dd629c603768ae5c6e
Author: Wuhen- Li <[email protected]>
AuthorDate: Tue Sep 22 14:16:29 2026 +0800
test(file_index): verify Java table read compatibility (#876)
---
crates/paimon/testdata/file_index/README.md | 38 +++
...-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet | Bin 0 -> 564 bytes
...manifest-982c31e0-7caa-4462-a270-3b130005e913-0 | Bin 0 -> 2273 bytes
...est-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0 | Bin 0 -> 1158 bytes
...est-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1 | Bin 0 -> 1263 bytes
.../default.db/bitmap_embedded/schema/schema-0 | 25 ++
.../default.db/bitmap_embedded/snapshot/EARLIEST | 1 +
.../default.db/bitmap_embedded/snapshot/LATEST | 1 +
.../default.db/bitmap_embedded/snapshot/snapshot-1 | 18 ++
...-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet | Bin 0 -> 564 bytes
...532-1aa9-4878-bb30-24862120212c-0.parquet.index | Bin 0 -> 130 bytes
...manifest-392c52de-fdae-4262-81fb-4049e95c37de-0 | Bin 0 -> 2211 bytes
...est-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0 | Bin 0 -> 1158 bytes
...est-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1 | Bin 0 -> 1263 bytes
.../default.db/bitmap_sidecar/schema/schema-0 | 25 ++
.../default.db/bitmap_sidecar/snapshot/EARLIEST | 1 +
.../default.db/bitmap_sidecar/snapshot/LATEST | 1 +
.../default.db/bitmap_sidecar/snapshot/snapshot-1 | 18 ++
...-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet | Bin 0 -> 564 bytes
...manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0 | Bin 0 -> 2255 bytes
...est-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0 | Bin 0 -> 1158 bytes
...est-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1 | Bin 0 -> 1263 bytes
.../bloom_filter_embedded/schema/schema-0 | 27 ++
.../bloom_filter_embedded/snapshot/EARLIEST | 1 +
.../bloom_filter_embedded/snapshot/LATEST | 1 +
.../bloom_filter_embedded/snapshot/snapshot-1 | 18 ++
...-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet | Bin 0 -> 564 bytes
...26c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index | Bin 0 -> 78 bytes
...manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0 | Bin 0 -> 2212 bytes
...est-list-baa54195-a358-4f0b-9ca9-51503661a534-0 | Bin 0 -> 1158 bytes
...est-list-baa54195-a358-4f0b-9ca9-51503661a534-1 | Bin 0 -> 1260 bytes
.../bloom_filter_sidecar/schema/schema-0 | 27 ++
.../bloom_filter_sidecar/snapshot/EARLIEST | 1 +
.../bloom_filter_sidecar/snapshot/LATEST | 1 +
.../bloom_filter_sidecar/snapshot/snapshot-1 | 18 ++
.../JavaFileIndexTableFixtureGeneratorTest.java | 108 ++++++++
crates/paimon/testdata/file_index/regenerate.sh | 64 +++++
.../paimon/tests/file_index_java_fixture_test.rs | 280 +++++++++++++++++++++
38 files changed, 674 insertions(+)
diff --git a/crates/paimon/testdata/file_index/README.md
b/crates/paimon/testdata/file_index/README.md
new file mode 100644
index 00000000..dee46286
--- /dev/null
+++ b/crates/paimon/testdata/file_index/README.md
@@ -0,0 +1,38 @@
+# Java FileIndex table fixtures
+
+These four committed tables were produced through Apache Paimon Java's normal
+catalog, append writer, commit, snapshot, and manifest paths. No index bytes or
+metadata were assembled or edited by hand.
+
+Provenance:
+
+- Apache Paimon Java commit: `1d368b4a5932f8221fd28e2555001abdb8fb12ee`
+ (`2.2-SNAPSHOT`).
+- Generator: `java/JavaFileIndexTableFixtureGeneratorTest.java`.
+- Tables: Bitmap and Bloom Filter, each with an embedded (`1 MB` threshold) and
+ `.index` sidecar (`1 B` threshold) form.
+- Common options: `bucket=1`, `bucket-key=id`, `file.format=parquet`, and
+ `file-index.read.enabled=true`.
+- Bloom options: `items=16`, `fpp=0.01`.
+- Rows, in physical order: `(1, "keep")`, `(1, "drop")`,
+ `(NULL, "null-id")`, `(3, "three")`.
+
+The Rust test expects `id = 1` to return both duplicate rows, `id IS NULL` to
+return the null row, `id = 1 AND payload = 'keep'` to retain only the residual
+match, and `id = 2` to return no rows. The last predicate lies inside the data
+file's `[1, 3]` min/max range, so manifest statistics cannot prune it; it is
also
+the observable FileIndex skip case.
+
+To regenerate into an empty temporary directory:
+
+```bash
+git clone https://github.com/apache/paimon.git /tmp/paimon-java
+git -C /tmp/paimon-java checkout 1d368b4a5932f8221fd28e2555001abdb8fb12ee
+crates/paimon/testdata/file_index/regenerate.sh \
+ /tmp/paimon-java /tmp/paimon-file-index-fixtures
+```
+
+After reviewing the generated metadata and running the Rust compatibility test,
+replace `default.db/` with the generated `default.db/` directory. UUID-bearing
+file names may change between runs; the rows, table options, storage shape, and
+query expectations are deterministic.
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet
new file mode 100644
index 00000000..d411cc37
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/bucket-0/data-c6f0f8ca-a74f-42ce-af35-fc4f4ef2d003-0.parquet
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0
new file mode 100644
index 00000000..004f091e
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-982c31e0-7caa-4462-a270-3b130005e913-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0
new file mode 100644
index 00000000..d61cedd4
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1
new file mode 100644
index 00000000..ea7c30f5
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/manifest/manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0
new file mode 100644
index 00000000..a0325d02
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/schema/schema-0
@@ -0,0 +1,25 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "payload",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "1",
+ "file-index.bitmap.columns" : "id",
+ "bucket-key" : "id",
+ "file-index.read.enabled" : "true",
+ "file.format" : "parquet",
+ "file-index.in-manifest-threshold" : "1 MB"
+ },
+ "timeMillis" : 1789814325908
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1
new file mode 100644
index 00000000..df0c91ae
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_embedded/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "c566080a-5959-4fba-b8ec-ed10fea2d95f",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-9d29ca76-96e0-424d-8ccf-28d75ee5b70e-1",
+ "deltaManifestListSize" : 1263,
+ "commitUser" : "9fa4d883-e8c7-4e38-8f41-40006e74ac3c",
+ "writerVersion" :
"java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789814327145,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 4,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet
new file mode 100644
index 00000000..d411cc37
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index
new file mode 100644
index 00000000..c4886674
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/bucket-0/data-d9e65532-1aa9-4878-bb30-24862120212c-0.parquet.index
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0
new file mode 100644
index 00000000..205ac329
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-392c52de-fdae-4262-81fb-4049e95c37de-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0
new file mode 100644
index 00000000..e1ea4553
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1
new file mode 100644
index 00000000..0f1da8f8
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/manifest/manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0
new file mode 100644
index 00000000..4fc7be2f
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/schema/schema-0
@@ -0,0 +1,25 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "payload",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "1",
+ "file-index.bitmap.columns" : "id",
+ "bucket-key" : "id",
+ "file-index.read.enabled" : "true",
+ "file.format" : "parquet",
+ "file-index.in-manifest-threshold" : "1 B"
+ },
+ "timeMillis" : 1789814327169
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1
new file mode 100644
index 00000000..24b6b600
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bitmap_sidecar/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "6476f620-ef94-47ba-8250-c49ab25124f2",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-973ebadd-00a4-4672-a3b1-81e48cc9ec83-1",
+ "deltaManifestListSize" : 1263,
+ "commitUser" : "71026591-a529-45f5-9b77-71d993270c4b",
+ "writerVersion" :
"java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789814327179,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 4,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet
new file mode 100644
index 00000000..d411cc37
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/bucket-0/data-9ce3b83d-6c45-4dc1-a34e-819b5fb2e331-0.parquet
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0
new file mode 100644
index 00000000..c56a063b
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-8dd937f3-747e-4f87-a296-489d11ef050c-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0
new file mode 100644
index 00000000..715016ae
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1
new file mode 100644
index 00000000..b592f52e
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/manifest/manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0
new file mode 100644
index 00000000..4aa8eb0b
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/schema/schema-0
@@ -0,0 +1,27 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "payload",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "1",
+ "file-index.bloom-filter.id.items" : "16",
+ "file-index.in-manifest-threshold" : "1 MB",
+ "file-index.bloom-filter.columns" : "id",
+ "file-index.bloom-filter.id.fpp" : "0.01",
+ "bucket-key" : "id",
+ "file-index.read.enabled" : "true",
+ "file.format" : "parquet"
+ },
+ "timeMillis" : 1789814327180
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1
new file mode 100644
index 00000000..35573b11
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_embedded/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "e21872d6-c6a1-4ef4-9a26-fe324ccd15cf",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-ac933951-c2f2-4af6-b3d4-d20e80e79189-1",
+ "deltaManifestListSize" : 1263,
+ "commitUser" : "1930fc30-d2c7-493c-b2cc-554a5b504d22",
+ "writerVersion" :
"java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789814327189,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 4,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet
new file mode 100644
index 00000000..d411cc37
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index
new file mode 100644
index 00000000..5a5fcc6a
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/bucket-0/data-2e01926c-6cd0-4dd5-b576-80ddf6e55632-0.parquet.index
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0
new file mode 100644
index 00000000..a3525f30
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-214eaa6e-a8e8-4903-a41a-3fa04732e5e4-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0
new file mode 100644
index 00000000..d8d82bf6
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1
new file mode 100644
index 00000000..84f9fd47
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/manifest/manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0
new file mode 100644
index 00000000..c37d6b22
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/schema/schema-0
@@ -0,0 +1,27 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "payload",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "1",
+ "file-index.bloom-filter.id.items" : "16",
+ "file-index.in-manifest-threshold" : "1 B",
+ "file-index.bloom-filter.columns" : "id",
+ "file-index.bloom-filter.id.fpp" : "0.01",
+ "bucket-key" : "id",
+ "file-index.read.enabled" : "true",
+ "file.format" : "parquet"
+ },
+ "timeMillis" : 1789814327190
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1
new file mode 100644
index 00000000..d6a12ad5
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/bloom_filter_sidecar/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "4775d461-aa56-46ac-b12f-7aa8976a8b7f",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-baa54195-a358-4f0b-9ca9-51503661a534-1",
+ "deltaManifestListSize" : 1260,
+ "commitUser" : "04f8ab29-9471-4204-8e7c-6340649da117",
+ "writerVersion" :
"java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1789814327197,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 4,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
b/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
new file mode 100644
index 00000000..6c592f7b
--- /dev/null
+++
b/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fixture;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.paimon.options.CatalogOptions.WAREHOUSE;
+
+/** Generates committed Java tables used by paimon-rust FileIndex
compatibility tests. */
+public class JavaFileIndexTableFixtureGeneratorTest {
+
+ private static final String DATABASE = "default";
+
+ @Test
+ public void generate() throws Exception {
+ String output =
System.getProperty("generate.file-index.fixture.output");
+ if (output == null || output.trim().isEmpty()) {
+ throw new IllegalArgumentException(
+ "Set
-Dgenerate.file-index.fixture.output=<empty-output-directory>");
+ }
+
+ Path warehouse = Paths.get(output).toAbsolutePath().normalize();
+ Options catalogOptions = new Options();
+ catalogOptions.set(WAREHOUSE, warehouse.toUri().toString());
+ Catalog catalog =
CatalogFactory.createCatalog(CatalogContext.create(catalogOptions));
+ catalog.createDatabase(DATABASE, false);
+
+ writeTable(catalog, "bitmap_embedded", "bitmap", "1 MB");
+ writeTable(catalog, "bitmap_sidecar", "bitmap", "1 B");
+ writeTable(catalog, "bloom_filter_embedded", "bloom-filter", "1 MB");
+ writeTable(catalog, "bloom_filter_sidecar", "bloom-filter", "1 B");
+ }
+
+ private static void writeTable(
+ Catalog catalog, String tableName, String indexType, String
manifestThreshold)
+ throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put("bucket", "1");
+ options.put("bucket-key", "id");
+ options.put("file.format", "parquet");
+ options.put("file-index.read.enabled", "true");
+ options.put("file-index.in-manifest-threshold", manifestThreshold);
+ options.put("file-index." + indexType + ".columns", "id");
+ if ("bloom-filter".equals(indexType)) {
+ options.put("file-index.bloom-filter.id.items", "16");
+ options.put("file-index.bloom-filter.id.fpp", "0.01");
+ }
+
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("payload", DataTypes.STRING())
+ .options(options)
+ .build();
+ Identifier identifier = Identifier.create(DATABASE, tableName);
+ catalog.createTable(identifier, schema, false);
+ Table table = catalog.getTable(identifier);
+
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite();
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(row(1, "keep"));
+ write.write(row(1, "drop"));
+ write.write(row(null, "null-id"));
+ write.write(row(3, "three"));
+ commit.commit(write.prepareCommit());
+ }
+ }
+
+ private static GenericRow row(Integer id, String payload) {
+ return GenericRow.of(id, BinaryString.fromString(payload));
+ }
+}
diff --git a/crates/paimon/testdata/file_index/regenerate.sh
b/crates/paimon/testdata/file_index/regenerate.sh
new file mode 100755
index 00000000..2905c723
--- /dev/null
+++ b/crates/paimon/testdata/file_index/regenerate.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+ echo "usage: $0 <paimon-java-checkout> <empty-output-directory>" >&2
+ exit 2
+fi
+
+readonly EXPECTED_COMMIT="1d368b4a5932f8221fd28e2555001abdb8fb12ee"
+readonly JAVA_CHECKOUT="$1"
+readonly OUTPUT="$2"
+readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+readonly SOURCE="$SCRIPT_DIR/java/JavaFileIndexTableFixtureGeneratorTest.java"
+readonly
TARGET="$JAVA_CHECKOUT/paimon-core/src/test/java/org/apache/paimon/fixture/JavaFileIndexTableFixtureGeneratorTest.java"
+
+actual_commit="$(git -C "$JAVA_CHECKOUT" rev-parse HEAD)"
+if [[ "$actual_commit" != "$EXPECTED_COMMIT" ]]; then
+ echo "expected Apache Paimon Java $EXPECTED_COMMIT, got $actual_commit" >&2
+ exit 1
+fi
+if [[ -e "$TARGET" ]]; then
+ echo "refusing to overwrite $TARGET" >&2
+ exit 1
+fi
+if [[ -e "$OUTPUT" ]] && [[ -n "$(find "$OUTPUT" -mindepth 1 -print -quit)"
]]; then
+ echo "output directory must be empty: $OUTPUT" >&2
+ exit 1
+fi
+
+mkdir -p "$(dirname "$TARGET")" "$OUTPUT"
+cp "$SOURCE" "$TARGET"
+trap 'rm -f "$TARGET"' EXIT
+
+mvn -f "$JAVA_CHECKOUT/pom.xml" \
+ -pl paimon-codegen-loader -am -Pfast-build \
+ -DskipTests \
+ package
+
+mvn -f "$JAVA_CHECKOUT/pom.xml" \
+ -pl paimon-core -am -Pfast-build \
+ -DfailIfNoTests=false \
+ -DwildcardSuites=none \
+ -Dtest=JavaFileIndexTableFixtureGeneratorTest \
+ -Dgenerate.file-index.fixture.output="$OUTPUT" \
+ test
+
+echo "generated fixtures under $OUTPUT/default.db"
diff --git a/crates/paimon/tests/file_index_java_fixture_test.rs
b/crates/paimon/tests/file_index_java_fixture_test.rs
new file mode 100644
index 00000000..87a86fed
--- /dev/null
+++ b/crates/paimon/tests/file_index_java_fixture_test.rs
@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! End-to-end compatibility coverage for FileIndex tables written and
committed
+//! by Apache Paimon Java. See `testdata/file_index/README.md` for provenance
and
+//! regeneration instructions.
+
+#![cfg(all(not(target_os = "windows"), feature = "storage-fs"))]
+
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use arrow_array::{Array, Int32Array, RecordBatch, StringArray};
+use futures::TryStreamExt;
+use opendal::Operator;
+use paimon::catalog::Identifier;
+use paimon::io::{FileIO, FileIOBuilder, FileIOProvider};
+use paimon::spec::{Datum, Predicate, PredicateBuilder};
+use paimon::table::{SchemaManager, Table};
+
+const FIXTURE: &str = "testdata/file_index/default.db";
+
+#[derive(Clone, Copy)]
+struct FixtureCase {
+ table: &'static str,
+ embedded: bool,
+}
+
+const CASES: &[FixtureCase] = &[
+ FixtureCase {
+ table: "bitmap_embedded",
+ embedded: true,
+ },
+ FixtureCase {
+ table: "bitmap_sidecar",
+ embedded: false,
+ },
+ FixtureCase {
+ table: "bloom_filter_embedded",
+ embedded: true,
+ },
+ FixtureCase {
+ table: "bloom_filter_sidecar",
+ embedded: false,
+ },
+];
+
+#[derive(Debug)]
+struct FsProbe {
+ operator: Operator,
+ data_file_opens: AtomicUsize,
+}
+
+impl FsProbe {
+ fn new() -> Arc<Self> {
+ let mut config = opendal_service_fs::FsConfig::default();
+ config.root = Some("/".to_string());
+ Arc::new(Self {
+ operator: Operator::from_config(config).expect("build filesystem
operator"),
+ data_file_opens: AtomicUsize::new(0),
+ })
+ }
+
+ fn reset(&self) {
+ self.data_file_opens.store(0, Ordering::SeqCst);
+ }
+
+ fn data_file_opens(&self) -> usize {
+ self.data_file_opens.load(Ordering::SeqCst)
+ }
+}
+
+#[async_trait::async_trait]
+impl FileIOProvider for FsProbe {
+ async fn create(&self, path: &str) -> paimon::Result<(Operator, String)> {
+ if path.ends_with(".parquet") {
+ self.data_file_opens.fetch_add(1, Ordering::SeqCst);
+ }
+ let relative = path
+ .strip_prefix("file://")
+ .expect("fixture paths must be file URLs")
+ .trim_start_matches('/')
+ .to_string();
+ Ok((self.operator.clone(), relative))
+ }
+}
+
+fn copy_dir(src: &Path, dst: &Path) {
+ std::fs::create_dir_all(dst).expect("create staged fixture directory");
+ for entry in std::fs::read_dir(src).expect("read fixture directory") {
+ let entry = entry.expect("read fixture entry");
+ let from = entry.path();
+ let to = dst.join(entry.file_name());
+ if from.is_dir() {
+ copy_dir(&from, &to);
+ } else {
+ std::fs::copy(&from, &to).expect("copy fixture file");
+ }
+ }
+}
+
+async fn open_fixture(case: FixtureCase) -> (tempfile::TempDir, Table,
Arc<FsProbe>) {
+ let src = Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join(FIXTURE)
+ .join(case.table);
+ let temp = tempfile::tempdir().expect("create temp dir");
+ let staged = temp.path().join(case.table);
+ copy_dir(&src, &staged);
+
+ let probe = FsProbe::new();
+ let file_io: FileIO = FileIOBuilder::new("file")
+ .with_provider(probe.clone())
+ .build()
+ .expect("build probed FileIO");
+ let location = format!("file://{}", staged.display());
+ let schema = SchemaManager::new(file_io.clone(), location.clone())
+ .latest()
+ .await
+ .expect("load fixture schemas")
+ .expect("fixture table has no schema");
+ let table = Table::new(
+ file_io,
+ Identifier::new("default", case.table),
+ location,
+ (*schema).clone(),
+ None,
+ );
+ (temp, table, probe)
+}
+
+fn rows(batches: &[RecordBatch]) -> Vec<(Option<i32>, String)> {
+ let mut rows = Vec::new();
+ for batch in batches {
+ let ids = batch
+ .column_by_name("id")
+ .expect("id column")
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .expect("id is int32");
+ let payloads = batch
+ .column_by_name("payload")
+ .expect("payload column")
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .expect("payload is utf8");
+ rows.extend((0..batch.num_rows()).map(|row| {
+ let id = (!ids.is_null(row)).then(|| ids.value(row));
+ (id, payloads.value(row).to_string())
+ }));
+ }
+ rows.sort_unstable();
+ rows
+}
+
+async fn query(
+ table: &Table,
+ index_enabled: bool,
+ predicate: Predicate,
+) -> Vec<(Option<i32>, String)> {
+ let table = table.copy_with_options(HashMap::from([(
+ "file-index.read.enabled".to_string(),
+ index_enabled.to_string(),
+ )]));
+ let mut builder = table.new_read_builder();
+ builder.with_filter(predicate);
+ let plan = builder.new_scan().plan().await.expect("plan fixture query");
+ let batches = builder
+ .new_read()
+ .expect("create fixture reader")
+ .to_arrow(plan.splits())
+ .expect("create fixture Arrow stream")
+ .try_collect::<Vec<_>>()
+ .await
+ .expect("read fixture rows");
+ rows(&batches)
+}
+
+#[tokio::test]
+async fn reads_java_written_file_indexes_and_prunes_data_files() {
+ for case in CASES.iter().copied() {
+ let (_temp, table, probe) = open_fixture(case).await;
+ let predicates = PredicateBuilder::new(table.schema().fields());
+ let id_one = predicates.equal("id", Datum::Int(1)).unwrap();
+ let null_id = predicates.is_null("id").unwrap();
+ let missing_id = predicates.equal("id", Datum::Int(2)).unwrap();
+ let residual = Predicate::and(vec![
+ id_one.clone(),
+ predicates
+ .equal("payload", Datum::String("keep".to_string()))
+ .unwrap(),
+ ]);
+
+ let mut planning_builder = table.new_read_builder();
+ planning_builder.with_filter(missing_id.clone());
+ let (plan, trace) = planning_builder
+ .new_scan()
+ .plan_with_trace()
+ .await
+ .expect("plan missing-value query");
+ assert_eq!(
+ trace.manifest_entries_pruned_by_data_stats, 0,
+ "{}",
+ case.table
+ );
+ assert_eq!(trace.final_files, 1, "{}", case.table);
+ let file = &plan.splits()[0].data_files()[0];
+ assert_eq!(file.row_count, 4, "{}", case.table);
+ assert_eq!(
+ file.embedded_index.is_some(),
+ case.embedded,
+ "{}",
+ case.table
+ );
+ assert_eq!(
+ file.extra_files.iter().any(|name| name.ends_with(".index")),
+ !case.embedded,
+ "{}",
+ case.table
+ );
+
+ for enabled in [false, true] {
+ assert_eq!(
+ query(&table, enabled, id_one.clone()).await,
+ vec![(Some(1), "drop".to_string()), (Some(1),
"keep".to_string())],
+ "{} with FileIndex enabled={enabled}",
+ case.table
+ );
+ assert_eq!(
+ query(&table, enabled, null_id.clone()).await,
+ vec![(None, "null-id".to_string())],
+ "{} with FileIndex enabled={enabled}",
+ case.table
+ );
+ assert_eq!(
+ query(&table, enabled, residual.clone()).await,
+ vec![(Some(1), "keep".to_string())],
+ "{} must retain row-level residual filtering with FileIndex
enabled={enabled}",
+ case.table
+ );
+ assert!(
+ query(&table, enabled, missing_id.clone()).await.is_empty(),
+ "{} with FileIndex enabled={enabled}",
+ case.table
+ );
+ }
+
+ probe.reset();
+ assert!(query(&table, true, missing_id.clone()).await.is_empty());
+ assert_eq!(
+ probe.data_file_opens(),
+ 0,
+ "{} must be skipped by its Java-written FileIndex",
+ case.table
+ );
+
+ probe.reset();
+ assert!(query(&table, false, missing_id).await.is_empty());
+ assert!(
+ probe.data_file_opens() > 0,
+ "{} must open the data file when FileIndex reads are disabled",
+ case.table
+ );
+ }
+}
