This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 78ae3fe0 test(file_index): verify Java range-bitmap table reads (#953)
78ae3fe0 is described below
commit 78ae3fe0e0186505f968d90297b3f42e5e800c8b
Author: Wuhen- Li <[email protected]>
AuthorDate: Fri Sep 25 20:50:04 2026 +0800
test(file_index): verify Java range-bitmap table reads (#953)
---
crates/paimon/testdata/file_index/README.md | 19 ++++++----
...-791a54a7-4d19-43c3-8752-31cd6073433a-0.parquet | Bin 0 -> 565 bytes
...manifest-155640a3-e50d-4be7-ba72-de0afe2a8fec-0 | Bin 0 -> 2302 bytes
...est-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-0 | Bin 0 -> 1158 bytes
...est-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-1 | Bin 0 -> 1263 bytes
.../range_bitmap_embedded/schema/schema-0 | 25 ++++++++++++
.../range_bitmap_embedded/snapshot/EARLIEST | 1 +
.../range_bitmap_embedded/snapshot/LATEST | 1 +
.../range_bitmap_embedded/snapshot/snapshot-1 | 18 +++++++++
...-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet | Bin 0 -> 565 bytes
...c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet.index | Bin 0 -> 191 bytes
...manifest-e1c11d52-6e22-4d10-b62a-833b60216bf2-0 | Bin 0 -> 2211 bytes
...est-list-107ca82a-e4a8-4a88-8439-b15d770f0474-0 | Bin 0 -> 1158 bytes
...est-list-107ca82a-e4a8-4a88-8439-b15d770f0474-1 | Bin 0 -> 1263 bytes
.../range_bitmap_sidecar/schema/schema-0 | 25 ++++++++++++
.../range_bitmap_sidecar/snapshot/EARLIEST | 1 +
.../range_bitmap_sidecar/snapshot/LATEST | 1 +
.../range_bitmap_sidecar/snapshot/snapshot-1 | 18 +++++++++
.../JavaFileIndexTableFixtureGeneratorTest.java | 8 +++-
.../paimon/tests/file_index_java_fixture_test.rs | 42 ++++++++++++++++++---
20 files changed, 145 insertions(+), 14 deletions(-)
diff --git a/crates/paimon/testdata/file_index/README.md
b/crates/paimon/testdata/file_index/README.md
index dee46286..7dc86b00 100644
--- a/crates/paimon/testdata/file_index/README.md
+++ b/crates/paimon/testdata/file_index/README.md
@@ -1,6 +1,6 @@
# Java FileIndex table fixtures
-These four committed tables were produced through Apache Paimon Java's normal
+These six committed tables were produced through Apache Paimon Java's normal
catalog, append writer, commit, snapshot, and manifest paths. No index bytes or
metadata were assembled or edited by hand.
@@ -9,19 +9,22 @@ Provenance:
- Apache Paimon Java commit: `1d368b4a5932f8221fd28e2555001abdb8fb12ee`
(`2.2-SNAPSHOT`).
- Generator: `java/JavaFileIndexTableFixtureGeneratorTest.java`.
-- Tables: Bitmap and Bloom Filter, each with an embedded (`1 MB` threshold) and
- `.index` sidecar (`1 B` threshold) form.
+- Tables: Bitmap, Bloom Filter, and Range Bitmap, each with an embedded (`1 MB`
+ threshold) and `.index` sidecar (`1 B` threshold) form.
- Common options: `bucket=1`, `bucket-key=id`, `file.format=parquet`, and
`file-index.read.enabled=true`.
- Bloom options: `items=16`, `fpp=0.01`.
-- Rows, in physical order: `(1, "keep")`, `(1, "drop")`,
- `(NULL, "null-id")`, `(3, "three")`.
+- Rows, in physical order: `(1, "keep")`, `(1, "drop")`, and
+ `(NULL, "null-id")`, followed by `(3, "three")` for Bitmap and Bloom Filter
+ or `(9, "nine")` for Range Bitmap.
The Rust test expects `id = 1` to return both duplicate rows, `id IS NULL` to
return the null row, `id = 1 AND payload = 'keep'` to retain only the residual
-match, and `id = 2` to return no rows. The last predicate lies inside the data
-file's `[1, 3]` min/max range, so manifest statistics cannot prune it; it is
also
-the observable FileIndex skip case.
+match, and `id = 2` to return no rows for Bitmap and Bloom Filter. Range Bitmap
+also checks `id BETWEEN 1 AND 3` returns both duplicate rows and `id BETWEEN 3
+AND 7` returns no rows. Each missing-value predicate lies inside its data
+file's min/max range (`[1, 3]` or `[1, 9]`), so manifest statistics cannot
+prune it; these are the observable FileIndex skip cases.
To regenerate into an empty temporary directory:
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/bucket-0/data-791a54a7-4d19-43c3-8752-31cd6073433a-0.parquet
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/bucket-0/data-791a54a7-4d19-43c3-8752-31cd6073433a-0.parquet
new file mode 100644
index 00000000..51874a62
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/bucket-0/data-791a54a7-4d19-43c3-8752-31cd6073433a-0.parquet
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-155640a3-e50d-4be7-ba72-de0afe2a8fec-0
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-155640a3-e50d-4be7-ba72-de0afe2a8fec-0
new file mode 100644
index 00000000..3f15ad7c
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-155640a3-e50d-4be7-ba72-de0afe2a8fec-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-0
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-0
new file mode 100644
index 00000000..a3367dad
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-1
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-1
new file mode 100644
index 00000000..b7653984
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/manifest/manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-1
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/schema/schema-0
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/schema/schema-0
new file mode 100644
index 00000000..dbc24ab0
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/schema/schema-0
@@ -0,0 +1,25 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "payload",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "1",
+ "file-index.range-bitmap.columns" : "id",
+ "bucket-key" : "id",
+ "file-index.read.enabled" : "true",
+ "file.format" : "parquet",
+ "file-index.in-manifest-threshold" : "1 MB"
+ },
+ "timeMillis" : 1790306753711
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/EARLIEST
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/LATEST
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/snapshot-1
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/snapshot-1
new file mode 100644
index 00000000..1164e594
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_embedded/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "eb6639ec-eb6f-405b-8525-3612fabf030f",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-ecab87c6-de6d-44d0-a6ae-3705321d971d-1",
+ "deltaManifestListSize" : 1263,
+ "commitUser" : "aa888b6c-37dc-49f1-82c6-7067442974e3",
+ "writerVersion" :
"java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1790306753723,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 4,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/bucket-0/data-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/bucket-0/data-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet
new file mode 100644
index 00000000..51874a62
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/bucket-0/data-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/bucket-0/data-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet.index
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/bucket-0/data-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet.index
new file mode 100644
index 00000000..846b7156
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/bucket-0/data-9db69c4d-d10c-4f94-aed9-c16cd7fbe1e1-0.parquet.index
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-e1c11d52-6e22-4d10-b62a-833b60216bf2-0
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-e1c11d52-6e22-4d10-b62a-833b60216bf2-0
new file mode 100644
index 00000000..d3ff990c
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-e1c11d52-6e22-4d10-b62a-833b60216bf2-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-0
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-0
new file mode 100644
index 00000000..0abb57be
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-0
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-1
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-1
new file mode 100644
index 00000000..1c0e7842
Binary files /dev/null and
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/manifest/manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-1
differ
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/schema/schema-0
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/schema/schema-0
new file mode 100644
index 00000000..285454bd
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/schema/schema-0
@@ -0,0 +1,25 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "payload",
+ "type" : "STRING"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "1",
+ "file-index.range-bitmap.columns" : "id",
+ "bucket-key" : "id",
+ "file-index.read.enabled" : "true",
+ "file.format" : "parquet",
+ "file-index.in-manifest-threshold" : "1 B"
+ },
+ "timeMillis" : 1790306753724
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/EARLIEST
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/LATEST
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/LATEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/snapshot-1
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/snapshot-1
new file mode 100644
index 00000000..bdb3e6f9
--- /dev/null
+++
b/crates/paimon/testdata/file_index/default.db/range_bitmap_sidecar/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "0780e80a-3837-4b40-876e-29a9579141c0",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-107ca82a-e4a8-4a88-8439-b15d770f0474-1",
+ "deltaManifestListSize" : 1263,
+ "commitUser" : "1a7d1612-bcc4-4552-b69a-8b603f9c2646",
+ "writerVersion" :
"java-2.2-SNAPSHOT-1d368b4a5932f8221fd28e2555001abdb8fb12ee",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1790306753730,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 4,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
b/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
index 6c592f7b..b411d82a 100644
---
a/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
+++
b/crates/paimon/testdata/file_index/java/JavaFileIndexTableFixtureGeneratorTest.java
@@ -64,6 +64,8 @@ public class JavaFileIndexTableFixtureGeneratorTest {
writeTable(catalog, "bitmap_sidecar", "bitmap", "1 B");
writeTable(catalog, "bloom_filter_embedded", "bloom-filter", "1 MB");
writeTable(catalog, "bloom_filter_sidecar", "bloom-filter", "1 B");
+ writeTable(catalog, "range_bitmap_embedded", "range-bitmap", "1 MB");
+ writeTable(catalog, "range_bitmap_sidecar", "range-bitmap", "1 B");
}
private static void writeTable(
@@ -97,7 +99,11 @@ public class JavaFileIndexTableFixtureGeneratorTest {
write.write(row(1, "keep"));
write.write(row(1, "drop"));
write.write(row(null, "null-id"));
- write.write(row(3, "three"));
+ if ("range-bitmap".equals(indexType)) {
+ write.write(row(9, "nine"));
+ } else {
+ write.write(row(3, "three"));
+ }
commit.commit(write.prepareCommit());
}
}
diff --git a/crates/paimon/tests/file_index_java_fixture_test.rs
b/crates/paimon/tests/file_index_java_fixture_test.rs
index 87a86fed..7268dd1e 100644
--- a/crates/paimon/tests/file_index_java_fixture_test.rs
+++ b/crates/paimon/tests/file_index_java_fixture_test.rs
@@ -40,24 +40,39 @@ const FIXTURE: &str = "testdata/file_index/default.db";
struct FixtureCase {
table: &'static str,
embedded: bool,
+ range_bitmap: bool,
}
const CASES: &[FixtureCase] = &[
FixtureCase {
table: "bitmap_embedded",
embedded: true,
+ range_bitmap: false,
},
FixtureCase {
table: "bitmap_sidecar",
embedded: false,
+ range_bitmap: false,
},
FixtureCase {
table: "bloom_filter_embedded",
embedded: true,
+ range_bitmap: false,
},
FixtureCase {
table: "bloom_filter_sidecar",
embedded: false,
+ range_bitmap: false,
+ },
+ FixtureCase {
+ table: "range_bitmap_embedded",
+ embedded: true,
+ range_bitmap: true,
+ },
+ FixtureCase {
+ table: "range_bitmap_sidecar",
+ embedded: false,
+ range_bitmap: true,
},
];
@@ -198,7 +213,13 @@ async fn
reads_java_written_file_indexes_and_prunes_data_files() {
let predicates = PredicateBuilder::new(table.schema().fields());
let id_one = predicates.equal("id", Datum::Int(1)).unwrap();
let null_id = predicates.is_null("id").unwrap();
- let missing_id = predicates.equal("id", Datum::Int(2)).unwrap();
+ let missing = if case.range_bitmap {
+ predicates
+ .between("id", Datum::Int(3), Datum::Int(7))
+ .unwrap()
+ } else {
+ predicates.equal("id", Datum::Int(2)).unwrap()
+ };
let residual = Predicate::and(vec![
id_one.clone(),
predicates
@@ -207,7 +228,7 @@ async fn
reads_java_written_file_indexes_and_prunes_data_files() {
]);
let mut planning_builder = table.new_read_builder();
- planning_builder.with_filter(missing_id.clone());
+ planning_builder.with_filter(missing.clone());
let (plan, trace) = planning_builder
.new_scan()
.plan_with_trace()
@@ -253,15 +274,26 @@ async fn
reads_java_written_file_indexes_and_prunes_data_files() {
"{} must retain row-level residual filtering with FileIndex
enabled={enabled}",
case.table
);
+ if case.range_bitmap {
+ let range_hit = predicates
+ .between("id", Datum::Int(1), Datum::Int(3))
+ .unwrap();
+ assert_eq!(
+ query(&table, enabled, range_hit).await,
+ vec![(Some(1), "drop".to_string()), (Some(1),
"keep".to_string())],
+ "{} range query with FileIndex enabled={enabled}",
+ case.table
+ );
+ }
assert!(
- query(&table, enabled, missing_id.clone()).await.is_empty(),
+ query(&table, enabled, missing.clone()).await.is_empty(),
"{} with FileIndex enabled={enabled}",
case.table
);
}
probe.reset();
- assert!(query(&table, true, missing_id.clone()).await.is_empty());
+ assert!(query(&table, true, missing.clone()).await.is_empty());
assert_eq!(
probe.data_file_opens(),
0,
@@ -270,7 +302,7 @@ async fn
reads_java_written_file_indexes_and_prunes_data_files() {
);
probe.reset();
- assert!(query(&table, false, missing_id).await.is_empty());
+ assert!(query(&table, false, missing).await.is_empty());
assert!(
probe.data_file_opens() > 0,
"{} must open the data file when FileIndex reads are disabled",
