This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new b9bab593 perf(btree): load null bitmap on demand (#1008)
b9bab593 is described below
commit b9bab5934b3f4521ede3f36e2b8a89f2f770c15b
Author: QuakeWang <[email protected]>
AuthorDate: Sat Oct 3 18:07:10 2026 +0800
perf(btree): load null bitmap on demand (#1008)
---
crates/paimon/src/btree/mod.rs | 6 +-
crates/paimon/src/btree/query.rs | 2 +-
crates/paimon/src/btree/reader.rs | 38 ++-
crates/paimon/src/btree/tests.rs | 336 ++++++++++++++++++++-
.../paimon/src/table/global_index_scanner/tests.rs | 119 +++++++-
docs/src/releases.md | 5 +
6 files changed, 481 insertions(+), 25 deletions(-)
diff --git a/crates/paimon/src/btree/mod.rs b/crates/paimon/src/btree/mod.rs
index ddee2d62..9baa8662 100644
--- a/crates/paimon/src/btree/mod.rs
+++ b/crates/paimon/src/btree/mod.rs
@@ -23,13 +23,13 @@
//! | Footer | |
//! +-----------------------------------+ |
//! | Index Block | +--> Loaded on open
-//! +-----------------------------------+ |
-//! | Bloom Filter Block | |
//! +-----------------------------------+------+
+//! | Bloom Filter Block | |
+//! +-----------------------------------+ |
//! | Null Bitmap Block | |
//! +-----------------------------------+ |
//! | Data Block | |
-//! +-----------------------------------+ +--> Loaded on requested
+//! +-----------------------------------+ +--> Loaded on demand
//! | ...... | |
//! +-----------------------------------+ |
//! | Data Block | |
diff --git a/crates/paimon/src/btree/query.rs b/crates/paimon/src/btree/query.rs
index b5166b5a..4d377bc8 100644
--- a/crates/paimon/src/btree/query.rs
+++ b/crates/paimon/src/btree/query.rs
@@ -79,7 +79,7 @@ where
let key_refs: Vec<&[u8]> = keys.iter().map(|k|
k.as_slice()).collect();
self.query_in(&key_refs).await
}
- PredicateOperator::IsNull => Ok(self.null_bitmap().clone()),
+ PredicateOperator::IsNull => Ok(self.null_bitmap().await?.clone()),
PredicateOperator::IsNotNull => self.all_non_null_rows().await,
PredicateOperator::NotEq => {
let key = serialize_datum(&literals[0], data_type);
diff --git a/crates/paimon/src/btree/reader.rs
b/crates/paimon/src/btree/reader.rs
index 2e8f30e9..579190c1 100644
--- a/crates/paimon/src/btree/reader.rs
+++ b/crates/paimon/src/btree/reader.rs
@@ -20,7 +20,7 @@
//! Supports:
//! - Point lookup (equal)
//! - Range queries (less than, greater than, between, etc.)
-//! - Null bitmap reading
+//! - On-demand null bitmap reading
//! - IN / NOT IN queries
use crate::btree::block::{BlockHandle, BlockReader};
@@ -64,11 +64,13 @@ fn try_sort_by<T>(
}
}
-/// BTree index reader with on-demand async data block loading.
+/// BTree index reader with on-demand async block loading.
pub struct BTreeIndexReader<F: Fn(&[u8], &[u8]) -> crate::Result<Ordering>> {
reader: Box<dyn FileRead>,
sst_reader: SstFileReader,
- null_bitmap: RoaringTreemap,
+ null_bitmap_handle: Option<BlockHandle>,
+ // Keep lazy synchronization state from inflating enums that embed this
reader.
+ null_bitmap: Box<OnceCell<RoaringTreemap>>,
min_key: Option<Vec<u8>>,
max_key: Option<Vec<u8>>,
key_comparator: F,
@@ -80,8 +82,8 @@ pub struct BTreeIndexReader<F: Fn(&[u8], &[u8]) ->
crate::Result<Ordering>> {
impl<F: Fn(&[u8], &[u8]) -> crate::Result<Ordering>> BTreeIndexReader<F> {
/// Open a BTree index reader from a FileRead and file metadata.
- /// Only reads footer, index block, and null bitmap on open.
- /// Data blocks are read on demand during queries.
+ /// Only reads the footer and index block on open.
+ /// Data blocks, the Bloom filter, and the null bitmap are read on demand.
pub async fn open(
reader: Box<dyn FileRead>,
file_size: u64,
@@ -136,16 +138,11 @@ impl<F: Fn(&[u8], &[u8]) -> crate::Result<Ordering>>
BTreeIndexReader<F> {
let index_block = read_block_from_bytes(&index_bytes, idx.size)?;
let sst_reader = SstFileReader::from_index_block(index_block);
- // 3. Read null bitmap
- let null_bitmap = match &footer.null_bitmap_handle {
- Some(h) => read_null_bitmap(reader.as_ref(), h).await?,
- None => RoaringTreemap::new(),
- };
-
Ok(Self {
reader,
sst_reader,
- null_bitmap,
+ null_bitmap_handle: footer.null_bitmap_handle,
+ null_bitmap: Box::new(OnceCell::new()),
min_key: meta.first_key.clone(),
max_key: meta.last_key.clone(),
key_comparator,
@@ -156,9 +153,20 @@ impl<F: Fn(&[u8], &[u8]) -> crate::Result<Ordering>>
BTreeIndexReader<F> {
})
}
- /// Get the null bitmap (row ids of null keys).
- pub fn null_bitmap(&self) -> &RoaringTreemap {
- &self.null_bitmap
+ /// Load the null bitmap (row ids of null keys) on first access.
+ /// Concurrent callers share successful initialization within this reader.
+ /// I/O, CRC, and deserialization errors are returned here rather than on
open;
+ /// failed loads are not cached and may be retried. An absent block
returns an
+ /// empty bitmap without reading the file.
+ pub async fn null_bitmap(&self) -> io::Result<&RoaringTreemap> {
+ self.null_bitmap
+ .get_or_try_init(|| async {
+ match &self.null_bitmap_handle {
+ Some(handle) => read_null_bitmap(self.reader.as_ref(),
handle).await,
+ None => Ok(RoaringTreemap::new()),
+ }
+ })
+ .await
}
/// Compare two keys, reporting a stored key that cannot belong to the
column's
diff --git a/crates/paimon/src/btree/tests.rs b/crates/paimon/src/btree/tests.rs
index c9a0f20c..cf43a9cc 100644
--- a/crates/paimon/src/btree/tests.rs
+++ b/crates/paimon/src/btree/tests.rs
@@ -262,6 +262,327 @@ fn read_footer(data: &[u8]) -> BTreeFileFooter {
BTreeFileFooter::read_footer(&data[data.len() -
BTREE_FOOTER_ENCODED_LENGTH..]).unwrap()
}
+#[tokio::test]
+async fn test_null_bitmap_is_loaded_on_demand() {
+ let data_type = DataType::Int(crate::spec::IntType::new());
+ let key = |value| crate::btree::serialize_datum(&Datum::Int(value),
&data_type);
+
+ for version in [1, 2] {
+ for (has_nulls, has_non_nulls) in [(true, true), (false, true), (true,
false)] {
+ let buf = VecFileWrite::new();
+ let mut writer = BTreeIndexWriter::with_comparator(
+ Box::new(buf.clone()),
+ 256,
+ BlockCompressionType::None,
+ crate::btree::make_key_comparator(&data_type),
+ )
+ .with_file_version(version)
+ .unwrap();
+ if has_nulls {
+ writer.write(None, 10).await.unwrap();
+ writer.write(None, 30).await.unwrap();
+ }
+ if has_non_nulls {
+ writer.write(Some(&key(1)), 20).await.unwrap();
+ writer.write(Some(&key(2)), 40).await.unwrap();
+ }
+ let result = writer.finish().await.unwrap();
+ let data = Bytes::from(buf.to_vec());
+ let footer = read_footer(&data);
+ let null_range = footer
+ .null_bitmap_handle
+ .map(|handle| handle.offset..handle.offset +
u64::from(handle.size) + 4);
+ assert_eq!(null_range.is_some(), has_nulls);
+ let ranges = Arc::new(Mutex::new(Vec::new()));
+ let reader = BTreeIndexReader::open(
+ Box::new(RecordingFileRead {
+ data: data.clone(),
+ ranges: ranges.clone(),
+ }),
+ data.len() as u64,
+ &result.meta,
+ crate::btree::make_key_comparator(&data_type),
+ )
+ .await
+ .unwrap();
+
+ let index = footer.index_block_handle;
+ let expected_open_ranges = vec![
+ data.len() as u64 - BTREE_FOOTER_ENCODED_LENGTH as
u64..data.len() as u64,
+ index.offset..index.offset + index.full_block_size() as u64,
+ ];
+ assert_eq!(*ranges.lock().unwrap(), expected_open_ranges);
+ let null_reads = || {
+ ranges
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|range| null_range.as_ref() == Some(*range))
+ .count()
+ };
+ assert_eq!(null_reads(), 0);
+
+ let expected_equal = if has_non_nulls { vec![20] } else { vec![] };
+ let expected_non_null = if has_non_nulls { vec![20, 40] } else {
vec![] };
+ for (op, literals, expected) in [
+ (PredicateOperator::Eq, vec![Datum::Int(1)], expected_equal),
+ (
+ PredicateOperator::Between,
+ vec![Datum::Int(1), Datum::Int(2)],
+ expected_non_null.clone(),
+ ),
+ (PredicateOperator::IsNotNull, vec![], expected_non_null),
+ ] {
+ let rows = reader.query(op, &literals,
&data_type).await.unwrap();
+ assert_eq!(rows.iter().collect::<Vec<_>>(), expected,
"{op:?}");
+ assert_eq!(null_reads(), 0, "{op:?}");
+ }
+
+ let mut expected_reads = ranges.lock().unwrap().clone();
+ let expected_null = if has_nulls { vec![10, 30] } else { vec![] };
+ let rows = reader
+ .query(PredicateOperator::IsNull, &[], &data_type)
+ .await
+ .unwrap();
+ assert_eq!(rows.iter().collect::<Vec<_>>(), expected_null);
+ if let Some(range) = &null_range {
+ expected_reads.push(range.clone());
+ }
+ assert_eq!(*ranges.lock().unwrap(), expected_reads);
+ assert_eq!(null_reads(), usize::from(has_nulls));
+ for _ in 0..2 {
+ let rows = reader
+ .query(PredicateOperator::IsNull, &[], &data_type)
+ .await
+ .unwrap();
+ assert_eq!(rows.iter().collect::<Vec<_>>(), expected_null);
+ }
+ let (left, right) = tokio::join!(
+ reader.query(PredicateOperator::IsNull, &[], &data_type),
+ reader.query(PredicateOperator::IsNull, &[], &data_type),
+ );
+ assert_eq!(left.unwrap().iter().collect::<Vec<_>>(),
expected_null);
+ assert_eq!(right.unwrap().iter().collect::<Vec<_>>(),
expected_null);
+ assert_eq!(*ranges.lock().unwrap(), expected_reads);
+ }
+ }
+}
+
+#[tokio::test]
+async fn test_null_bitmap_corruption_is_reported_on_access() {
+ for version in [1, 2] {
+ for corrupt_crc in [true, false] {
+ let buf = VecFileWrite::new();
+ let mut writer =
+ BTreeIndexWriter::new(Box::new(buf.clone()), 256,
BlockCompressionType::None)
+ .with_file_version(version)
+ .unwrap();
+ writer.write(None, 10).await.unwrap();
+ writer.write(Some(&int_key(1)), 20).await.unwrap();
+ let result = writer.finish().await.unwrap();
+ let mut data = buf.to_vec();
+ let footer = read_footer(&data);
+ let handle = footer.null_bitmap_handle.unwrap();
+ let bitmap_start = handle.offset as usize;
+ let crc_start = bitmap_start + handle.size as usize;
+ if corrupt_crc {
+ data[crc_start] ^= 1;
+ } else {
+ // The first bitmap cookie follows the u64 map count and u32
key.
+ let cookie_start = bitmap_start + 12;
+ data[cookie_start..cookie_start + 4].fill(0);
+ let crc = crc32fast::hash(&data[bitmap_start..crc_start]);
+ data[crc_start..crc_start +
4].copy_from_slice(&crc.to_le_bytes());
+ assert!(
+
roaring::RoaringTreemap::deserialize_from(&data[bitmap_start..crc_start])
+ .is_err()
+ );
+ }
+ let data = Bytes::from(data);
+ let ranges = Arc::new(Mutex::new(Vec::new()));
+ let reader = BTreeIndexReader::open(
+ Box::new(RecordingFileRead {
+ data: data.clone(),
+ ranges: ranges.clone(),
+ }),
+ data.len() as u64,
+ &result.meta,
+ int_cmp,
+ )
+ .await
+ .expect("null bitmap corruption must not prevent opening the
reader");
+ assert_eq!(ranges.lock().unwrap().len(), 2);
+ assert_eq!(
+ reader
+ .query_equal(&int_key(1))
+ .await
+ .unwrap()
+ .iter()
+ .collect::<Vec<_>>(),
+ vec![20],
+ );
+ let null_range = handle.offset..handle.offset +
u64::from(handle.size) + 4;
+ assert!(!ranges.lock().unwrap().contains(&null_range));
+
+ let data_type = DataType::Int(crate::spec::IntType::new());
+ for _ in 0..2 {
+ let error = reader
+ .query(PredicateOperator::IsNull, &[], &data_type)
+ .await
+ .expect_err("null bitmap corruption must not produce an
empty result");
+ assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
+ if corrupt_crc {
+ assert!(error.to_string().contains("Null bitmap CRC
mismatch"));
+ } else {
+ assert!(error.to_string().contains("unknown cookie
value"));
+ }
+ }
+ assert_eq!(
+ ranges
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|range| **range == null_range)
+ .count(),
+ 2,
+ "failed initialization must not be cached",
+ );
+ }
+ }
+}
+
+#[tokio::test]
+async fn test_null_bitmap_read_failure_can_be_retried() {
+ struct FailingNullBitmapRead {
+ inner: RecordingFileRead,
+ null_range: Range<u64>,
+ null_reads: AtomicUsize,
+ }
+
+ #[async_trait::async_trait]
+ impl FileRead for FailingNullBitmapRead {
+ async fn read(&self, range: Range<u64>) -> crate::Result<Bytes> {
+ if range == self.null_range && self.null_reads.fetch_add(1,
AtomicOrdering::SeqCst) == 0
+ {
+ self.inner.ranges.lock().unwrap().push(range);
+ return Err(crate::Error::UnexpectedError {
+ message: "Injected null bitmap read failure".to_string(),
+ source: None,
+ });
+ }
+ self.inner.read(range).await
+ }
+ }
+
+ let buf = VecFileWrite::new();
+ let mut writer = BTreeIndexWriter::new(Box::new(buf.clone()), 256,
BlockCompressionType::None);
+ writer.write(None, 10).await.unwrap();
+ writer.write(Some(&int_key(1)), 20).await.unwrap();
+ let result = writer.finish().await.unwrap();
+ let data = Bytes::from(buf.to_vec());
+ let handle = read_footer(&data).null_bitmap_handle.unwrap();
+ let null_range = handle.offset..handle.offset + u64::from(handle.size) + 4;
+ let ranges = Arc::new(Mutex::new(Vec::new()));
+ let reader = BTreeIndexReader::open(
+ Box::new(FailingNullBitmapRead {
+ inner: RecordingFileRead {
+ data: data.clone(),
+ ranges: ranges.clone(),
+ },
+ null_range: null_range.clone(),
+ null_reads: AtomicUsize::new(0),
+ }),
+ data.len() as u64,
+ &result.meta,
+ int_cmp,
+ )
+ .await
+ .expect("opening must not read the null bitmap");
+ assert_eq!(ranges.lock().unwrap().len(), 2);
+ let data_type = DataType::Int(crate::spec::IntType::new());
+ let error = reader
+ .query(PredicateOperator::IsNull, &[], &data_type)
+ .await
+ .expect_err("the first null bitmap load must return the injected I/O
error");
+ assert_eq!(error.kind(), std::io::ErrorKind::Other);
+ assert!(error
+ .to_string()
+ .contains("Injected null bitmap read failure"));
+ assert_eq!(ranges.lock().unwrap().len(), 3);
+ assert_eq!(ranges.lock().unwrap().last(), Some(&null_range));
+ assert_eq!(
+ reader
+ .query_equal(&int_key(1))
+ .await
+ .unwrap()
+ .iter()
+ .collect::<Vec<_>>(),
+ vec![20],
+ );
+ let rows = reader
+ .query(PredicateOperator::IsNull, &[], &data_type)
+ .await
+ .expect("a failed load must allow a successful retry");
+ assert_eq!(rows.iter().collect::<Vec<_>>(), vec![10]);
+ assert_eq!(
+ ranges
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|range| **range == null_range)
+ .count(),
+ 2,
+ );
+ let reads_after_success = ranges.lock().unwrap().clone();
+ assert_eq!(reader.null_bitmap().await.unwrap(), &rows);
+ assert_eq!(*ranges.lock().unwrap(), reads_after_success);
+}
+
+#[tokio::test]
+async fn test_null_bitmap_concurrent_first_queries_share_one_read() {
+ struct YieldingFileRead(RecordingFileRead);
+
+ #[async_trait::async_trait]
+ impl FileRead for YieldingFileRead {
+ async fn read(&self, range: Range<u64>) -> crate::Result<Bytes> {
+ let bytes = self.0.read(range).await?;
+ tokio::task::yield_now().await;
+ Ok(bytes)
+ }
+ }
+
+ let buf = VecFileWrite::new();
+ let mut writer = BTreeIndexWriter::new(Box::new(buf.clone()), 256,
BlockCompressionType::None);
+ writer.write(None, 10).await.unwrap();
+ writer.write(None, 30).await.unwrap();
+ let result = writer.finish().await.unwrap();
+ let data = Bytes::from(buf.to_vec());
+ let handle = read_footer(&data).null_bitmap_handle.unwrap();
+ let null_range = handle.offset..handle.offset + u64::from(handle.size) + 4;
+ let ranges = Arc::new(Mutex::new(Vec::new()));
+ let reader = BTreeIndexReader::open(
+ Box::new(YieldingFileRead(RecordingFileRead {
+ data: data.clone(),
+ ranges: ranges.clone(),
+ })),
+ data.len() as u64,
+ &result.meta,
+ int_cmp,
+ )
+ .await
+ .unwrap();
+ assert_eq!(ranges.lock().unwrap().len(), 2);
+ let data_type = DataType::Int(crate::spec::IntType::new());
+ let (left, right) = tokio::join!(
+ reader.query(PredicateOperator::IsNull, &[], &data_type),
+ reader.query(PredicateOperator::IsNull, &[], &data_type),
+ );
+ assert_eq!(left.unwrap().iter().collect::<Vec<_>>(), vec![10, 30]);
+ assert_eq!(right.unwrap().iter().collect::<Vec<_>>(), vec![10, 30]);
+ assert_eq!(ranges.lock().unwrap().len(), 3);
+ assert_eq!(ranges.lock().unwrap().last(), Some(&null_range));
+}
+
#[tokio::test]
async fn test_java_v2_posting_encodings() {
// Existing low-level tests use arbitrary big-endian keys. Java's INT key
@@ -295,7 +616,12 @@ async fn test_java_v2_posting_encodings() {
);
}
assert_eq!(
- reader.null_bitmap().iter().collect::<Vec<_>>(),
+ reader
+ .null_bitmap()
+ .await
+ .unwrap()
+ .iter()
+ .collect::<Vec<_>>(),
vec![4, (1u64 << 32) + 20000]
);
assert_eq!(
@@ -581,7 +907,7 @@ async fn test_null_keys() {
let reader = write_and_open(&buf, &result, int_cmp).await;
- let null_bm = reader.null_bitmap();
+ let null_bm = reader.null_bitmap().await.unwrap();
assert!(null_bm.contains(10));
assert!(null_bm.contains(30));
assert!(!null_bm.contains(20));
@@ -605,7 +931,7 @@ async fn test_only_nulls() {
assert!(result.meta.has_nulls);
let reader = write_and_open(&buf, &result, int_cmp).await;
- let null_bm = reader.null_bitmap();
+ let null_bm = reader.null_bitmap().await.unwrap();
assert_eq!(null_bm.len(), 3);
}
@@ -1303,7 +1629,7 @@ async fn test_nulls_with_range_query() {
assert!(!all.contains(0));
assert!(!all.contains(100));
- let nulls = reader.null_bitmap();
+ let nulls = reader.null_bitmap().await.unwrap();
assert_eq!(nulls.len(), 2);
assert!(nulls.contains(0));
assert!(nulls.contains(100));
@@ -1432,7 +1758,7 @@ async fn test_java_compat_int_with_nulls() {
let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)),
true);
let reader = open_testdata("btree_int_100_with_nulls.bin", &meta,
le_int_cmp).await;
- let null_bm = reader.null_bitmap();
+ let null_bm = reader.null_bitmap().await.unwrap();
assert!(!null_bm.is_empty());
let all = reader.all_non_null_rows().await.unwrap();
diff --git a/crates/paimon/src/table/global_index_scanner/tests.rs
b/crates/paimon/src/table/global_index_scanner/tests.rs
index 84ebbacd..2870bb1b 100644
--- a/crates/paimon/src/table/global_index_scanner/tests.rs
+++ b/crates/paimon/src/table/global_index_scanner/tests.rs
@@ -21,7 +21,9 @@ use super::predicates::*;
use super::row_ranges::*;
use super::*;
use crate::btree::test_util::VecFileWrite;
-use crate::btree::{serialize_datum, BTreeIndexWriter, BlockCompressionType};
+use crate::btree::{
+ make_key_comparator, serialize_datum, BTreeFileFooter, BTreeIndexWriter,
BlockCompressionType,
+};
use crate::fm_index::{FMGlobalIndexWriter, FMWriteOptions};
use crate::spec::{DataType, Datum, IndexFileMeta, PredicateOperator};
use crate::table::bitmap_global_index_format::{
@@ -1489,6 +1491,121 @@ async fn test_evaluate_global_index_eq() {
assert_eq!(ranges, vec![RowRange::new(25, 25)]);
}
+#[tokio::test]
+async fn test_evaluate_btree_null_bitmap_and_deferred_crc_error() {
+ let data_type = DataType::Int(crate::spec::IntType::new());
+ for version in [1, 2] {
+ for corrupt_crc in [false, true] {
+ let buf = VecFileWrite::new();
+ let mut writer = BTreeIndexWriter::with_comparator(
+ Box::new(buf.clone()),
+ 256,
+ BlockCompressionType::None,
+ make_key_comparator(&data_type),
+ )
+ .with_file_version(version)
+ .unwrap();
+ writer.write(None, 0).await.unwrap();
+ writer.write(Some(&le_int_key(1)), 1).await.unwrap();
+ writer.write(Some(&le_int_key(2)), 2).await.unwrap();
+ writer.write(None, 3).await.unwrap();
+ let result = writer.finish().await.unwrap();
+ let mut data = buf.to_vec();
+ if corrupt_crc {
+ let footer = BTreeFileFooter::read_footer(
+ &data[data.len() - BTreeFileFooter::ENCODED_LENGTH..],
+ )
+ .unwrap();
+ let handle = footer.null_bitmap_handle.unwrap();
+ data[handle.offset as usize + handle.size as usize] ^= 1;
+ }
+ let file_io =
crate::io::FileIOBuilder::new("memory").build().unwrap();
+ let table_path = "memory:/table";
+ let file_name = "nullable.btree";
+ let mut entry = make_global_index_entry(file_name, 1, 100, 103,
&result.meta);
+ entry.index_file.file_size = data.len() as i64;
+ entry.index_file.row_count = result.row_count as i64;
+ file_io
+ .new_output(&format!("{table_path}/index/{file_name}"))
+ .unwrap()
+ .write(bytes::Bytes::from(data))
+ .await
+ .unwrap();
+ let entries = [entry];
+ let fields = int_schema_fields();
+
+ for (op, literals, expected) in [
+ (
+ PredicateOperator::Eq,
+ vec![Datum::Int(1)],
+ vec![RowRange::new(101, 101)],
+ ),
+ (
+ PredicateOperator::Between,
+ vec![Datum::Int(1), Datum::Int(2)],
+ vec![RowRange::new(101, 102)],
+ ),
+ (
+ PredicateOperator::IsNotNull,
+ vec![],
+ vec![RowRange::new(101, 102)],
+ ),
+ ] {
+ let predicate = Predicate::Leaf {
+ column: "id".to_string(),
+ index: 0,
+ data_type: data_type.clone(),
+ op,
+ literals,
+ };
+ let rows = evaluate_global_index_fast(
+ &file_io,
+ table_path,
+ &entries,
+ &[predicate],
+ &fields,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ rows,
+ Some(expected),
+ "{op:?}, V{version}, CRC={corrupt_crc}"
+ );
+ }
+
+ let predicate = Predicate::Leaf {
+ column: "id".to_string(),
+ index: 0,
+ data_type: data_type.clone(),
+ op: PredicateOperator::IsNull,
+ literals: vec![],
+ };
+ let rows =
+ evaluate_global_index_fast(&file_io, table_path, &entries,
&[predicate], &fields)
+ .await;
+ if corrupt_crc {
+ let error = rows.expect_err("a corrupt null bitmap must fail
the IS NULL scan");
+ let Error::DataInvalid {
+ message,
+ source: Some(source),
+ } = error
+ else {
+ panic!("expected a global index query error with its I/O
source");
+ };
+ assert!(message.contains("Global index query failed"));
+ assert!(message.contains(file_name));
+ assert!(source.to_string().contains("Null bitmap CRC
mismatch"));
+ } else {
+ assert_eq!(
+ rows.unwrap(),
+ Some(vec![RowRange::new(100, 100), RowRange::new(103,
103)]),
+ );
+ }
+ }
+ }
+}
+
#[tokio::test]
async fn test_evaluate_global_index_uses_known_file_size() {
let (file_io, table_path, file_name, _tmp) =
diff --git a/docs/src/releases.md b/docs/src/releases.md
index 355ac14c..4202ddf9 100644
--- a/docs/src/releases.md
+++ b/docs/src/releases.md
@@ -42,6 +42,11 @@ Key features:
- Standalone `BlobDescriptor` batch reads and seekable streams, with Go batch
and stream APIs and `MAP<STRING, BLOB>` descriptor reads
- Extensible and cached I/O through caller-provided OpenDAL operators, an
in-memory local block cache, and C cache callbacks
+API compatibility changes:
+
+- `BTreeIndexReader::null_bitmap()` is now asynchronous and returns
`io::Result<&RoaringTreemap>`. Downstream Rust callers must replace
`reader.null_bitmap()` with `reader.null_bitmap().await?` and handle load
errors.
+- BTree readers load the null bitmap on first access and reuse successful
loads within the reader. Null bitmap I/O, CRC, and deserialization errors now
surface on first access rather than when opening the reader. Failed loads can
be retried.
+
## Past Releases
### [0.3.0](https://github.com/apache/paimon-rust/releases/tag/v0.3.0)
