This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch 58_maintenance
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/58_maintenance by this push:
new 51b5eb2d0b [58_maintenance] Write empty validity buffers for arrays
without nulls in IPC writer (#11817)
51b5eb2d0b is described below
commit 51b5eb2d0bf0335078ec155f2e36e92dbf546ee5
Author: Liam Feehery <[email protected]>
AuthorDate: Thu Oct 22 11:13:00 2026 -0400
[58_maintenance] Write empty validity buffers for arrays without nulls in
IPC writer (#11817)
* Backport of #11694
* For inclusion in #11665
Cherry-pick of
[49219f8](https://github.com/apache/arrow-rs/commit/49219f867f77422525627a07b84cd7a87207dcbe)
Co-authored-by: Aleksandar Zecevic <[email protected]>
---
arrow-flight/src/encode.rs | 12 +++++------
arrow-ipc/src/writer.rs | 51 +++++++++++++++++++++++++++++++++++-----------
2 files changed, 45 insertions(+), 18 deletions(-)
diff --git a/arrow-flight/src/encode.rs b/arrow-flight/src/encode.rs
index 191da02413..48d561ada9 100644
--- a/arrow-flight/src/encode.rs
+++ b/arrow-flight/src/encode.rs
@@ -1918,7 +1918,7 @@ mod tests {
])
.unwrap();
- verify_encoded_split(batch, 120).await;
+ verify_encoded_split(batch, 88).await;
}
#[tokio::test]
@@ -1929,7 +1929,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 4312).await;
+ verify_encoded_split(batch, 4304).await;
}
#[tokio::test]
@@ -1965,7 +1965,7 @@ mod tests {
// 5k over limit (which is 2x larger than limit of 5k)
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 5808).await;
+ verify_encoded_split(batch, 5784).await;
}
#[tokio::test]
@@ -1995,7 +1995,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 3336).await;
+ verify_encoded_split(batch, 3328).await;
}
#[tokio::test]
@@ -2009,7 +2009,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 5288).await;
+ verify_encoded_split(batch, 5280).await;
}
#[tokio::test]
@@ -2034,7 +2034,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 4136).await;
+ verify_encoded_split(batch, 4112).await;
}
/// Return size, in memory of flight data
diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs
index 46e2dd7739..0bcb3ca036 100644
--- a/arrow-ipc/src/writer.rs
+++ b/arrow-ipc/src/writer.rs
@@ -36,7 +36,6 @@ use arrow_array::builder::BufferBuilder;
use arrow_array::cast::*;
use arrow_array::types::{Int16Type, Int32Type, Int64Type, RunEndIndexType};
use arrow_array::*;
-use arrow_buffer::bit_util;
use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ToByteSlice};
use arrow_data::{ArrayData, ArrayDataBuilder, BufferSpec, layout};
use arrow_schema::*;
@@ -1825,16 +1824,11 @@ fn write_array_data(
nodes.push(crate::FieldNode::new(num_rows as i64, num_rows as i64));
}
if has_validity_bitmap(array_data.data_type(), write_options) {
- // write null buffer if exists
+ // Fields with null_count == 0 may write a 0-length validity buffer
+ // instead of a materialized bitmap (see FieldNode in Message.fbs)
let null_buffer = match array_data.nulls() {
- None => {
- // create a buffer and fill it with valid bits
- let num_bytes = bit_util::ceil(num_rows, 8);
- let buffer = MutableBuffer::new(num_bytes);
- let buffer = buffer.with_bitset(num_bytes, true);
- buffer.into()
- }
- Some(buffer) => buffer.inner().sliced(),
+ Some(buffer) if null_count > 0 => buffer.inner().sliced(),
+ _ => MutableBuffer::new(0).into(),
};
offset = write_buffer(
@@ -2755,6 +2749,37 @@ mod tests {
assert_eq!(record_batch_slice, deserialized_batch);
}
+ #[test]
+ fn test_write_empty_validity_buffer_without_nulls() {
+ let batch = RecordBatch::try_from_iter([
+ ("a", Arc::new(Int32Array::from(vec![1, 2])) as _),
+ // Keeps a null buffer, but the slice has no nulls
+ (
+ "b",
+ Arc::new(Int32Array::from(vec![None, Some(2),
Some(3)]).slice(1, 2)) as _,
+ ),
+ ("c", Arc::new(Int32Array::from(vec![Some(1), None])) as _),
+ ])
+ .unwrap();
+ assert_eq!(batch.column(1).nulls().map(|n| n.null_count()), Some(0));
+
+ let (_, encoded) = IpcDataGenerator::default()
+ .encode(
+ &batch,
+ &mut DictionaryTracker::new(false),
+ &Default::default(),
+ &mut Default::default(),
+ )
+ .unwrap();
+ let message = crate::root_as_message(&encoded.ipc_message).unwrap();
+ let buffers =
message.header_as_record_batch().unwrap().buffers().unwrap();
+ // Each column writes a validity buffer, then a values buffer
+ let validity_lengths: Vec<_> = buffers.iter().step_by(2).map(|b|
b.length()).collect();
+ assert_eq!(validity_lengths, [0, 0, 1]);
+
+ assert_eq!(deserialize_stream(serialize_stream(&batch)), batch);
+ }
+
#[test]
fn truncate_ipc_dictionary_array() {
fn create_batch() -> RecordBatch {
@@ -3902,13 +3927,15 @@ mod tests {
const IPC_ALIGNMENT: usize = 8;
let num_cols = 2;
- let num_rows = 1;
let mut fields = Vec::new();
let mut arrays = Vec::new();
for i in 0..num_cols {
let field = Field::new(format!("col_{i}"),
DataType::Decimal128(38, 10), true);
- let array = Decimal128Array::from(vec![num_cols as i128;
num_rows]);
+ // The null makes the writer emit a 1-byte validity buffer. With
IPC_ALIGNMENT = 8,
+ // the writer pads that buffer to 8 bytes, so the decimal values
that follow start
+ // at offset 8 of the body, which is not a multiple of 16.
+ let array = Decimal128Array::from(vec![Some(num_cols as i128),
None]);
fields.push(field);
arrays.push(Arc::new(array) as Arc<dyn Array>);
}
