This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11138-258f331c94ae360ffa91d84341ca211ccd9f8449 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit a2c689866fd24b91095814d0653368ffd5d63159 Author: RanaPriyansh <[email protected]> AuthorDate: Sun Oct 25 23:03:34 2026 +0000 bench(ipc): cover LZ4 reader and writer buffer layouts (#11138) # Which issue does this PR close? Related to #11129. These benchmarks do not resolve the decoder initialization question. # Rationale for this change The IPC reader and stream-writer benchmarks lacked equivalent mixed-schema LZ4 coverage. # What changes are included in this PR? Add one mixed-schema LZ4 case for `StreamReader` and one for `StreamWriter`, matching the existing ZSTD benchmark patterns. The shared `create_batch` fixture now includes a non-null fixed-size binary field. Each case reads or writes ten batches. Reader validation remains enabled; writer iterations finish the stream and reuse the output buffer. The benchmark setup uses the existing fixture and timed-loop structure. It does not add correctness checks or compression-layout assertions to benchmark setup. The fixed-size binary field is representative fixture coverage, not a separate layout experiment. # Are these changes tested? The mixed LZ4 and ZSTD reader and writer cases passed Criterion functional mode. The corresponding uncompressed controls passed with no default features. Stable formatting, targeted reader and writer Clippy, and `arrow-ipc` all-targets/all-features Clippy passed with Rust 1.98.1 on aarch64 macOS. Full workspace tests and Clippy were not run. No performance improvement is claimed. # Are there any user-facing changes? No library or API changes. AI assistance generated the benchmark changes, executed checks, and assisted review. --------- Co-authored-by: RIchard Baah <[email protected]> --- arrow-ipc/benches/ipc_reader.rs | 30 ++++++++++++++++++++++++++++-- arrow-ipc/benches/ipc_writer.rs | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/arrow-ipc/benches/ipc_reader.rs b/arrow-ipc/benches/ipc_reader.rs index 5d7c373ee5..c73e3d5dee 100644 --- a/arrow-ipc/benches/ipc_reader.rs +++ b/arrow-ipc/benches/ipc_reader.rs @@ -16,7 +16,7 @@ // under the License. use arrow_array::builder::{Date32Builder, Decimal128Builder, Int32Builder}; -use arrow_array::{RecordBatch, builder::StringBuilder}; +use arrow_array::{FixedSizeBinaryArray, RecordBatch, builder::StringBuilder}; use arrow_buffer::Buffer; use arrow_ipc::convert::try_fb_to_schema; use arrow_ipc::reader::{FileDecoder, FileReader, StreamReader, read_footer_length}; @@ -75,6 +75,22 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + group.bench_function("StreamReader/read_10/lz4", |b| { + let buffer = ipc_stream( + IpcWriteOptions::default() + .try_with_compression(Some(CompressionType::LZ4_FRAME)) + .unwrap(), + ); + b.iter(move || { + let projection = None; + let mut reader = StreamReader::try_new(buffer.as_slice(), projection).unwrap(); + for _ in 0..10 { + reader.next().unwrap().unwrap(); + } + assert!(reader.next().is_none()); + }) + }); + group.bench_function("StreamReader/no_validation/read_10/zstd", |b| { let buffer = ipc_stream( IpcWriteOptions::default() @@ -267,6 +283,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch { Field::new("c1", DataType::Utf8, true), Field::new("c2", DataType::Date32, true), Field::new("c3", DataType::Decimal128(11, 2), true), + Field::new("c4", DataType::FixedSizeBinary(16), false), ])); let mut a = Int32Builder::new(); let mut b = StringBuilder::new(); @@ -274,6 +291,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch { let mut d = Decimal128Builder::new() .with_precision_and_scale(11, 2) .unwrap(); + let fixed_value = [0xa5; 16]; for i in 0..num_rows { a.append_value(i as i32); c.append_value(i as i32); @@ -288,9 +306,17 @@ fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch { let b = b.finish(); let c = c.finish(); let d = d.finish(); + let e = + FixedSizeBinaryArray::try_from_iter((0..num_rows).map(|_| fixed_value.as_slice())).unwrap(); RecordBatch::try_new( schema.clone(), - vec![Arc::new(a), Arc::new(b), Arc::new(c), Arc::new(d)], + vec![ + Arc::new(a), + Arc::new(b), + Arc::new(c), + Arc::new(d), + Arc::new(e), + ], ) .unwrap() } diff --git a/arrow-ipc/benches/ipc_writer.rs b/arrow-ipc/benches/ipc_writer.rs index 2882bf66b7..9beaaa6ea6 100644 --- a/arrow-ipc/benches/ipc_writer.rs +++ b/arrow-ipc/benches/ipc_writer.rs @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -use arrow_array::RecordBatch; use arrow_array::builder::{ Date32Builder, Decimal128Builder, Int32Builder, StringBuilder, StringDictionaryBuilder, }; use arrow_array::types::UInt32Type; +use arrow_array::{FixedSizeBinaryArray, RecordBatch}; use arrow_ipc::CompressionType; use arrow_ipc::writer::{ DictionaryHandling, FileWriter, IpcWriteOptions, StreamEncoder, StreamWriter, @@ -63,6 +63,24 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + group.bench_function("StreamWriter/write_10/lz4", |b| { + let batch = create_batch(8192, true); + let mut buffer = Vec::with_capacity(2 * 1024 * 1024); + b.iter(move || { + buffer.clear(); + let options = IpcWriteOptions::default() + .try_with_compression(Some(CompressionType::LZ4_FRAME)) + .unwrap(); + let mut writer = + StreamWriter::try_new_with_options(&mut buffer, batch.schema().as_ref(), options) + .unwrap(); + for _ in 0..10 { + writer.write(&batch).unwrap(); + } + writer.finish().unwrap(); + }) + }); + group.bench_function("StreamEncoder/encode_10", |b| { let batch = create_batch(8192, true); b.iter(move || { @@ -250,6 +268,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch { Field::new("c1", DataType::Utf8, true), Field::new("c2", DataType::Date32, true), Field::new("c3", DataType::Decimal128(11, 2), true), + Field::new("c4", DataType::FixedSizeBinary(16), false), ])); let mut a = Int32Builder::new(); let mut b = StringBuilder::new(); @@ -257,6 +276,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch { let mut d = Decimal128Builder::new() .with_precision_and_scale(11, 2) .unwrap(); + let fixed_value = [0xa5; 16]; for i in 0..num_rows { a.append_value(i as i32); c.append_value(i as i32); @@ -271,9 +291,17 @@ fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch { let b = b.finish(); let c = c.finish(); let d = d.finish(); + let e = + FixedSizeBinaryArray::try_from_iter((0..num_rows).map(|_| fixed_value.as_slice())).unwrap(); RecordBatch::try_new( schema.clone(), - vec![Arc::new(a), Arc::new(b), Arc::new(c), Arc::new(d)], + vec![ + Arc::new(a), + Arc::new(b), + Arc::new(c), + Arc::new(d), + Arc::new(e), + ], ) .unwrap() }
