This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new a2c689866f bench(ipc): cover LZ4 reader and writer buffer layouts
(#11138)
a2c689866f is described below
commit a2c689866fd24b91095814d0653368ffd5d63159
Author: RanaPriyansh <[email protected]>
AuthorDate: Sun Oct 25 23:03:34 2026 +0000
bench(ipc): cover LZ4 reader and writer buffer layouts (#11138)
# Which issue does this PR close?
Related to #11129. These benchmarks do not resolve the decoder
initialization question.
# Rationale for this change
The IPC reader and stream-writer benchmarks lacked equivalent
mixed-schema LZ4 coverage.
# What changes are included in this PR?
Add one mixed-schema LZ4 case for `StreamReader` and one for
`StreamWriter`, matching the existing ZSTD benchmark patterns. The
shared `create_batch` fixture now includes a non-null fixed-size binary
field. Each case reads or writes ten batches. Reader validation remains
enabled; writer iterations finish the stream and reuse the output
buffer.
The benchmark setup uses the existing fixture and timed-loop structure.
It does not add correctness checks or compression-layout assertions to
benchmark setup. The fixed-size binary field is representative fixture
coverage, not a separate layout experiment.
# Are these changes tested?
The mixed LZ4 and ZSTD reader and writer cases passed Criterion
functional mode. The corresponding uncompressed controls passed with no
default features. Stable formatting, targeted reader and writer Clippy,
and `arrow-ipc` all-targets/all-features Clippy passed with Rust 1.98.1
on aarch64 macOS. Full workspace tests and Clippy were not run. No
performance improvement is claimed.
# Are there any user-facing changes?
No library or API changes.
AI assistance generated the benchmark changes, executed checks, and
assisted review.
---------
Co-authored-by: RIchard Baah <[email protected]>
---
arrow-ipc/benches/ipc_reader.rs | 30 ++++++++++++++++++++++++++++--
arrow-ipc/benches/ipc_writer.rs | 32 ++++++++++++++++++++++++++++++--
2 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/arrow-ipc/benches/ipc_reader.rs b/arrow-ipc/benches/ipc_reader.rs
index 5d7c373ee5..c73e3d5dee 100644
--- a/arrow-ipc/benches/ipc_reader.rs
+++ b/arrow-ipc/benches/ipc_reader.rs
@@ -16,7 +16,7 @@
// under the License.
use arrow_array::builder::{Date32Builder, Decimal128Builder, Int32Builder};
-use arrow_array::{RecordBatch, builder::StringBuilder};
+use arrow_array::{FixedSizeBinaryArray, RecordBatch, builder::StringBuilder};
use arrow_buffer::Buffer;
use arrow_ipc::convert::try_fb_to_schema;
use arrow_ipc::reader::{FileDecoder, FileReader, StreamReader,
read_footer_length};
@@ -75,6 +75,22 @@ fn criterion_benchmark(c: &mut Criterion) {
})
});
+ group.bench_function("StreamReader/read_10/lz4", |b| {
+ let buffer = ipc_stream(
+ IpcWriteOptions::default()
+ .try_with_compression(Some(CompressionType::LZ4_FRAME))
+ .unwrap(),
+ );
+ b.iter(move || {
+ let projection = None;
+ let mut reader = StreamReader::try_new(buffer.as_slice(),
projection).unwrap();
+ for _ in 0..10 {
+ reader.next().unwrap().unwrap();
+ }
+ assert!(reader.next().is_none());
+ })
+ });
+
group.bench_function("StreamReader/no_validation/read_10/zstd", |b| {
let buffer = ipc_stream(
IpcWriteOptions::default()
@@ -267,6 +283,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) ->
RecordBatch {
Field::new("c1", DataType::Utf8, true),
Field::new("c2", DataType::Date32, true),
Field::new("c3", DataType::Decimal128(11, 2), true),
+ Field::new("c4", DataType::FixedSizeBinary(16), false),
]));
let mut a = Int32Builder::new();
let mut b = StringBuilder::new();
@@ -274,6 +291,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) ->
RecordBatch {
let mut d = Decimal128Builder::new()
.with_precision_and_scale(11, 2)
.unwrap();
+ let fixed_value = [0xa5; 16];
for i in 0..num_rows {
a.append_value(i as i32);
c.append_value(i as i32);
@@ -288,9 +306,17 @@ fn create_batch(num_rows: usize, allow_nulls: bool) ->
RecordBatch {
let b = b.finish();
let c = c.finish();
let d = d.finish();
+ let e =
+ FixedSizeBinaryArray::try_from_iter((0..num_rows).map(|_|
fixed_value.as_slice())).unwrap();
RecordBatch::try_new(
schema.clone(),
- vec![Arc::new(a), Arc::new(b), Arc::new(c), Arc::new(d)],
+ vec![
+ Arc::new(a),
+ Arc::new(b),
+ Arc::new(c),
+ Arc::new(d),
+ Arc::new(e),
+ ],
)
.unwrap()
}
diff --git a/arrow-ipc/benches/ipc_writer.rs b/arrow-ipc/benches/ipc_writer.rs
index 2882bf66b7..9beaaa6ea6 100644
--- a/arrow-ipc/benches/ipc_writer.rs
+++ b/arrow-ipc/benches/ipc_writer.rs
@@ -15,11 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-use arrow_array::RecordBatch;
use arrow_array::builder::{
Date32Builder, Decimal128Builder, Int32Builder, StringBuilder,
StringDictionaryBuilder,
};
use arrow_array::types::UInt32Type;
+use arrow_array::{FixedSizeBinaryArray, RecordBatch};
use arrow_ipc::CompressionType;
use arrow_ipc::writer::{
DictionaryHandling, FileWriter, IpcWriteOptions, StreamEncoder,
StreamWriter,
@@ -63,6 +63,24 @@ fn criterion_benchmark(c: &mut Criterion) {
})
});
+ group.bench_function("StreamWriter/write_10/lz4", |b| {
+ let batch = create_batch(8192, true);
+ let mut buffer = Vec::with_capacity(2 * 1024 * 1024);
+ b.iter(move || {
+ buffer.clear();
+ let options = IpcWriteOptions::default()
+ .try_with_compression(Some(CompressionType::LZ4_FRAME))
+ .unwrap();
+ let mut writer =
+ StreamWriter::try_new_with_options(&mut buffer,
batch.schema().as_ref(), options)
+ .unwrap();
+ for _ in 0..10 {
+ writer.write(&batch).unwrap();
+ }
+ writer.finish().unwrap();
+ })
+ });
+
group.bench_function("StreamEncoder/encode_10", |b| {
let batch = create_batch(8192, true);
b.iter(move || {
@@ -250,6 +268,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) ->
RecordBatch {
Field::new("c1", DataType::Utf8, true),
Field::new("c2", DataType::Date32, true),
Field::new("c3", DataType::Decimal128(11, 2), true),
+ Field::new("c4", DataType::FixedSizeBinary(16), false),
]));
let mut a = Int32Builder::new();
let mut b = StringBuilder::new();
@@ -257,6 +276,7 @@ fn create_batch(num_rows: usize, allow_nulls: bool) ->
RecordBatch {
let mut d = Decimal128Builder::new()
.with_precision_and_scale(11, 2)
.unwrap();
+ let fixed_value = [0xa5; 16];
for i in 0..num_rows {
a.append_value(i as i32);
c.append_value(i as i32);
@@ -271,9 +291,17 @@ fn create_batch(num_rows: usize, allow_nulls: bool) ->
RecordBatch {
let b = b.finish();
let c = c.finish();
let d = d.finish();
+ let e =
+ FixedSizeBinaryArray::try_from_iter((0..num_rows).map(|_|
fixed_value.as_slice())).unwrap();
RecordBatch::try_new(
schema.clone(),
- vec![Arc::new(a), Arc::new(b), Arc::new(c), Arc::new(d)],
+ vec![
+ Arc::new(a),
+ Arc::new(b),
+ Arc::new(c),
+ Arc::new(d),
+ Arc::new(e),
+ ],
)
.unwrap()
}
