This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 0be3a5f636 bench(parquet): add wide-schema writer benchmark with
repeated batches (#10878)
0be3a5f636 is described below
commit 0be3a5f6363fd2a473d9fda5bd7f8303fe7f08d9
Author: Adrian Garcia Badaracco <[email protected]>
AuthorDate: Thu Aug 27 10:24:27 2026 -0500
bench(parquet): add wide-schema writer benchmark with repeated batches
(#10878)
# Which issue does this PR close?
- Contributes to #9722.
# Rationale for this change
`parquet/benches/writer_overhead.rs` writes a single one-row batch per
file. That
measures column writer construction, allocation and metadata assembly,
and it
measures them well, but it never reaches the work a wide schema repeats
on every
batch: the per-column setup inside `ColumnWriter::write_batch`, and the
per-mini-batch and per-page checks below it.
With one `write` call per file, a change to that per-batch path is
invisible here.
# What changes are included in this PR?
Adds `writer_overhead/{1000,5000}_cols/repeated_batches`. It holds the
number of
column writers fixed and writes 32 batches of 32 rows into a single row
group,
reusing the same wide schema and the same per-column `WriterProperties`
as the
existing cases, so the only difference from `writer_overhead/{n}_cols`
is the
number of `write` calls.
The 10,000 column case is deliberately left out of the new benchmark so
that one
iteration stays well under a second. On the machine used here an
iteration is
roughly 13 ms at 1,000 columns and 90 ms at 5,000 columns.
# Are these changes tested?
No, this is a benchmark-only change. Run it with:
cargo bench -p parquet --bench writer_overhead
# Are there any user-facing changes?
No.
# AI usage
This PR was written with Claude Code and reviewed by a human.
---------
Co-authored-by: Claude Opus 5 <[email protected]>
---
parquet/benches/writer_overhead.rs | 58 ++++++++++++++++++++++++++++++++++----
1 file changed, 53 insertions(+), 5 deletions(-)
diff --git a/parquet/benches/writer_overhead.rs
b/parquet/benches/writer_overhead.rs
index fc4f616eb2..e0bbe7b551 100644
--- a/parquet/benches/writer_overhead.rs
+++ b/parquet/benches/writer_overhead.rs
@@ -19,9 +19,9 @@
//!
//! These benchmarks measure the structural cost of creating, writing, and
//! closing a parquet file with many columns while keeping actual data
-//! encoding negligible (1 row per column). This isolates overhead such as
-//! `WriterProperties` per-column lookups, `GenericColumnWriter` allocation,
-//! and metadata assembly.
+//! encoding negligible. This isolates overhead such as `WriterProperties`
+//! per-column lookups, `GenericColumnWriter` allocation, metadata assembly,
+//! and the per-column work repeated by every `write` call.
use criterion::{Criterion, criterion_group, criterion_main};
use std::hint::black_box;
@@ -45,8 +45,12 @@ fn make_wide_schema(num_columns: usize) -> SchemaRef {
}
fn make_single_row_batch(schema: &SchemaRef) -> RecordBatch {
+ make_batch(schema, 1)
+}
+
+fn make_batch(schema: &SchemaRef, num_rows: usize) -> RecordBatch {
let columns: Vec<Arc<dyn arrow_array::Array>> = (0..schema.fields().len())
- .map(|_| Arc::new(Float32Array::from(vec![0.0f32])) as _)
+ .map(|_| Arc::new(Float32Array::from(vec![0.0f32; num_rows])) as _)
.collect();
RecordBatch::try_new(schema.clone(), columns).unwrap()
}
@@ -64,6 +68,12 @@ fn make_per_column_props(schema: &SchemaRef) ->
WriterProperties {
builder.build()
}
+/// Measures the per-column-writer overhead of a wide schema, by writing a
+/// single one-row batch per file.
+///
+/// Writing one batch per file means this is dominated by column writer
+/// construction and metadata assembly. [`bench_writer_repeated_batches`]
+/// isolates the work that repeats on every batch instead.
fn bench_writer_overhead(c: &mut Criterion) {
for &num_cols in COLUMN_COUNTS {
let schema = make_wide_schema(num_cols);
@@ -82,5 +92,43 @@ fn bench_writer_overhead(c: &mut Criterion) {
}
}
-criterion_group!(benches, bench_writer_overhead);
+/// Measures the per-`write` (rather than per-column-writer) overhead of a wide
+/// schema, by writing many small batches into a single row group.
+///
+/// This keeps the number of column writers fixed and increases the number of
+/// `write` calls, isolating the work that repeats on every batch.
+fn bench_writer_repeated_batches(c: &mut Criterion) {
+ // Number of batches written per file, and the number of rows in each.
+ const BATCH_COUNT: usize = 32;
+ const BATCH_ROWS: usize = 32;
+
+ // Kept below the widest case in `COLUMN_COUNTS` so that one iteration
stays
+ // well under a second.
+ for &num_cols in &[1_000, 5_000] {
+ let schema = make_wide_schema(num_cols);
+ let batch = make_batch(&schema, BATCH_ROWS);
+ let props = make_per_column_props(&schema);
+
+ c.bench_function(
+ &format!("writer_overhead/{num_cols}_cols/repeated_batches"),
+ |b| {
+ b.iter(|| {
+ let mut writer =
+ ArrowWriter::try_new(Empty::default(), schema.clone(),
Some(props.clone()))
+ .unwrap();
+ for _ in 0..BATCH_COUNT {
+ writer.write(black_box(&batch)).unwrap();
+ }
+ black_box(writer.close()).unwrap();
+ });
+ },
+ );
+ }
+}
+
+criterion_group!(
+ benches,
+ bench_writer_overhead,
+ bench_writer_repeated_batches
+);
criterion_main!(benches);
