This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new a64d97ace0 bench(arrow): concat of many small FixedSizeList arrays
(#11434)
a64d97ace0 is described below
commit a64d97ace046290d94dcd697cb03dc7ecf3cfdd3
Author: mightsleep <[email protected]>
AuthorDate: Thu Oct 8 02:43:44 2026 +0000
bench(arrow): concat of many small FixedSizeList arrays (#11434)
# Rationale for this change
No benchmark covers concatenating many small FixedSizeList arrays;
"concat fixed size lists" concatenates two large arrays, where the copy
dominates. Many one-row arrays is what DataFusion's
ScalarValue::iter_to_array does for FixedSizeList scalars
(GroupsAccumulatorAdapter::evaluate, first_value/last_value) and what
Rerun's chunk batcher does per logged row. Split out so the follow-up
fast path can be compared against main.
# What changes are included in this PR?
- concat 1024 arrays fixed size list f32x3 1
- concat 1024 arrays fixed size list f32x3 nulls 1, built as slices of
one array since the generator's fixed seed would give every one-row
array the same validity
# Are these changes tested?
Benchmarks only.
# Are there any significant user-facing changes?
No.
---
arrow/benches/concatenate_kernel.rs | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/arrow/benches/concatenate_kernel.rs
b/arrow/benches/concatenate_kernel.rs
index 6ae97c90a9..7e4f90f354 100644
--- a/arrow/benches/concatenate_kernel.rs
+++ b/arrow/benches/concatenate_kernel.rs
@@ -192,6 +192,27 @@ fn add_benchmark(c: &mut Criterion) {
b.iter(|| bench_concat(&v1, &v2))
});
+ // A column assembled from one-row arrays, as ScalarValue::iter_to_array
in DataFusion does
+ {
+ let input = (0..1024)
+ .map(|_| create_primitive_fixed_size_list_array::<Float32Type>(1,
0.0, 0.0, 3))
+ .collect::<Vec<_>>();
+ let arrays: Vec<_> = input.iter().map(|arr| arr as &dyn
Array).collect();
+ c.bench_function("concat 1024 arrays fixed size list f32x3 1", |b| {
+ b.iter(|| bench_concat_arrays(&arrays))
+ });
+ }
+
+ {
+ // Slices, as the generator's fixed seed would give every one-row
array the same validity
+ let input =
create_primitive_fixed_size_list_array::<Float32Type>(1024, 0.2, 0.0, 3);
+ let slices = (0..1024).map(|i| input.slice(i, 1)).collect::<Vec<_>>();
+ let arrays: Vec<_> = slices.iter().map(|arr| arr as &dyn
Array).collect();
+ c.bench_function("concat 1024 arrays fixed size list f32x3 nulls 1",
|b| {
+ b.iter(|| bench_concat_arrays(&arrays))
+ });
+ }
+
{
let batch_size = 1024;
let batch_count = 2;
