This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 0a73a0b3 perf(compute): use stack scratch for null propagation (#1229)
0a73a0b3 is described below
commit 0a73a0b32dc99ea7cb043a15bb0a40266574a6bc
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 23:38:29 2026 +0200
perf(compute): use stack scratch for null propagation (#1229)
## What
- Use an 8-entry stack scratch array for the common input counts in
`propagateNulls`.
- Keep an exact-capacity heap fallback for larger batches.
## Benchmark
Apple M1 Pro, Go 1.26.3. Representative results from `go test -vet=off
./arrow/compute -run "^$" -bench "^BenchmarkPropagateNullsScratch$"
-benchmem -benchtime=400ms -count=3 -cpu=1`.
| Case | Before | After |
| --- | ---: | ---: |
| nullable/5 | 138 ns/op, 48 B/op, 1 alloc | 95 ns/op, 0 B/op, 0 allocs
|
| nullable/8 | 221 ns/op, 64 B/op, 1 alloc | 166 ns/op, 0 B/op, 0 allocs
|
| nullable/9 | 255 ns/op, 80 B/op, 1 alloc | 256 ns/op, 80 B/op, 1 alloc
|
## Tests
- `go test -vet=off ./arrow/compute -count=1`
- `PARQUET_TEST_DATA=parquet-testing/data go test ./... -count=1`
---
arrow/compute/executor.go | 10 +-
arrow/compute/propagate_nulls_benchmark_test.go | 132 ++++++++++++++++++++++++
2 files changed, 139 insertions(+), 3 deletions(-)
diff --git a/arrow/compute/executor.go b/arrow/compute/executor.go
index c32c9768..c9e4c2ef 100644
--- a/arrow/compute/executor.go
+++ b/arrow/compute/executor.go
@@ -246,10 +246,14 @@ func propagateNulls(ctx *exec.KernelCtx, batch
*exec.ExecSpan, out *exec.ArraySp
return fmt.Errorf("%w: can only propagate nulls into
pre-allocated memory when output offset is non-zero", arrow.ErrInvalid)
}
- var (
+ var scratch [8]*exec.ArraySpan
+ arrsWithNulls := scratch[:0]
+ if len(batch.Values) > len(scratch) {
arrsWithNulls = make([]*exec.ArraySpan, 0, len(batch.Values))
- isAllNull bool
- prealloc = out.Buffers[0].Buf != nil
+ }
+ var (
+ isAllNull bool
+ prealloc = out.Buffers[0].Buf != nil
)
for i := range batch.Values {
diff --git a/arrow/compute/propagate_nulls_benchmark_test.go
b/arrow/compute/propagate_nulls_benchmark_test.go
new file mode 100644
index 00000000..e81d0b74
--- /dev/null
+++ b/arrow/compute/propagate_nulls_benchmark_test.go
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build go1.24
+
+package compute
+
+import (
+ "context"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/compute/exec"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+)
+
+func BenchmarkPropagateNullsScratch(b *testing.B) {
+ cases := []struct {
+ name string
+ inputCount int
+ nullableInput bool
+ }{
+ {name: "all-valid/1", inputCount: 1},
+ {name: "all-valid/2", inputCount: 2},
+ {name: "nullable/1", inputCount: 1, nullableInput: true},
+ {name: "nullable/2", inputCount: 2, nullableInput: true},
+ {name: "nullable/4", inputCount: 4, nullableInput: true},
+ {name: "nullable/5", inputCount: 5, nullableInput: true},
+ {name: "nullable/8", inputCount: 8, nullableInput: true},
+ {name: "nullable/9", inputCount: 9, nullableInput: true},
+ }
+
+ for _, tc := range cases {
+ b.Run(tc.name, func(b *testing.B) {
+ batch := newPropagateNullsBenchmarkBatch(tc.inputCount,
tc.nullableInput)
+ out := newPropagateNullsBenchmarkOutput()
+ ctx := &exec.KernelCtx{}
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ if err := propagateNulls(ctx, batch, out); err
!= nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+func newPropagateNullsBenchmarkBatch(inputCount int, nullableInput bool)
*exec.ExecSpan {
+ const length int64 = 64
+
+ batch := &exec.ExecSpan{
+ Len: length,
+ Values: make([]exec.ExecValue, inputCount),
+ }
+ bitmap := []byte{0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+ for i := range batch.Values {
+ batch.Values[i].Array.Type = arrow.PrimitiveTypes.Int32
+ batch.Values[i].Array.Len = length
+ if nullableInput {
+ batch.Values[i].Array.Nulls = 1
+ batch.Values[i].Array.Buffers[0].Buf = bitmap
+ }
+ }
+ return batch
+}
+
+func newPropagateNullsBenchmarkOutput() *exec.ArraySpan {
+ out := &exec.ArraySpan{
+ Type: arrow.PrimitiveTypes.Int32,
+ Len: 64,
+ }
+ out.Buffers[0].Buf = make([]byte, 8)
+ return out
+}
+
+func BenchmarkPropagateNullsSmallArrayAdd(b *testing.B) {
+ const length = 16
+ values := make([]int32, length)
+ valid := make([]bool, length)
+ for i := range valid {
+ valid[i] = i != 0
+ }
+
+ leftBuilder := array.NewInt32Builder(memory.DefaultAllocator)
+ leftBuilder.AppendValues(values, valid)
+ left := leftBuilder.NewInt32Array()
+ leftBuilder.Release()
+ defer left.Release()
+
+ rightBuilder := array.NewInt32Builder(memory.DefaultAllocator)
+ rightBuilder.AppendValues(values, nil)
+ right := rightBuilder.NewInt32Array()
+ rightBuilder.Release()
+ defer right.Release()
+
+ leftDatum := NewDatum(left)
+ defer leftDatum.Release()
+ rightDatum := NewDatum(right)
+ defer rightDatum.Release()
+
+ ctx := SetExecCtx(context.Background(), ExecCtx{
+ Registry: GetFunctionRegistry(),
+ ChunkSize: length,
+ PreallocContiguous: true,
+ })
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ out, err := CallFunction(ctx, "add", nil, leftDatum, rightDatum)
+ if err != nil {
+ b.Fatal(err)
+ }
+ out.Release()
+ }
+}
