This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 9b1b0958 perf(arrow/array): optimize bulk appends for binary builders
(#1188)
9b1b0958 is described below
commit 9b1b09582b9912e627852759fb1de13a075e2da7
Author: Minh Vu <[email protected]>
AuthorDate: Wed Aug 26 17:40:45 2026 +0200
perf(arrow/array): optimize bulk appends for binary builders (#1188)
## Summary
- reserve the full batch once in `BinaryBuilder.AppendNulls` and
`AppendEmptyValues`
- fill the repeated current offsets directly for both 32-bit and 64-bit
offset buffers
- update the validity bitmap and counters once per batch
- add coverage and benchmarks for Binary, LargeBinary, String, and
LargeString
This follows the same batching shape used by [Arrow C++ binary
builders](https://github.com/apache/arrow/blob/485499fd02ea2b0c323d67871fbe96aae4232504/cpp/src/arrow/array/builder_binary.h#L100-L140).
## Benchmark
65,536 appended values, representative medians on darwin/arm64:
| Builder | Operation | Before | After | Change |
| --- | --- | ---: | ---: | ---: |
| Binary | nulls | 403 us | 93 us | -77% |
| LargeBinary | nulls | 475 us | 158 us | -67% |
| String | nulls | 402 us | 85 us | -79% |
| String | empty | 458 us | 84 us | -82% |
| LargeString | nulls | 532 us | 159 us | -70% |
| LargeString | empty | 517 us | 154 us | -70% |
Allocations per build drop from 27 to 8 for 32-bit offsets and from 28
to 8 for 64-bit offsets. Single-value performance stays about the same.
## Tests
- `go test ./arrow/array -count=1`
- `go test -race ./arrow/array -run
TestBinaryBuilderBulkAppendNullsAndEmptyValues -count=1`
- `go vet ./arrow/array`
---
arrow/array/binarybuilder.go | 38 +++++-
arrow/array/binarybuilder_bulk_test.go | 233 +++++++++++++++++++++++++++++++++
2 files changed, 267 insertions(+), 4 deletions(-)
diff --git a/arrow/array/binarybuilder.go b/arrow/array/binarybuilder.go
index 2391bbe2..8d7c5977 100644
--- a/arrow/array/binarybuilder.go
+++ b/arrow/array/binarybuilder.go
@@ -25,6 +25,7 @@ import (
"unsafe"
"github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/bitutil"
"github.com/apache/arrow-go/v18/arrow/internal/debug"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/apache/arrow-go/v18/internal/json"
@@ -127,9 +128,15 @@ func (b *BinaryBuilder) AppendNull() {
}
func (b *BinaryBuilder) AppendNulls(n int) {
- for i := 0; i < n; i++ {
- b.AppendNull()
+ if n <= 0 {
+ return
}
+
+ b.Reserve(n)
+ b.appendCurrentOffsets(n)
+ bitutil.SetBitsTo(b.nullBitmap.Bytes(), int64(b.length), int64(n),
false)
+ b.length += n
+ b.nulls += n
}
func (b *BinaryBuilder) AppendEmptyValue() {
@@ -139,9 +146,13 @@ func (b *BinaryBuilder) AppendEmptyValue() {
}
func (b *BinaryBuilder) AppendEmptyValues(n int) {
- for i := 0; i < n; i++ {
- b.AppendEmptyValue()
+ if n <= 0 {
+ return
}
+
+ b.Reserve(n)
+ b.appendCurrentOffsets(n)
+ b.unsafeAppendBoolsToBitmap(nil, n)
}
// AppendValues will append the values in the v slice. The valid slice
determines which values
@@ -348,6 +359,25 @@ func (b *BinaryBuilder) appendNextOffset() {
b.appendOffsetVal(numBytes)
}
+func (b *BinaryBuilder) appendCurrentOffsets(n int) {
+ numBytes := b.values.Len()
+ debug.Assert(uint64(numBytes) <= b.maxCapacity, "exceeded maximum
capacity of binary array")
+ start := b.offsets.Len() * b.offsetByteWidth
+ b.offsets.Advance(n * b.offsetByteWidth)
+ switch b.offsetByteWidth {
+ case arrow.Int32SizeBytes:
+ offsets :=
arrow.Int32Traits.CastFromBytes(b.offsets.Bytes()[start:])
+ for i := range offsets {
+ offsets[i] = int32(numBytes)
+ }
+ case arrow.Int64SizeBytes:
+ offsets :=
arrow.Int64Traits.CastFromBytes(b.offsets.Bytes()[start:])
+ for i := range offsets {
+ offsets[i] = int64(numBytes)
+ }
+ }
+}
+
func (b *BinaryBuilder) AppendValueFromString(s string) error {
if s == NullValueStr {
b.AppendNull()
diff --git a/arrow/array/binarybuilder_bulk_test.go
b/arrow/array/binarybuilder_bulk_test.go
new file mode 100644
index 00000000..8448cb30
--- /dev/null
+++ b/arrow/array/binarybuilder_bulk_test.go
@@ -0,0 +1,233 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package array_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+type binaryBuilderFactory func(memory.Allocator) array.Builder
+
+var binaryBuilderFactories = []struct {
+ name string
+ new binaryBuilderFactory
+}{
+ {"binary", func(mem memory.Allocator) array.Builder {
+ return array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
+ }},
+ {"large_binary", func(mem memory.Allocator) array.Builder {
+ return array.NewBinaryBuilder(mem,
arrow.BinaryTypes.LargeBinary)
+ }},
+ {"string", func(mem memory.Allocator) array.Builder {
+ return array.NewStringBuilder(mem)
+ }},
+ {"large_string", func(mem memory.Allocator) array.Builder {
+ return array.NewLargeStringBuilder(mem)
+ }},
+}
+
+func TestBinaryBuilderBulkAppendNullsAndEmptyValues(t *testing.T) {
+ for _, tc := range binaryBuilderFactories {
+ t.Run(tc.name, func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ builder := tc.new(mem)
+ defer builder.Release()
+ appendBinaryBuilderValue(builder, "abc")
+ builder.AppendNulls(5)
+ builder.AppendEmptyValues(4)
+ appendBinaryBuilderValue(builder, "de")
+
+ arr := builder.NewArray().(array.BinaryLike)
+ defer arr.Release()
+ require.NoError(t, arr.(interface{ ValidateFull() error
}).ValidateFull())
+
+ assert.Equal(t, 11, arr.Len())
+ assert.Equal(t, 5, arr.NullN())
+ assert.Equal(t, []byte("abcde"), arr.ValueBytes())
+ for i := range arr.Len() {
+ assert.Equal(t, i == 0 || i >= 6,
arr.IsValid(i), "value %d", i)
+ switch i {
+ case 0:
+ assert.Equal(t, int64(0),
arr.ValueOffset64(i))
+ assert.Equal(t, 3, arr.ValueLen(i))
+ case 10:
+ assert.Equal(t, int64(3),
arr.ValueOffset64(i))
+ assert.Equal(t, 2, arr.ValueLen(i))
+ default:
+ assert.Equal(t, int64(3),
arr.ValueOffset64(i))
+ assert.Zero(t, arr.ValueLen(i))
+ }
+ }
+ })
+ }
+}
+
+func appendBinaryBuilderValue(builder array.Builder, value string) {
+ switch builder := builder.(type) {
+ case *array.BinaryBuilder:
+ builder.Append([]byte(value))
+ case *array.StringBuilder:
+ builder.Append(value)
+ case *array.LargeStringBuilder:
+ builder.Append(value)
+ default:
+ panic(fmt.Sprintf("unexpected binary builder %T", builder))
+ }
+}
+
+func TestBinaryBuilderBulkAppendMatchesScalar(t *testing.T) {
+ starts := []int{0, 1, 7, 8, 9, 15, 16, 17}
+ batchSizes := []int{-1, 0, 1, 2, 7, 8, 9, 16, 17}
+ operations := []struct {
+ name string
+ bulk func(array.Builder, int)
+ scalar func(array.Builder, int)
+ }{
+ {
+ name: "nulls",
+ bulk: func(builder array.Builder, n int) {
+ builder.AppendNulls(n)
+ },
+ scalar: func(builder array.Builder, n int) {
+ for i := 0; i < n; i++ {
+ builder.AppendNull()
+ }
+ },
+ },
+ {
+ name: "empty_values",
+ bulk: func(builder array.Builder, n int) {
+ builder.AppendEmptyValues(n)
+ },
+ scalar: func(builder array.Builder, n int) {
+ for i := 0; i < n; i++ {
+ builder.AppendEmptyValue()
+ }
+ },
+ },
+ }
+
+ for _, factory := range binaryBuilderFactories {
+ t.Run(factory.name, func(t *testing.T) {
+ for operationIndex, operation := range operations {
+ t.Run(operation.name, func(t *testing.T) {
+ reuseOperation :=
operations[1-operationIndex]
+ for _, start := range starts {
+ for _, batchSize := range
batchSizes {
+
t.Run(fmt.Sprintf("start_%d_batch_%d", start, batchSize), func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer
mem.AssertSize(t, 0)
+
+ bulk :=
factory.new(mem)
+ defer
bulk.Release()
+ scalar :=
factory.new(mem)
+ defer
scalar.Release()
+
+
appendBinaryBuilderPrefix(bulk, start)
+
appendBinaryBuilderPrefix(scalar, start)
+
operation.bulk(bulk, batchSize)
+
operation.scalar(scalar, batchSize)
+
assertBinaryBuilderArrayParity(t, bulk, scalar)
+
+
appendBinaryBuilderPrefix(bulk, start)
+
appendBinaryBuilderPrefix(scalar, start)
+
reuseOperation.bulk(bulk, 9)
+
reuseOperation.scalar(scalar, 9)
+
assertBinaryBuilderArrayParity(t, bulk, scalar)
+ })
+ }
+ }
+ })
+ }
+ })
+ }
+}
+
+func appendBinaryBuilderPrefix(builder array.Builder, n int) {
+ for i := 0; i < n; i++ {
+ switch i % 4 {
+ case 0:
+ appendBinaryBuilderValue(builder,
fmt.Sprintf("value-%d", i))
+ case 1:
+ builder.AppendNull()
+ case 2:
+ builder.AppendEmptyValue()
+ case 3:
+ appendBinaryBuilderValue(builder,
fmt.Sprintf("tail-%d", i))
+ }
+ }
+}
+
+func assertBinaryBuilderArrayParity(t *testing.T, bulk, scalar array.Builder) {
+ t.Helper()
+
+ assert.Equal(t, scalar.Len(), bulk.Len())
+ assert.Equal(t, scalar.NullN(), bulk.NullN())
+
+ bulkArray := bulk.NewArray()
+ defer bulkArray.Release()
+ scalarArray := scalar.NewArray()
+ defer scalarArray.Release()
+
+ require.NoError(t, bulkArray.(interface{ ValidateFull() error
}).ValidateFull())
+ require.NoError(t, scalarArray.(interface{ ValidateFull() error
}).ValidateFull())
+ assert.True(t, array.Equal(bulkArray, scalarArray))
+}
+
+func BenchmarkBinaryBuilderBulkAppend(b *testing.B) {
+ for _, tc := range binaryBuilderFactories {
+ b.Run(tc.name, func(b *testing.B) {
+ for _, rows := range []int{1, 16, 1024, 65536} {
+ b.Run(fmt.Sprintf("rows_%d", rows), func(b
*testing.B) {
+ b.Run("nulls", func(b *testing.B) {
+
benchmarkBinaryBuilderBulkAppend(b, tc.new, rows, false)
+ })
+ b.Run("empty", func(b *testing.B) {
+
benchmarkBinaryBuilderBulkAppend(b, tc.new, rows, true)
+ })
+ })
+ }
+ })
+ }
+}
+
+func benchmarkBinaryBuilderBulkAppend(b *testing.B, factory
binaryBuilderFactory, rows int, empty bool) {
+ builder := factory(memory.DefaultAllocator)
+ defer builder.Release()
+ b.ReportAllocs()
+
+ for b.Loop() {
+ if empty {
+ builder.AppendEmptyValues(rows)
+ } else {
+ builder.AppendNulls(rows)
+ }
+
+ arr := builder.NewArray()
+ arr.Release()
+ }
+}
