This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 8d2e8f70 perf(arrow/array): batch run-end encoded appends (#1214)
8d2e8f70 is described below
commit 8d2e8f70cf06048fd03d85822af387940fd91040
Author: Minh Vu <[email protected]>
AuthorDate: Wed Aug 26 22:59:52 2026 +0200
perf(arrow/array): batch run-end encoded appends (#1214)
## What
- Batch `RunEndEncodedBuilder.AppendNulls` into one physical null value
and one run.
- Batch `RunEndEncodedBuilder.AppendEmptyValues` into one physical empty
value and one run.
- Keep zero and negative counts as no-ops.
- Preserve the empty-value state used by JSON decoding.
The old implementation called the scalar append method once per logical
value. This change keeps the same API while avoiding the repeated
physical values and run ends.
## Benchmark
Apple M1 Pro, 65,536 rows, `int32` run ends and `int32` encoded values:
| Case | Before | After | Speedup |
| --- | ---: | ---: | ---: |
| Nulls | ~1.0-1.6 ms, 1.16 MB, 55 allocs/op | ~1.0-1.5 us, 1.7 KB, 19
allocs/op | ~1,250x |
| Empty values | ~1.2-2.7 ms, 1.16 MB, 55 allocs/op | ~1.2-1.5 us, 1.7
KB, 19 allocs/op | ~1,750x |
The benchmark covers `int16`, `int32`, and `int64` run-end types,
`int32` and string encoded values, and multiple batch sizes.
## Tests
- `go test ./arrow/array -count=1`
- `go test ./... -run '^$' -count=1 -p 2`
---
arrow/array/encoded.go | 17 +++++--
arrow/array/encoded_builder_bulk_test.go | 86 ++++++++++++++++++++++++++++++++
arrow/array/encoded_test.go | 44 +++++++++++++---
3 files changed, 137 insertions(+), 10 deletions(-)
diff --git a/arrow/array/encoded.go b/arrow/array/encoded.go
index e3cf21fa..8e174f96 100644
--- a/arrow/array/encoded.go
+++ b/arrow/array/encoded.go
@@ -502,9 +502,13 @@ func (b *RunEndEncodedBuilder) AppendNull() {
}
func (b *RunEndEncodedBuilder) AppendNulls(n int) {
- for i := 0; i < n; i++ {
- b.AppendNull()
+ if n <= 0 {
+ return
}
+
+ b.finishRun()
+ b.values.AppendNull()
+ b.addLength(uint64(n))
}
func (b *RunEndEncodedBuilder) UnsafeAppendBoolToBitmap(v bool) {
@@ -523,9 +527,14 @@ func (b *RunEndEncodedBuilder) AppendEmptyValue() {
}
func (b *RunEndEncodedBuilder) AppendEmptyValues(n int) {
- for i := 0; i < n; i++ {
- b.AppendEmptyValue()
+ if n <= 0 {
+ return
}
+
+ b.finishRun()
+ b.values.AppendEmptyValue()
+ b.addLength(uint64(n))
+ b.lastValueWasEmpty = true
}
func (b *RunEndEncodedBuilder) Reserve(n int) {
diff --git a/arrow/array/encoded_builder_bulk_test.go
b/arrow/array/encoded_builder_bulk_test.go
new file mode 100644
index 00000000..92b21a64
--- /dev/null
+++ b/arrow/array/encoded_builder_bulk_test.go
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package array_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+)
+
+func BenchmarkRunEndEncodedBuilderBulkAppend(b *testing.B) {
+ runEndTypes := []struct {
+ name string
+ typ arrow.DataType
+ maxRows int
+ }{
+ {name: "int16", typ: arrow.PrimitiveTypes.Int16, maxRows: 1 <<
14},
+ {name: "int32", typ: arrow.PrimitiveTypes.Int32, maxRows: 1 <<
16},
+ {name: "int64", typ: arrow.PrimitiveTypes.Int64, maxRows: 1 <<
16},
+ }
+ encodedTypes := []struct {
+ name string
+ typ arrow.DataType
+ }{
+ {name: "int32", typ: arrow.PrimitiveTypes.Int32},
+ {name: "string", typ: arrow.BinaryTypes.String},
+ }
+
+ for _, runEndType := range runEndTypes {
+ b.Run(runEndType.name, func(b *testing.B) {
+ for _, encodedType := range encodedTypes {
+ b.Run(encodedType.name, func(b *testing.B) {
+ for _, rows := range []int{1, 16, 1024,
1 << 14, 1 << 16} {
+ if rows > runEndType.maxRows {
+ continue
+ }
+
+ b.Run(fmt.Sprintf("rows_%d",
rows), func(b *testing.B) {
+ b.Run("nulls", func(b
*testing.B) {
+
benchmarkRunEndEncodedBuilderBulkAppend(b, runEndType.typ, encodedType.typ,
rows, false)
+ })
+ b.Run("empty", func(b
*testing.B) {
+
benchmarkRunEndEncodedBuilderBulkAppend(b, runEndType.typ, encodedType.typ,
rows, true)
+ })
+ })
+ }
+ })
+ }
+ })
+ }
+}
+
+func benchmarkRunEndEncodedBuilderBulkAppend(b *testing.B, runEndType,
encodedType arrow.DataType, rows int, empty bool) {
+ builder := array.NewRunEndEncodedBuilder(memory.DefaultAllocator,
runEndType, encodedType)
+ defer builder.Release()
+ b.ReportAllocs()
+
+ for b.Loop() {
+ if empty {
+ builder.AppendEmptyValues(rows)
+ } else {
+ builder.AppendNulls(rows)
+ }
+
+ arr := builder.NewArray()
+ arr.Release()
+ }
+}
diff --git a/arrow/array/encoded_test.go b/arrow/array/encoded_test.go
index 3370d0f1..c8a5c4e2 100644
--- a/arrow/array/encoded_test.go
+++ b/arrow/array/encoded_test.go
@@ -380,14 +380,48 @@ func TestRunEndEncodedBuilderEmptyValues(t *testing.T) {
arr := bldr.NewRunEndEncodedArray()
defer arr.Release()
+ assert.Equal(t, []int16{1, 3},
arr.RunEndsArr().(*array.Int16).Int16Values())
values := arr.Values().(*array.String)
- assert.Equal(t, 3, values.Len())
+ assert.Equal(t, 2, values.Len())
for i := 0; i < values.Len(); i++ {
assert.False(t, values.IsNull(i))
assert.Empty(t, values.Value(i))
}
}
+func TestRunEndEncodedBuilderBulkAppendNullsAndEmptyValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
arrow.BinaryTypes.String)
+ defer bldr.Release()
+
+ bldr.AppendNulls(0)
+ bldr.AppendEmptyValues(0)
+ bldr.AppendNulls(-1)
+ bldr.AppendEmptyValues(-1)
+ assert.Zero(t, bldr.Len())
+
+ bldr.AppendEmptyValues(2)
+ bldr.AppendNulls(3)
+ bldr.AppendEmptyValues(4)
+
+ arr := bldr.NewRunEndEncodedArray()
+ defer arr.Release()
+ require.NoError(t, arr.ValidateFull())
+
+ assert.Equal(t, 9, arr.Len())
+ assert.Equal(t, []int16{2, 5, 9},
arr.RunEndsArr().(*array.Int16).Int16Values())
+
+ values := arr.Values().(*array.String)
+ assert.Equal(t, 3, values.Len())
+ assert.False(t, values.IsNull(0))
+ assert.Empty(t, values.Value(0))
+ assert.True(t, values.IsNull(1))
+ assert.False(t, values.IsNull(2))
+ assert.Empty(t, values.Value(2))
+}
+
func TestRunEndEncodedBuilderEmptyValueBeforeUnmarshalNull(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
@@ -424,14 +458,12 @@ func
TestRunEndEncodedBuilderEmptyValuesBeforeUnmarshalNulls(t *testing.T) {
arr := bldr.NewRunEndEncodedArray()
defer arr.Release()
- assert.Equal(t, []int16{1, 2, 4},
arr.RunEndsArr().(*array.Int16).Int16Values())
+ assert.Equal(t, []int16{2, 4},
arr.RunEndsArr().(*array.Int16).Int16Values())
values := arr.Values().(*array.String)
- assert.Equal(t, 3, values.Len())
+ assert.Equal(t, 2, values.Len())
assert.False(t, values.IsNull(0))
assert.Empty(t, values.Value(0))
- assert.False(t, values.IsNull(1))
- assert.Empty(t, values.Value(1))
- assert.True(t, values.IsNull(2))
+ assert.True(t, values.IsNull(1))
}
func TestRunEndEncodedBuilderDictionaryEmptyValue(t *testing.T) {
