This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 80bfd314 perf(arrow/array): bulk append fixed-size list nulls and
empty values (#1183)
80bfd314 is described below
commit 80bfd314fb89197600827ad907c6f1d500eb3d9c
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 14 18:15:01 2026 +0200
perf(arrow/array): bulk append fixed-size list nulls and empty values
(#1183)
## What does this PR do?
- reserves fixed-size list parent slots once in `AppendNulls` and
`AppendEmptyValues`
- updates the parent validity bitmap in bulk
- appends all child null or empty values with one bulk builder call
The child append count is `listSize * n`, so the parent and child
lengths stay in sync.
## Why?
Both methods currently loop over every parent value. Each parent append
also loops over every child value.
This removes the parent loop and lets child builders use their bulk
append paths. The change is independent, but it also composes with #1173
and #1180.
## Benchmarks
Apple M1 Pro, 65,536 `FixedSizeList<Int32>` rows, `GOMAXPROCS=1`.
Standalone on `main`:
| list width | nulls before | nulls after | change | empty before |
empty after | change |
| ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| 4 | 1.10 ms | 0.86 ms | -22% | 1.55 ms | 1.07 ms | -31% |
| 16 | 3.76 ms | 2.74 ms | -27% | 5.43 ms | 3.90 ms | -28% |
| 64 | 13.03 ms | 11.07 ms | -15% | 19.35 ms | 14.86 ms | -23% |
With #1173 and #1180 applied underneath:
| list width | nulls before | nulls after | change | empty before |
empty after | change |
| ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| 4 | 1.56 ms | 0.18 ms | -88% | 2.01 ms | 0.20 ms | -90% |
| 16 | 4.69 ms | 0.33 ms | -93% | 6.89 ms | 0.42 ms | -94% |
| 64 | 16.27 ms | 0.74 ms | -95% | 22.18 ms | 0.92 ms | -96% |
```text
GOMAXPROCS=1 go test ./arrow/array -run '^$' -bench
'^BenchmarkFixedSizeListBuilderBulkAppend$/(nulls|empty)/rows=65536/width=(4|16|64)$'
-benchmem -benchtime=300ms -count=5
```
## Tests
- added coverage for unaligned parent validity bits
- verifies parent validity and null counts
- verifies child length, null slots, and zero-valued empty slots
- ran `go test ./arrow/...`
- ran `go test -race ./arrow/array`
- ran `go vet ./arrow/array`
---
arrow/array/fixed_size_list.go | 26 +++++++--
arrow/array/fixed_size_list_test.go | 104 ++++++++++++++++++++++++++++++++++++
2 files changed, 126 insertions(+), 4 deletions(-)
diff --git a/arrow/array/fixed_size_list.go b/arrow/array/fixed_size_list.go
index f13797cc..55477f11 100644
--- a/arrow/array/fixed_size_list.go
+++ b/arrow/array/fixed_size_list.go
@@ -241,9 +241,19 @@ func (b *FixedSizeListBuilder) AppendNull() {
// AppendNulls will append n null values to the underlying values by itself
func (b *FixedSizeListBuilder) AppendNulls(n int) {
- for i := 0; i < n; i++ {
- b.AppendNull()
+ if n <= 0 {
+ return
+ }
+
+ b.Reserve(n)
+ if n == 1 {
+ b.unsafeAppendBoolToBitmap(false)
+ } else {
+ bitutil.SetBitsTo(b.nullBitmap.Bytes(), int64(b.length),
int64(n), false)
+ b.length += n
+ b.nulls += n
}
+ b.values.AppendNulls(n * int(b.n))
}
func (b *FixedSizeListBuilder) AppendEmptyValue() {
@@ -254,9 +264,17 @@ func (b *FixedSizeListBuilder) AppendEmptyValue() {
}
func (b *FixedSizeListBuilder) AppendEmptyValues(n int) {
- for i := 0; i < n; i++ {
- b.AppendEmptyValue()
+ if n <= 0 {
+ return
+ }
+
+ b.Reserve(n)
+ if n == 1 {
+ b.unsafeAppendBoolToBitmap(true)
+ } else {
+ b.unsafeAppendBoolsToBitmap(nil, n)
}
+ b.values.AppendEmptyValues(n * int(b.n))
}
func (b *FixedSizeListBuilder) AppendValues(valid []bool) {
diff --git a/arrow/array/fixed_size_list_test.go
b/arrow/array/fixed_size_list_test.go
index b899647a..6326d9c2 100644
--- a/arrow/array/fixed_size_list_test.go
+++ b/arrow/array/fixed_size_list_test.go
@@ -137,6 +137,110 @@ func TestFixedSizeListArrayBulkAppend(t *testing.T) {
}
}
+func TestFixedSizeListArrayBulkAppendNullsAndEmptyValues(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ lb := array.NewFixedSizeListBuilder(pool, 3, arrow.PrimitiveTypes.Int32)
+ defer lb.Release()
+ vb := lb.ValueBuilder().(*array.Int32Builder)
+
+ lb.Append(true)
+ vb.AppendValues([]int32{1, 2, 3}, nil)
+ lb.AppendNulls(5)
+ lb.AppendEmptyValues(4)
+ lb.Append(true)
+ vb.AppendValues([]int32{4, 5, 6}, nil)
+
+ arr := lb.NewListArray()
+ defer arr.Release()
+ assert.Equal(t, 11, arr.Len())
+ assert.Equal(t, 5, arr.NullN())
+ valid := []bool{true, false, false, false, false, false, true, true,
true, true, true}
+ for i, want := range valid {
+ assert.Equal(t, want, arr.IsValid(i), "list value %d", i)
+ }
+
+ values := arr.ListValues().(*array.Int32)
+ assert.Equal(t, 33, values.Len())
+ assert.Equal(t, 15, values.NullN())
+ assert.Equal(t, []int32{1, 2, 3}, values.Int32Values()[:3])
+ for i := 3; i < 18; i++ {
+ assert.True(t, values.IsNull(i), "child value %d", i)
+ }
+ for i := 18; i < 30; i++ {
+ assert.True(t, values.IsValid(i), "child value %d", i)
+ assert.Zero(t, values.Value(i), "child value %d", i)
+ }
+ assert.Equal(t, []int32{4, 5, 6}, values.Int32Values()[30:])
+}
+
+func TestFixedSizeListArrayBulkAppendNested(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ lb := array.NewFixedSizeListBuilder(pool, 2, arrow.FixedSizeListOf(3,
arrow.PrimitiveTypes.Int32))
+ defer lb.Release()
+
+ lb.AppendNulls(2)
+ lb.AppendEmptyValues(2)
+
+ arr := lb.NewListArray()
+ defer arr.Release()
+ assert.NoError(t, arr.ValidateFull())
+ assert.Equal(t, 4, arr.Len())
+ assert.Equal(t, 2, arr.NullN())
+
+ children := arr.ListValues().(*array.FixedSizeList)
+ assert.Equal(t, 8, children.Len())
+ assert.Equal(t, 4, children.NullN())
+
+ values := children.ListValues().(*array.Int32)
+ assert.Equal(t, 24, values.Len())
+ assert.Equal(t, 12, values.NullN())
+ for i := 0; i < 12; i++ {
+ assert.True(t, values.IsNull(i), "child value %d", i)
+ }
+ for i := 12; i < 24; i++ {
+ assert.True(t, values.IsValid(i), "child value %d", i)
+ assert.Zero(t, values.Value(i), "child value %d", i)
+ }
+}
+
+func BenchmarkFixedSizeListBuilderBulkAppend(b *testing.B) {
+ for _, rows := range []int{1, 8, 64, 1024, 65536} {
+ for _, width := range []int32{1, 4, 16, 64} {
+ name := fmt.Sprintf("rows=%d/width=%d", rows, width)
+ b.Run("nulls/"+name, func(b *testing.B) {
+ benchmarkFixedSizeListBuilderBulkAppend(b,
rows, width, false)
+ })
+ b.Run("empty/"+name, func(b *testing.B) {
+ benchmarkFixedSizeListBuilderBulkAppend(b,
rows, width, true)
+ })
+ }
+ }
+}
+
+func benchmarkFixedSizeListBuilderBulkAppend(b *testing.B, rows int, width
int32, empty bool) {
+ bldr := array.NewFixedSizeListBuilder(memory.DefaultAllocator, width,
arrow.PrimitiveTypes.Int32)
+ defer bldr.Release()
+ b.ReportAllocs()
+ b.StopTimer()
+
+ for i := 0; i < b.N; i++ {
+ b.StartTimer()
+ if empty {
+ bldr.AppendEmptyValues(rows)
+ } else {
+ bldr.AppendNulls(rows)
+ }
+ b.StopTimer()
+
+ arr := bldr.NewListArray()
+ arr.Release()
+ }
+}
+
func TestFixedSizeListArrayStringer(t *testing.T) {
pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer pool.AssertSize(t, 0)
