This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 4c667c96 perf(arrow/array): copy same-dictionary indices directly
(#1225)
4c667c96 is described below
commit 4c667c96e0cfaa6836ddf6504bd4188ceaa0aae7
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 21:20:45 2026 +0200
perf(arrow/array): copy same-dictionary indices directly (#1225)
## What does this change?
- Same-dictionary concatenation used to create one temporary buffer
wrapper per input chunk.
- Use `concatFixedWidthBuffers` directly for the dictionary indices.
- Keep the dictionary-unification path unchanged for different
dictionaries.
## Benchmark
Command:
```text
go test ./arrow/array -run '^$' -bench
'^BenchmarkConcatenateSameDictionary$' -benchmem -benchtime=200ms -count=3
```
Apple M1 Pro, Go 1.26.3. The benchmark concatenates 65,536 rows with the
same four-value string dictionary, split across different chunk counts.
| Chunks | Before ns/op | After ns/op | Before B/op | After B/op |
Allocs before/after |
| ---: | ---: | ---: | ---: | ---: | ---: |
| 1 | 33,736 | 33,262 | 271,049 | 270,962 | 11 / 9 |
| 8 | 33,791 | 33,809 | 272,353 | 271,650 | 26 / 17 |
| 64 | 42,134 | 39,250 | 282,785 | 277,153 | 138 / 73 |
| 1,024 | 198,280 | 157,632 | 462,626 | 371,234 | 2,058 / 1,033 |
| 8,192 | 1,202,369 | 928,859 | 1,778,211 | 1,057,314 | 16,394 / 8,201 |
## Tests
- `go test ./arrow/array -count=1`
- `go test ./arrow/... ./internal/...`
---
arrow/array/concat.go | 3 +-
arrow/array/dictionary_concat_benchmark_test.go | 83 +++++++++++++++++++++++++
arrow/array/dictionary_concat_test.go | 55 ++++++++++++++++
3 files changed, 139 insertions(+), 2 deletions(-)
diff --git a/arrow/array/concat.go b/arrow/array/concat.go
index 3a9efe9e..510ebe54 100644
--- a/arrow/array/concat.go
+++ b/arrow/array/concat.go
@@ -595,11 +595,10 @@ func concat(data []arrow.ArrayData, mem memory.Allocator)
(arr arrow.ArrayData,
dict.Release()
}
- indexBuffers := gatherBuffersFixedWidthType(data, 1, idxType)
if dictsSame {
out.dictionary = dict0.Data().(*Data)
out.dictionary.Retain()
- out.buffers[1] = concatBuffers(indexBuffers, mem)
+ out.buffers[1] = concatFixedWidthBuffers(data, 1,
idxType.BitWidth()/8, out.length, mem)
break
}
diff --git a/arrow/array/dictionary_concat_benchmark_test.go
b/arrow/array/dictionary_concat_benchmark_test.go
new file mode 100644
index 00000000..14a8b7bf
--- /dev/null
+++ b/arrow/array/dictionary_concat_benchmark_test.go
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+)
+
+func BenchmarkConcatenateSameDictionary(b *testing.B) {
+ const totalValues = 1 << 16
+
+ mem := memory.NewGoAllocator()
+ dictType := &arrow.DictionaryType{IndexType:
arrow.PrimitiveTypes.Int32, ValueType: arrow.BinaryTypes.String}
+
+ values := make([]int32, totalValues)
+ for i := range values {
+ values[i] = int32(i % 4)
+ }
+ indicesBuilder := array.NewInt32Builder(mem)
+ indicesBuilder.AppendValues(values, nil)
+ indices := indicesBuilder.NewInt32Array()
+ indicesBuilder.Release()
+ defer indices.Release()
+
+ dictionaryBuilder := array.NewStringBuilder(mem)
+ dictionaryBuilder.AppendValues([]string{"zero", "one", "two", "three"},
nil)
+ dictionary := dictionaryBuilder.NewStringArray()
+ dictionaryBuilder.Release()
+ defer dictionary.Release()
+
+ backing := array.NewDictionaryArray(dictType, indices, dictionary)
+ defer backing.Release()
+
+ for _, chunkCount := range []int{1, 8, 64, 1024, 8192} {
+ chunkCount := chunkCount
+ b.Run(fmt.Sprintf("chunks-%d", chunkCount), func(b *testing.B) {
+ chunkSize := totalValues / chunkCount
+ chunks := make([]arrow.Array, chunkCount)
+ for i := range chunks {
+ begin := int64(i * chunkSize)
+ chunks[i] = array.NewSlice(backing, begin,
begin+int64(chunkSize))
+ }
+ defer func() {
+ for _, chunk := range chunks {
+ chunk.Release()
+ }
+ }()
+
+ b.SetBytes(int64(totalValues * arrow.Int32SizeBytes))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ result, err := array.Concatenate(chunks, mem)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if result.Len() != totalValues {
+ b.Fatalf("result length = %d, want %d",
result.Len(), totalValues)
+ }
+ result.Release()
+ }
+ })
+ }
+}
diff --git a/arrow/array/dictionary_concat_test.go
b/arrow/array/dictionary_concat_test.go
new file mode 100644
index 00000000..bfd02827
--- /dev/null
+++ b/arrow/array/dictionary_concat_test.go
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestConcatSameDictionarySlices(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType:
arrow.PrimitiveTypes.Int32, ValueType: arrow.BinaryTypes.String}
+ backing, err := array.DictArrayFromJSON(mem, dictType, `[0, 1, null, 2,
1, 0]`, `["a", "b", "c"]`)
+ require.NoError(t, err)
+ defer backing.Release()
+
+ inputs := []arrow.Array{
+ array.NewSlice(backing, 1, 5),
+ array.NewSlice(backing, 0, 2),
+ }
+ for _, input := range inputs {
+ defer input.Release()
+ }
+
+ actual, err := array.Concatenate(inputs, mem)
+ require.NoError(t, err)
+ defer actual.Release()
+
+ expected, err := array.DictArrayFromJSON(mem, dictType, `[1, null, 2,
1, 0, 1]`, `["a", "b", "c"]`)
+ require.NoError(t, err)
+ defer expected.Release()
+
+ assert.True(t, array.Equal(expected, actual))
+}
