This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 330d6130 perf(arrow/array): avoid temporary buffers when concatenating
(#1194)
330d6130 is described below
commit 330d6130a59055bc65c1403ffd215c50b44b7b5b
Author: Minh Vu <[email protected]>
AuthorDate: Wed Aug 26 18:23:48 2026 +0200
perf(arrow/array): avoid temporary buffers when concatenating (#1194)
## Summary
- copy fixed-width values directly from source buffer ranges into the
output buffer
- avoid creating one temporary `memory.Buffer` for every input chunk
- add a benchmark with a fixed number of values and increasing chunk
counts
## Benchmarks
Apple M1 Pro, 65,536 int64 values:
| chunks | old time | new time | delta | old allocs | new allocs |
| ---: | ---: | ---: | ---: | ---: | ---: |
| 1 | 65.92 us | 59.86 us | ~ | 7 | 5 |
| 8 | 48.58 us | 51.12 us | ~ | 15 | 6 |
| 64 | 64.33 us | 64.18 us | ~ | 71 | 6 |
| 1,024 | 154.88 us | 97.89 us | -36.80% | 1,031 | 6 |
| 8,192 | 571.7 us | 214.5 us | -62.48% | 8,199 | 6 |
At 1,024 chunks, B/op drops by 14.22%. At 8,192 chunks, it drops by
52.06%.
## Tests
- `go test ./...`
---
arrow/array/concat.go | 20 +++++++++-
arrow/array/concat_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 110 insertions(+), 1 deletion(-)
diff --git a/arrow/array/concat.go b/arrow/array/concat.go
index 974df418..3a9efe9e 100644
--- a/arrow/array/concat.go
+++ b/arrow/array/concat.go
@@ -172,6 +172,24 @@ func concatBuffers(bufs []*memory.Buffer, mem
memory.Allocator) *memory.Buffer {
return out
}
+func concatFixedWidthBuffers(data []arrow.ArrayData, idx, byteWidth, length
int, mem memory.Allocator) *memory.Buffer {
+ out := memory.NewResizableBuffer(mem)
+ out.Resize(length * byteWidth)
+ dst := out.Bytes()
+ for _, d := range data {
+ buf := d.Buffers()[idx]
+ if buf == nil {
+ continue
+ }
+
+ begin := d.Offset() * byteWidth
+ nbytes := d.Len() * byteWidth
+ copy(dst, buf.Bytes()[begin:begin+nbytes])
+ dst = dst[nbytes:]
+ }
+ return out
+}
+
func handle32BitOffsets(outLen int, buffers []*memory.Buffer, out
*memory.Buffer) (*memory.Buffer, []rng, error) {
dst := arrow.Int32Traits.CastFromBytes(out.Bytes())
valuesRanges := make([]rng, len(buffers))
@@ -598,7 +616,7 @@ func concat(data []arrow.ArrayData, mem memory.Allocator)
(arr arrow.ArrayData,
return nil, err
}
case arrow.FixedWidthDataType:
- out.buffers[1] =
concatBuffers(gatherBuffersFixedWidthType(data, 1, dt), mem)
+ out.buffers[1] = concatFixedWidthBuffers(data, 1,
dt.BitWidth()/8, out.length, mem)
case arrow.BinaryViewDataType:
out.buffers = out.buffers[:2]
for _, d := range data {
diff --git a/arrow/array/concat_test.go b/arrow/array/concat_test.go
index 5b595e7c..df906bd9 100644
--- a/arrow/array/concat_test.go
+++ b/arrow/array/concat_test.go
@@ -99,6 +99,97 @@ func TestConcatenate(t *testing.T) {
}
}
+func BenchmarkConcatenateFixedWidth(b *testing.B) {
+ mem := memory.NewGoAllocator()
+
+ const totalValues = 1 << 16
+ values := make([]int64, totalValues)
+ builder := array.NewInt64Builder(mem)
+ builder.AppendValues(values, nil)
+ backing := builder.NewInt64Array()
+ builder.Release()
+ defer backing.Release()
+
+ for _, chunkCount := range []int{1, 8, 64, 1024, 8192} {
+ b.Run(fmt.Sprintf("chunks=%d", chunkCount), func(b *testing.B) {
+ chunkSize := totalValues / chunkCount
+ inputs := make([]arrow.Array, chunkCount)
+ for i := range inputs {
+ begin := int64(i * chunkSize)
+ inputs[i] = array.NewSlice(backing, begin,
begin+int64(chunkSize))
+ }
+ defer func() {
+ for _, input := range inputs {
+ input.Release()
+ }
+ }()
+
+ b.SetBytes(int64(totalValues * arrow.Int64SizeBytes))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ result, err := array.Concatenate(inputs, mem)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if result.Len() != totalValues {
+ b.Fatalf("result length = %d, want %d",
result.Len(), totalValues)
+ }
+ result.Release()
+ }
+ })
+ }
+}
+
+func TestConcatenateFixedWidthSlices(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ tests := []struct {
+ name string
+ build func(memory.Allocator) arrow.Array
+ }{
+ {"int32", func(mem memory.Allocator) arrow.Array {
+ builder := array.NewInt32Builder(mem)
+ builder.AppendValues([]int32{10, 20, 30, 40, 50}, nil)
+ result := builder.NewInt32Array()
+ builder.Release()
+ return result
+ }},
+ {"int64", func(mem memory.Allocator) arrow.Array {
+ builder := array.NewInt64Builder(mem)
+ builder.AppendValues([]int64{10, 20, 30, 40, 50}, nil)
+ result := builder.NewInt64Array()
+ builder.Release()
+ return result
+ }},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ backing := tt.build(mem)
+ defer backing.Release()
+
+ inputs := []arrow.Array{
+ array.NewSlice(backing, 1, 1),
+ array.NewSlice(backing, 2, 4),
+ array.NewSlice(backing, 4, 5),
+ }
+ for _, input := range inputs {
+ defer input.Release()
+ }
+
+ result, err := array.Concatenate(inputs, mem)
+ require.NoError(t, err)
+ defer result.Release()
+
+ expected := array.NewSlice(backing, 2, 5)
+ defer expected.Release()
+ assert.True(t, array.Equal(expected, result))
+ })
+ }
+}
+
type ConcatTestSuite struct {
suite.Suite
