This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new a00312ac perf(parquet): avoid scanning plain byte-array offsets (#1362)
a00312ac is described below
commit a00312ac02beb07af6e22276b71c91b3f0b89ea2
Author: Minh Vu <[email protected]>
AuthorDate: Tue Oct 13 16:28:11 2026 +0200
perf(parquet): avoid scanning plain byte-array offsets (#1362)
### What
Avoid scanning PLAIN Arrow byte-array offsets to compute encoded size.
### Why
The payload size is the last offset minus the first offset, so the
pre-pass is redundant.
### Implementation
Compute payload bytes from boundary offsets and add four bytes per
value.
### Benchmark
At 65,536 values on AMD EPYC, median time before → after:
- 0-byte values: 281.0 → 244.4 µs (-13.0%)
- 4-byte values: 296.8 → 275.2 µs (-7.3%)
- 16-byte values: 332.8 → 301.5 µs (-9.4%)
- 64-byte values: 375.5 → 335.2 µs (-10.7%)
- Allocations: 0 → 0
---
.../internal/encoding/byte_array_arrow_encoder.go | 9 +-
.../encoding/byte_array_arrow_encoder_test.go | 146 +++++++++++++++++++++
2 files changed, 150 insertions(+), 5 deletions(-)
diff --git a/parquet/internal/encoding/byte_array_arrow_encoder.go
b/parquet/internal/encoding/byte_array_arrow_encoder.go
index b5253a55..c9ec4b8f 100644
--- a/parquet/internal/encoding/byte_array_arrow_encoder.go
+++ b/parquet/internal/encoding/byte_array_arrow_encoder.go
@@ -32,14 +32,13 @@ func putArrowPlain[T arrowByteArrayOffset](sink
*PooledBufferWriter, values []by
return
}
- encodedSize := 0
- for i := 0; i < len(offsets)-1; i++ {
- encodedSize += int(offsets[i+1]-offsets[i]) + 4
- }
+ numValues := len(offsets) - 1
+ // Slices and valid runs can start at a nonzero offset in the data
buffer.
+ encodedSize := int(offsets[numValues]-offsets[0]) + 4*numValues
sink.Reserve(encodedSize)
out := sink.buf.Buf()[sink.pos : sink.pos+encodedSize]
- for i := 0; i < len(offsets)-1; i++ {
+ for i := 0; i < numValues; i++ {
start := int(offsets[i])
end := int(offsets[i+1])
binary.LittleEndian.PutUint32(out, uint32(end-start))
diff --git a/parquet/internal/encoding/byte_array_arrow_encoder_test.go
b/parquet/internal/encoding/byte_array_arrow_encoder_test.go
index 949d5e65..30a7e528 100644
--- a/parquet/internal/encoding/byte_array_arrow_encoder_test.go
+++ b/parquet/internal/encoding/byte_array_arrow_encoder_test.go
@@ -124,6 +124,152 @@ func TestByteArrayArrowEncodersMatchByteArrayInput(t
*testing.T) {
}
}
+func TestPlainByteArrayArrowOffsets(t *testing.T) {
+ t.Run("int32", testPlainByteArrayArrowOffsets[int32])
+ t.Run("int64", testPlainByteArrayArrowOffsets[int64])
+}
+
+func testPlainByteArrayArrowOffsets[T arrowByteArrayOffset](t *testing.T) {
+ t.Helper()
+ values, data, offsets32, _ := arrowByteArrayInput()
+ offsets := make([]T, len(offsets32))
+ for i, offset := range offsets32 {
+ offsets[i] = T(offset)
+ }
+ for _, tc := range []struct {
+ name string
+ offsets []T
+ validBits []byte
+ validBitsOffset int64
+ want []parquet.ByteArray
+ }{
+ {name: "nil"},
+ {name: "empty-slice", offsets: offsets[3:4]},
+ {name: "empty-values", offsets: []T{offsets[2], offsets[2],
offsets[2]}, want: []parquet.ByteArray{{}, {}}},
+ {name: "sliced", offsets: offsets[1:7], want: values[1:6]},
+ {name: "whole", offsets: offsets, want: values},
+ {
+ name: "spaced-slice", offsets: offsets[1:],
+ validBits: []byte{0b01011000, 0}, validBitsOffset: 3,
+ want: []parquet.ByteArray{values[1], values[2],
values[4]},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ enc := NewEncoder(parquet.Types.ByteArray,
parquet.Encodings.Plain, false, nil,
memory.DefaultAllocator).(*PlainByteArrayEncoder)
+ defer enc.Release()
+ enc.PutByteArray([]byte("before"))
+ want := []parquet.ByteArray{[]byte("before")}
+ for range 2 {
+ putArrowPlainSpaced(enc.sink, data, tc.offsets,
tc.validBits, tc.validBitsOffset)
+ want = append(want, tc.want...)
+ }
+ enc.PutByteArray([]byte("after"))
+ want = append(want, []byte("after"))
+ got, err := enc.FlushValues()
+ require.NoError(t, err)
+ defer got.Release()
+ require.Equal(t, encodedByteArrays(t,
parquet.Encodings.Plain, want, false, nil, 0), got.Bytes())
+ })
+ }
+}
+
+// Check that the total-span size calculation preserves the plain encoding
+// for variable-width values, sliced offsets, and disjoint valid runs.
+func TestPlainByteArrayArrowVariableLengthSlices(t *testing.T) {
+ t.Run("int32", testPlainByteArrayArrowVariableLengthSlices[int32])
+ t.Run("int64", testPlainByteArrayArrowVariableLengthSlices[int64])
+}
+
+func testPlainByteArrayArrowVariableLengthSlices[T arrowByteArrayOffset](t
*testing.T) {
+ t.Helper()
+ const nvalues = 129
+ values := make([]parquet.ByteArray, nvalues)
+ data := []byte("prefix outside the first value")
+ offsets := make([]T, nvalues+1)
+ for i := range values {
+ offsets[i] = T(len(data))
+ values[i] = make([]byte, (i*i*17+i*3)%57)
+ for j := range values[i] {
+ values[i][j] = byte(i + j)
+ }
+ data = append(data, values[i]...)
+ }
+ offsets[nvalues] = T(len(data))
+
+ for _, span := range []struct {
+ name string
+ begin, end int
+ }{
+ {"whole", 0, nvalues},
+ {"middle", 9, 93},
+ {"single", 63, 64},
+ {"empty", 48, 48},
+ {"tail", nvalues - 12, nvalues},
+ } {
+ for _, selection := range []string{"all", "sparse",
"long-runs", "none"} {
+ t.Run(span.name+"/"+selection, func(t *testing.T) {
+ var validBits []byte
+ if selection != "all" {
+ validBits = make([]byte,
bitutil.BytesForBits(int64(span.end-span.begin+3)))
+ }
+ want := []parquet.ByteArray{[]byte("before")}
+ for i := span.begin; i < span.end; i++ {
+ valid := selection == "all" ||
+ selection == "sparse" && i%11 <
3 ||
+ selection == "long-runs" &&
(i/16)%2 == 0
+ if valid {
+ want = append(want, values[i])
+ if validBits != nil {
+
bitutil.SetBit(validBits, 3+i-span.begin)
+ }
+ }
+ }
+ want = append(want, []byte("after"))
+
+ enc := NewEncoder(parquet.Types.ByteArray,
parquet.Encodings.Plain, false, nil,
memory.DefaultAllocator).(*PlainByteArrayEncoder)
+ defer enc.Release()
+ enc.PutByteArray([]byte("before"))
+ putArrowPlainSpaced(enc.sink, data,
offsets[span.begin:span.end+1], validBits, 3)
+ enc.PutByteArray([]byte("after"))
+ got, err := enc.FlushValues()
+ require.NoError(t, err)
+ defer got.Release()
+ require.Equal(t, encodedByteArrays(t,
parquet.Encodings.Plain, want, false, nil, 0), got.Bytes())
+ })
+ }
+ }
+}
+
+func BenchmarkPlainByteArrayPutArrow(b *testing.B) {
+ b.Run("int32", benchmarkPlainByteArrayPutArrow[int32])
+ b.Run("int64", benchmarkPlainByteArrayPutArrow[int64])
+}
+
+func benchmarkPlainByteArrayPutArrow[T arrowByteArrayOffset](b *testing.B) {
+ for _, nvalues := range []int{64, 65536} {
+ for _, width := range []int{0, 4, 16, 64} {
+ b.Run(fmt.Sprintf("values=%d/bytes=%d", nvalues,
width), func(b *testing.B) {
+ data := make([]byte, 4+nvalues*width)
+ offsets := make([]T, nvalues+1)
+ for i := range offsets {
+ offsets[i] = T(4 + i*width)
+ }
+ enc := NewEncoder(parquet.Types.ByteArray,
parquet.Encodings.Plain, false, nil,
memory.DefaultAllocator).(*PlainByteArrayEncoder)
+ defer enc.Release()
+ enc.sink.Reserve(nvalues * (4 + width))
+ b.SetBytes(int64(nvalues * (4 + width)))
+ b.ReportAllocs()
+ for b.Loop() {
+ // Reuse capacity to isolate encoding
from result-buffer allocation.
+ enc.sink.pos = 0
+ putArrowPlain(enc.sink, data, offsets)
+ }
+ b.ReportMetric(float64(nvalues), "values/op")
+ })
+ }
+ }
+}
+
func TestDeltaByteArrayArrowEncoderPreservesStateAcrossBatches(t *testing.T) {
values, data, offsets32, offsets64 := arrowByteArrayInput()
