This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 68c7223d perf(parquet): reuse boolean spaced scratch (#1231)
68c7223d is described below
commit 68c7223dcff880f81e27a8c2a9954de50f46dff2
Author: Minh Vu <[email protected]>
AuthorDate: Fri Aug 28 19:24:08 2026 +0200
perf(parquet): reuse boolean spaced scratch (#1231)
## What
- Reuse a `[]bool` scratch buffer in `PutSpaced` for Plain and RLE
boolean encoders.
- Reuse the same scratch for RLE `PutSpacedBitmap` writes.
- Keep the scratch capacity across repeated page writes.
## Benchmark
Apple M1 Pro, Go 1.26.3. The same encoder is reused, and scratch growth
happens before the timed loop.
Command: `go test -vet=off ./parquet/internal/encoding -run "^$" -bench
"^Benchmark(Plain|Rle)BooleanEncoderPutSpaced$" -benchmem
-benchtime=300ms -count=3 -cpu=1`
All-valid input, before vs after:
| Encoder | Values | Before | After |
| --- | ---: | ---: | ---: |
| Plain | 1,024 | 1,024 B/op, 1 alloc | 0 B/op, 0 allocs |
| Plain | 65,536 | 65,539 B/op, 1 alloc | 0 B/op, 0 allocs |
| Plain | 1,048,576 | 1,048,620 B/op, 1 alloc | 0 B/op, 0 allocs |
| RLE | 1,024 | 1,232 B/op, 4 allocs | 208 B/op, 3 allocs |
| RLE | 65,536 | 65,747 B/op, 4 allocs | 208 B/op, 3 allocs |
## Tests
- `go test ./parquet/internal/encoding -count=1`
- `PARQUET_TEST_DATA=parquet-testing/data go test ./... -count=1`
---
parquet/internal/encoding/boolean_encoder.go | 37 ++++++---
.../boolean_encoder_spaced_benchmark_test.go | 96 ++++++++++++++++++++++
2 files changed, 122 insertions(+), 11 deletions(-)
diff --git a/parquet/internal/encoding/boolean_encoder.go
b/parquet/internal/encoding/boolean_encoder.go
index a4111804..0f7fcc8b 100644
--- a/parquet/internal/encoding/boolean_encoder.go
+++ b/parquet/internal/encoding/boolean_encoder.go
@@ -35,8 +35,9 @@ const (
// PlainBooleanEncoder encodes bools as a bitmap as per the Plain Encoding
type PlainBooleanEncoder struct {
encoder
- bitsBuffer []byte
- wr utils.BitmapWriter
+ bitsBuffer []byte
+ spacedScratch []bool
+ wr utils.BitmapWriter
}
// Type for the PlainBooleanEncoder is parquet.Types.Boolean
@@ -113,9 +114,13 @@ func (enc *PlainBooleanEncoder) putBitmapScalar(bitmap
[]byte, offset, length in
// PutSpaced will use the validBits bitmap to determine which values are nulls
// and can be left out from the slice, and the encoded without those nulls.
func (enc *PlainBooleanEncoder) PutSpaced(in []bool, validBits []byte,
validBitsOffset int64) {
- bufferOut := make([]bool, len(in))
- nvalid := spacedCompress(in, bufferOut, validBits, validBitsOffset)
- enc.Put(bufferOut[:nvalid])
+ if cap(enc.spacedScratch) < len(in) {
+ enc.spacedScratch = make([]bool, len(in))
+ } else {
+ enc.spacedScratch = enc.spacedScratch[:len(in)]
+ }
+ nvalid := spacedCompress(in, enc.spacedScratch, validBits,
validBitsOffset)
+ enc.Put(enc.spacedScratch[:nvalid])
}
// PutSpacedBitmap encodes boolean values directly from a bitmap with validity
information,
@@ -186,6 +191,7 @@ type RleBooleanEncoder struct {
encoder
bufferedValues []bool
+ spacedScratch []bool
}
func (RleBooleanEncoder) Type() parquet.Type {
@@ -197,9 +203,13 @@ func (enc *RleBooleanEncoder) Put(in []bool) {
}
func (enc *RleBooleanEncoder) PutSpaced(in []bool, validBits []byte,
validBitsOffset int64) {
- bufferOut := make([]bool, len(in))
- nvalid := spacedCompress(in, bufferOut, validBits, validBitsOffset)
- enc.Put(bufferOut[:nvalid])
+ if cap(enc.spacedScratch) < len(in) {
+ enc.spacedScratch = make([]bool, len(in))
+ } else {
+ enc.spacedScratch = enc.spacedScratch[:len(in)]
+ }
+ nvalid := spacedCompress(in, enc.spacedScratch, validBits,
validBitsOffset)
+ enc.Put(enc.spacedScratch[:nvalid])
}
// PutSpacedBitmap encodes boolean values from a bitmap with validity
information.
@@ -219,7 +229,12 @@ func (enc *RleBooleanEncoder) PutSpacedBitmap(bitmap
[]byte, bitmapOffset int64,
// Extract valid bits to []bool for buffering
// Use SetBitRunReader to efficiently iterate over valid values
- bufferOut := make([]bool, numValid)
+ nvalid := int(numValid)
+ if cap(enc.spacedScratch) < nvalid {
+ enc.spacedScratch = make([]bool, nvalid)
+ } else {
+ enc.spacedScratch = enc.spacedScratch[:nvalid]
+ }
idx := 0
reader := bitutils.NewSetBitRunReader(validBits, validBitsOffset,
numValues)
@@ -231,12 +246,12 @@ func (enc *RleBooleanEncoder) PutSpacedBitmap(bitmap
[]byte, bitmapOffset int64,
// Convert this run of bits to bools
for i := int64(0); i < run.Length; i++ {
- bufferOut[idx] = bitutil.BitIsSet(bitmap,
int(bitmapOffset+run.Pos+i))
+ enc.spacedScratch[idx] = bitutil.BitIsSet(bitmap,
int(bitmapOffset+run.Pos+i))
idx++
}
}
- enc.Put(bufferOut)
+ enc.Put(enc.spacedScratch[:idx])
return numValid
}
diff --git a/parquet/internal/encoding/boolean_encoder_spaced_benchmark_test.go
b/parquet/internal/encoding/boolean_encoder_spaced_benchmark_test.go
new file mode 100644
index 00000000..092b0c09
--- /dev/null
+++ b/parquet/internal/encoding/boolean_encoder_spaced_benchmark_test.go
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encoding
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow/bitutil"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+)
+
+type booleanSpacedEncoder interface {
+ TypedEncoder
+ PutSpaced([]bool, []byte, int64)
+}
+
+func BenchmarkPlainBooleanEncoderPutSpaced(b *testing.B) {
+ benchmarkBooleanEncoderPutSpaced(b, parquet.Encodings.Plain)
+}
+
+func BenchmarkRleBooleanEncoderPutSpaced(b *testing.B) {
+ benchmarkBooleanEncoderPutSpaced(b, parquet.Encodings.RLE)
+}
+
+func benchmarkBooleanEncoderPutSpaced(b *testing.B, enc parquet.Encoding) {
+ patterns := []struct {
+ name string
+ valid func(int) bool
+ }{
+ {name: "all_valid", valid: func(int) bool { return true }},
+ {name: "ten_percent_null", valid: func(i int) bool { return
i%10 != 0 }},
+ {name: "fifty_percent_null", valid: func(i int) bool { return
i%2 != 0 }},
+ {name: "ninety_percent_null", valid: func(i int) bool { return
i%10 == 0 }},
+ }
+
+ for _, length := range []int{1024, 64 * 1024, 1024 * 1024} {
+ for _, pattern := range patterns {
+ b.Run(fmt.Sprintf("length_%d/%s", length,
pattern.name), func(b *testing.B) {
+ values := make([]bool, length)
+ for i := range values {
+ values[i] = i%3 == 0
+ }
+ validBits := make([]byte,
bitutil.BytesForBits(int64(length)))
+ for i := range length {
+ if pattern.valid(i) {
+ bitutil.SetBit(validBits, i)
+ }
+ }
+
+ encoder := NewEncoder(
+ parquet.Types.Boolean, enc, false, nil,
memory.DefaultAllocator,
+ ).(booleanSpacedEncoder)
+ defer encoder.Release()
+
+ encoder.PutSpaced(values, validBits, 0)
+ flushBooleanEncoder(b, encoder)
+
+ b.ReportAllocs()
+ b.SetBytes(int64(length))
+ b.ResetTimer()
+ for b.Loop() {
+ encoder.PutSpaced(values, validBits, 0)
+ flushBooleanEncoder(b, encoder)
+ }
+ })
+ }
+ }
+}
+
+func flushBooleanEncoder(b *testing.B, encoder booleanSpacedEncoder) {
+ buf, err := encoder.FlushValues()
+ if err != nil {
+ b.Fatal(err)
+ }
+ buf.Release()
+
+ if enc, ok := encoder.(*RleBooleanEncoder); ok {
+ enc.bufferedValues = enc.bufferedValues[:0]
+ }
+}
