This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new eda27cd1 perf(parquet): use typed memo insertion for fixed-width
dictionaries (#1312)
eda27cd1 is described below
commit eda27cd162dd2e92ba23be2c0cc00dcbd29eeddc
Author: Minh Vu <[email protected]>
AuthorDate: Tue Sep 15 20:57:56 2026 +0200
perf(parquet): use typed memo insertion for fixed-width dictionaries (#1312)
## What changed
- Use BinaryMemoTable.InsertOrGet for FLBA dictionary Put and
PutDictionary.
- Use the same typed insertion path for DictInt96Encoder.Put.
- Keep empty Put calls as no-ops.
- Add Int96 dictionary round-trip coverage and FLBA/Int96 benchmarks.
## Why
These encoders already use the binary memo table, but still called
GetOrInsert through the generic interface for every value. The typed
entry point avoids the extra interface conversion. Dictionary order and
encoded output stay unchanged.
## Benchmark
Apple M1 Pro, macOS arm64, Go 1.26.3. 65,535 values and 100 unique
values, median of five runs.
| Case | Before | After | Change |
| --- | ---: | ---: | ---: |
| FLBA, width 16 | 4.59 ms | 3.09 ms | 33% faster |
| Int96 | 5.41 ms | 3.87 ms | 29% faster |
| FLBA allocations | 131,117 | 65,581 | 50% fewer |
| Int96 allocations | 196,655 | 131,118 | 33% fewer |
Allocated bytes also fell by about 42% for FLBA and 33% for Int96.
## Tests
- go test ./... with Arrow and Parquet test data
- go test ./parquet/internal/encoding/...
- go test -race ./parquet/internal/encoding/...
- go vet ./parquet/internal/encoding/...
- go test -tags noasm ./parquet/internal/encoding/...
---
.../internal/encoding/encoding_benchmarks_test.go | 69 ++++++++++++++++++++++
parquet/internal/encoding/encoding_test.go | 1 +
.../encoding/fixed_len_byte_array_encoder.go | 10 +++-
parquet/internal/encoding/typed_encoder.go | 7 ++-
4 files changed, 84 insertions(+), 3 deletions(-)
diff --git a/parquet/internal/encoding/encoding_benchmarks_test.go
b/parquet/internal/encoding/encoding_benchmarks_test.go
index 43bcc4ea..ab650dc1 100644
--- a/parquet/internal/encoding/encoding_benchmarks_test.go
+++ b/parquet/internal/encoding/encoding_benchmarks_test.go
@@ -424,6 +424,75 @@ func BenchmarkEncodeDictByteArray(b *testing.B) {
}
}
+func BenchmarkEncodeDictFixedLenByteArray(b *testing.B) {
+ const (
+ nunique = 100
+ nvalues = 65535
+ )
+
+ for _, width := range []int{8, 16, 32} {
+ b.Run(fmt.Sprintf("width=%d", width), func(b *testing.B) {
+ values := make([]parquet.FixedLenByteArray, nvalues)
+ for i := range values {
+ value := make([]byte, width)
+ unique := uint32(i % nunique)
+ for j := 0; j < 4; j++ {
+ value[j] = byte(unique >> uint(8*j))
+ }
+ for j := 4; j < width; j++ {
+ value[j] = byte(j)
+ }
+ values[i] = value
+ }
+
+ col := schema.NewColumn(schema.NewFixedLenByteArrayNode(
+ "fixedlenbytearray",
parquet.Repetitions.Required, int32(width), -1), 0, 0)
+ b.SetBytes(int64(nvalues * width))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ enc :=
encoding.NewEncoder(parquet.Types.FixedLenByteArray,
parquet.Encodings.PlainDict,
+ true, col,
memory.DefaultAllocator).(*encoding.DictFixedLenByteArrayEncoder)
+ enc.Put(values)
+ buf, err := enc.FlushValues()
+ if err != nil {
+ b.Fatal(err)
+ }
+ buf.Release()
+ enc.Release()
+ }
+ })
+ }
+}
+
+func BenchmarkEncodeDictInt96(b *testing.B) {
+ const (
+ nunique = 100
+ nvalues = 65535
+ )
+
+ values := make([]parquet.Int96, nvalues)
+ for i := range values {
+ values[i] = parquet.NewInt96([3]uint32{uint32(i % nunique), 0,
0})
+ }
+ col := schema.NewColumn(schema.NewInt96Node("int96",
parquet.Repetitions.Required, -1), 0, 0)
+
+ b.SetBytes(int64(nvalues * parquet.Int96SizeBytes))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ enc := encoding.NewEncoder(parquet.Types.Int96,
parquet.Encodings.PlainDict,
+ true, col,
memory.DefaultAllocator).(*encoding.DictInt96Encoder)
+ enc.Put(values)
+ buf, err := enc.FlushValues()
+ if err != nil {
+ b.Fatal(err)
+ }
+ buf.Release()
+ enc.Release()
+ }
+}
+
func benchmarkEncodeDictNumeric[T int32 | int64 | float32 | float64](b
*testing.B, typ parquet.Type, col *schema.Column, valueSize int64) {
const (
nunique = 100
diff --git a/parquet/internal/encoding/encoding_test.go
b/parquet/internal/encoding/encoding_test.go
index 1589c133..27ce54a2 100644
--- a/parquet/internal/encoding/encoding_test.go
+++ b/parquet/internal/encoding/encoding_test.go
@@ -599,6 +599,7 @@ func TestDictEncoding(t *testing.T) {
{"Int64", reflect.TypeOf(int64(0))},
{"Float32", reflect.TypeOf(float32(0))},
{"Float64", reflect.TypeOf(float64(0))},
+ {"Int96", reflect.TypeOf(parquet.Int96{})},
{"ByteArray", reflect.TypeOf(parquet.ByteArray{})},
{"FixedLenByteArray",
reflect.TypeOf(parquet.FixedLenByteArray{})},
}
diff --git a/parquet/internal/encoding/fixed_len_byte_array_encoder.go
b/parquet/internal/encoding/fixed_len_byte_array_encoder.go
index cd3e0a84..26446e84 100644
--- a/parquet/internal/encoding/fixed_len_byte_array_encoder.go
+++ b/parquet/internal/encoding/fixed_len_byte_array_encoder.go
@@ -177,8 +177,13 @@ func (enc *DictFixedLenByteArrayEncoder) WriteDict(out
[]byte) {
// Put writes fixed length values to a dictionary encoded column
func (enc *DictFixedLenByteArrayEncoder) Put(in []parquet.FixedLenByteArray) {
+ if len(in) == 0 {
+ return
+ }
+
+ memo := enc.memo.(BinaryMemoTable)
for _, v := range in {
- memoIdx, found, err := enc.memo.GetOrInsert(v)
+ memoIdx, found, err := memo.InsertOrGet(v)
if err != nil {
panic(err)
}
@@ -241,8 +246,9 @@ func (enc *DictFixedLenByteArrayEncoder)
PutDictionary(values arrow.Array) error
enc.dictEncodedSize += enc.typeLen * values.Len()
data :=
values.Data().Buffers()[1].Bytes()[values.Data().Offset()*enc.typeLen:]
+ memo := enc.memo.(BinaryMemoTable)
for i := 0; i < values.Len(); i++ {
- _, _, err := enc.memo.GetOrInsert(data[i*enc.typeLen :
(i+1)*enc.typeLen])
+ _, _, err := memo.InsertOrGet(data[i*enc.typeLen :
(i+1)*enc.typeLen])
if err != nil {
return err
}
diff --git a/parquet/internal/encoding/typed_encoder.go
b/parquet/internal/encoding/typed_encoder.go
index 735ee925..b4861e08 100644
--- a/parquet/internal/encoding/typed_encoder.go
+++ b/parquet/internal/encoding/typed_encoder.go
@@ -462,8 +462,13 @@ func (enc *DictInt96Encoder) WriteDict(out []byte) {
// Put encodes the values passed in, adding to the index as needed
func (enc *DictInt96Encoder) Put(in []parquet.Int96) {
+ if len(in) == 0 {
+ return
+ }
+
+ memo := enc.memo.(BinaryMemoTable)
for _, v := range in {
- memoIdx, found, err := enc.memo.GetOrInsert(v[:])
+ memoIdx, found, err := memo.InsertOrGet(v[:])
if err != nil {
panic(err)
}
