This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new f7bacb41 fix: repair failing benchmarks and stop bench.sh masking
failures (#938)
f7bacb41 is described below
commit f7bacb41a611786f0989dbf0fdb104a518213ea2
Author: Matt Topol <[email protected]>
AuthorDate: Wed Jul 15 12:32:47 2026 -0400
fix: repair failing benchmarks and stop bench.sh masking failures (#938)
---
.github/workflows/benchmark.yml | 2 +-
arrow/array/builder_prealloc_bench_test.go | 5 +++--
arrow/compute/vector_run_end_test.go | 12 ++++++++++--
ci/scripts/bench.sh | 2 +-
.../internal/encoding/encoding_benchmarks_test.go | 18 ++++++++++++------
parquet/metadata/bloom_filter_block.go | 6 ++++++
parquet/metadata/bloom_filter_block_amd64.go | 2 ++
parquet/metadata/bloom_filter_block_default.go | 2 +-
parquet/metadata/bloom_filter_test.go | 21 +++++++++++++++++++++
9 files changed, 57 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 4b1fad9a..945fe2a4 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -62,5 +62,5 @@ jobs:
CONBENCH_REF: ${{ github.ref_name }}
CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
run: |
- python3 -m pip install
benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
+ python3 -m pip install
benchadapt@git+https://github.com/conbench/conbench.git@3af4a55206ad3918762cc8dd7d3012eadbe96a54#subdirectory=benchadapt/python
python3 ci/scripts/bench_adapt.py
diff --git a/arrow/array/builder_prealloc_bench_test.go
b/arrow/array/builder_prealloc_bench_test.go
index 813e1e2c..e2296590 100644
--- a/arrow/array/builder_prealloc_bench_test.go
+++ b/arrow/array/builder_prealloc_bench_test.go
@@ -19,6 +19,7 @@ package array_test
import (
"testing"
+ "github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/memory"
)
@@ -220,7 +221,7 @@ func BenchmarkStringBuilder_NoReserveData(b *testing.B) {
// BenchmarkBinaryBuilder_LargeData tests large binary data
func BenchmarkBinaryBuilder_LargeData(b *testing.B) {
mem := memory.NewGoAllocator()
- builder := array.NewBinaryBuilder(mem, nil)
+ builder := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
defer builder.Release()
// 1MB per element
@@ -251,7 +252,7 @@ func BenchmarkBinaryBuilder_LargeData_WithReserve(b
*testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
- builder := array.NewBinaryBuilder(mem, nil)
+ builder := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
builder.Reserve(1)
builder.ReserveData(dataSize)
b.StartTimer()
diff --git a/arrow/compute/vector_run_end_test.go
b/arrow/compute/vector_run_end_test.go
index 7f0d4cec..90d9e279 100644
--- a/arrow/compute/vector_run_end_test.go
+++ b/arrow/compute/vector_run_end_test.go
@@ -412,8 +412,16 @@ func BenchmarkRunEndKernels(b *testing.B) {
b.Run("run_ends_type="+runEndType.dt.String(),
func(b *testing.B) {
for _, valType := range
append(numericTypes, arrow.BinaryTypes.String, arrow.FixedWidthTypes.Boolean) {
b.Run("value_type="+valType.String(), func(b *testing.B) {
- benchRunEndEncode(b,
sz, a.nullProb, runEndType.dt, valType)
- benchRunEndDecode(b,
sz, a.nullProb, runEndType.dt, valType)
+ vsz := sz
+ // String uses int32
value offsets; cap elements so a
+ // cache-sized run
can't overflow them (was a panic).
+ if valType.ID() ==
arrow.STRING {
+ if maxElems :=
math.MaxInt32 / 32; vsz > maxElems {
+ vsz =
maxElems
+ }
+ }
+ benchRunEndEncode(b,
vsz, a.nullProb, runEndType.dt, valType)
+ benchRunEndDecode(b,
vsz, a.nullProb, runEndType.dt, valType)
})
}
})
diff --git a/ci/scripts/bench.sh b/ci/scripts/bench.sh
index f3fef829..a472b673 100644
--- a/ci/scripts/bench.sh
+++ b/ci/scripts/bench.sh
@@ -21,7 +21,7 @@
# as the second argument, it will create a file "bench_stats.json"
# in the directory this is called from containing a json representation
-set -ex
+set -exo pipefail
# Validate input arguments
if [ -z "$1" ]; then
diff --git a/parquet/internal/encoding/encoding_benchmarks_test.go
b/parquet/internal/encoding/encoding_benchmarks_test.go
index b53d8b19..ca0d7de7 100644
--- a/parquet/internal/encoding/encoding_benchmarks_test.go
+++ b/parquet/internal/encoding/encoding_benchmarks_test.go
@@ -797,13 +797,19 @@ func BenchmarkMemoTableHighCardinality(b *testing.B) {
for _, tt := range tests {
b.Run(fmt.Sprintf("%d unique n %d", tt.nunique, tt.nvalues),
func(b *testing.B) {
- rag := testutils.NewRandomArrayGenerator(0)
- dict := rag.Int32(int64(tt.nunique), 0,
math.MaxInt32-1, 0)
- indices := rag.Int32(tt.nvalues, 0,
int32(tt.nunique)-1, 0)
-
+ // Random generation does not guarantee uniqueness, so
seed the first
+ // nunique slots with distinct values (satisfying the
exact
+ // tbl.Size() == nunique assertion) and back-fill the
rest randomly.
values := make([]int32, tt.nvalues)
- for idx := range values {
- values[idx] =
dict.Value(int(indices.Value(idx)))
+ for i := int32(0); i < tt.nunique; i++ {
+ values[i] = i
+ }
+ if extra := tt.nvalues - int64(tt.nunique); extra > 0 {
+ rag := testutils.NewRandomArrayGenerator(0)
+ indices := rag.Int32(extra, 0, tt.nunique-1, 0)
+ for i := int64(0); i < extra; i++ {
+ values[int64(tt.nunique)+i] =
indices.Value(int(i))
+ }
}
b.ResetTimer()
b.Run("xxh3", func(b *testing.B) {
diff --git a/parquet/metadata/bloom_filter_block.go
b/parquet/metadata/bloom_filter_block.go
index bbd84267..7a161aa3 100644
--- a/parquet/metadata/bloom_filter_block.go
+++ b/parquet/metadata/bloom_filter_block.go
@@ -51,3 +51,9 @@ func insertBulkGo(bitset32 []uint32, hash []uint64) {
insertHash(bitset32, h)
}
}
+
+func checkBulkGo(bitset32 []uint32, hash []uint64, results []bool) {
+ for i, h := range hash {
+ results[i] = checkHash(bitset32, h)
+ }
+}
diff --git a/parquet/metadata/bloom_filter_block_amd64.go
b/parquet/metadata/bloom_filter_block_amd64.go
index 463e6967..8e709f9a 100644
--- a/parquet/metadata/bloom_filter_block_amd64.go
+++ b/parquet/metadata/bloom_filter_block_amd64.go
@@ -33,4 +33,6 @@ func init() {
checkHash = checkHashGo
insertHash, insertBulk = insertHashGo, insertBulkGo
}
+ // checkBulk has no vectorized variant; it wraps the selected checkHash.
+ checkBulk = checkBulkGo
}
diff --git a/parquet/metadata/bloom_filter_block_default.go
b/parquet/metadata/bloom_filter_block_default.go
index ca1ec750..f7e74200 100644
--- a/parquet/metadata/bloom_filter_block_default.go
+++ b/parquet/metadata/bloom_filter_block_default.go
@@ -19,5 +19,5 @@
package metadata
func init() {
- checkHash, insertHash, insertBulk = checkHashGo, insertHashGo,
insertBulkGo
+ checkHash, checkBulk, insertHash, insertBulk = checkHashGo,
checkBulkGo, insertHashGo, insertBulkGo
}
diff --git a/parquet/metadata/bloom_filter_test.go
b/parquet/metadata/bloom_filter_test.go
index 86126502..1eb43287 100644
--- a/parquet/metadata/bloom_filter_test.go
+++ b/parquet/metadata/bloom_filter_test.go
@@ -161,6 +161,27 @@ func BenchmarkFilterCheck(b *testing.B) {
b.SetBytes(bytesPerFilterBlock)
}
+func TestBlockSplitBloomFilterCheckBulk(t *testing.T) {
+ bf := blockSplitBloomFilter{bitset32: make([]uint32,
99*bitsSetPerBlock)}
+ hashes := []uint64{1, 2, 3, 42, 99, 12345, 67890}
+ for _, h := range hashes {
+ bf.InsertHash(h)
+ }
+
+ got := bf.CheckBulk(hashes)
+ if len(got) != len(hashes) {
+ t.Fatalf("CheckBulk returned %d results, want %d", len(got),
len(hashes))
+ }
+ for i, h := range hashes {
+ if !got[i] {
+ t.Errorf("CheckBulk reported inserted hash %d absent",
h)
+ }
+ if got[i] != bf.CheckHash(h) {
+ t.Errorf("CheckBulk[%d]=%v disagrees with CheckHash=%v
for hash %d", i, got[i], bf.CheckHash(h), h)
+ }
+ }
+}
+
func BenchmarkFilterCheckBulk(b *testing.B) {
bf := blockSplitBloomFilter{bitset32: make([]uint32,
99*bitsSetPerBlock)}
x := make([]uint64, 16)
