This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new a2f6a4fd fix(array): array data hashing coverage (#872)
a2f6a4fd is described below
commit a2f6a4fda40f7023602862e8b9422228fb621dd7
Author: Minh Vu <[email protected]>
AuthorDate: Thu Jul 2 18:16:41 2026 +0200
fix(array): array data hashing coverage (#872)
### What changed
array.Hash now writes the array data type fingerprint, length, offset,
every buffer with boundaries and nil markers, child data in order, and
dictionary data when present.
### Why
The old implementation wrote length twice and only considered the null
bitmap plus child data. That meant arrays with different value buffers,
types, offsets, or dictionaries could produce the same hash.
### Validation
- go test ./arrow/array -run TestHashIncludesDataMetadataAndBuffers
-count=1
- go test ./arrow/array -count=1
-
PARQUET_TEST_DATA=/Users/hoangvu/Code/OSS/arrow-go/parquet-testing/data
ARROW_TEST_DATA=/Users/hoangvu/Code/OSS/arrow-go/arrow-testing/data go
test ./... -count=1
---
arrow/array/data.go | 61 +++++++++++++++++++++++++++++-----
arrow/array/data_test.go | 85 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 138 insertions(+), 8 deletions(-)
diff --git a/arrow/array/data.go b/arrow/array/data.go
index 1772acb4..47d83f56 100644
--- a/arrow/array/data.go
+++ b/arrow/array/data.go
@@ -17,15 +17,13 @@
package array
import (
- "math/bits"
+ "encoding/binary"
"sync/atomic"
- "unsafe"
-
- "github.com/apache/arrow-go/v18/internal/utils/maphash"
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/internal/debug"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/internal/utils/maphash"
)
// Data represents the memory and metadata of an Arrow array.
@@ -275,12 +273,59 @@ func NewSliceData(data arrow.ArrayData, i, j int64)
arrow.ArrayData {
func Hash(h *maphash.MapHash, data arrow.ArrayData) {
a := data.(*Data)
- h.Write((*[bits.UintSize / 8]byte)(unsafe.Pointer(&a.length))[:])
- h.Write((*[bits.UintSize / 8]byte)(unsafe.Pointer(&a.length))[:])
- if len(a.buffers) > 0 && a.buffers[0] != nil {
- h.Write(a.buffers[0].Bytes())
+ hashString(h, a.dtype.Fingerprint())
+ hashInt(h, a.length)
+ hashInt(h, a.offset)
+
+ hashInt(h, len(a.buffers))
+ for _, b := range a.buffers {
+ if b == nil {
+ hashByte(h, 0)
+ continue
+ }
+
+ hashByte(h, 1)
+ hashBytes(h, b.Bytes())
}
+
+ hashInt(h, len(a.childData))
for _, c := range a.childData {
+ if c == nil {
+ hashByte(h, 0)
+ continue
+ }
+
+ hashByte(h, 1)
Hash(h, c)
}
+
+ if a.dictionary == nil {
+ hashByte(h, 0)
+ return
+ }
+
+ hashByte(h, 1)
+ Hash(h, a.dictionary)
+}
+
+func hashByte(h *maphash.MapHash, v byte) {
+ var b [1]byte
+ b[0] = v
+ h.Write(b[:])
+}
+
+func hashInt(h *maphash.MapHash, v int) {
+ var b [8]byte
+ binary.LittleEndian.PutUint64(b[:], uint64(int64(v)))
+ h.Write(b[:])
+}
+
+func hashBytes(h *maphash.MapHash, b []byte) {
+ hashInt(h, len(b))
+ h.Write(b)
+}
+
+func hashString(h *maphash.MapHash, s string) {
+ hashInt(h, len(s))
+ h.WriteString(s)
}
diff --git a/arrow/array/data_test.go b/arrow/array/data_test.go
index 7b690100..95eb3848 100644
--- a/arrow/array/data_test.go
+++ b/arrow/array/data_test.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/internal/utils/maphash"
"github.com/stretchr/testify/assert"
)
@@ -100,6 +101,90 @@ func TestDataResetClearsDictionary(t *testing.T) {
mem.AssertSize(t, 0)
}
+func TestHashIncludesDataMetadataAndBuffers(t *testing.T) {
+ seed := maphash.MakeSeed()
+ hash := func(data arrow.ArrayData) uint64 {
+ var h maphash.MapHash
+ h.SetSeed(seed)
+ Hash(&h, data)
+ return h.Sum64()
+ }
+
+ base := NewData(arrow.PrimitiveTypes.Int32, 2, []*memory.Buffer{
+ nil,
+ memory.NewBufferBytes([]byte{1, 0, 0, 0, 2, 0, 0, 0}),
+ }, nil, 0, 0)
+ defer base.Release()
+ baseHash := hash(base)
+
+ t.Run("value buffers", func(t *testing.T) {
+ other := NewData(arrow.PrimitiveTypes.Int32, 2,
[]*memory.Buffer{
+ nil,
+ memory.NewBufferBytes([]byte{3, 0, 0, 0, 4, 0, 0, 0}),
+ }, nil, 0, 0)
+ defer other.Release()
+
+ assert.NotEqual(t, baseHash, hash(other))
+ })
+
+ t.Run("data type", func(t *testing.T) {
+ other := NewData(arrow.PrimitiveTypes.Uint32, 2,
base.Buffers(), nil, 0, 0)
+ defer other.Release()
+
+ assert.NotEqual(t, baseHash, hash(other))
+ })
+
+ t.Run("offset", func(t *testing.T) {
+ other := NewData(arrow.PrimitiveTypes.Int32, 2, base.Buffers(),
nil, 0, 1)
+ defer other.Release()
+
+ assert.NotEqual(t, baseHash, hash(other))
+ })
+
+ t.Run("children", func(t *testing.T) {
+ childA := NewData(arrow.PrimitiveTypes.Int32, 1,
[]*memory.Buffer{
+ nil,
+ memory.NewBufferBytes([]byte{1, 0, 0, 0}),
+ }, nil, 0, 0)
+ defer childA.Release()
+ childB := NewData(arrow.PrimitiveTypes.Int32, 1,
[]*memory.Buffer{
+ nil,
+ memory.NewBufferBytes([]byte{2, 0, 0, 0}),
+ }, nil, 0, 0)
+ defer childB.Release()
+
+ offsets := memory.NewBufferBytes([]byte{0, 0, 0, 0, 1, 0, 0, 0})
+ left := NewData(arrow.ListOf(arrow.PrimitiveTypes.Int32), 1,
[]*memory.Buffer{nil, offsets}, []arrow.ArrayData{childA}, 0, 0)
+ defer left.Release()
+ right := NewData(arrow.ListOf(arrow.PrimitiveTypes.Int32), 1,
[]*memory.Buffer{nil, offsets}, []arrow.ArrayData{childB}, 0, 0)
+ defer right.Release()
+
+ assert.NotEqual(t, hash(left), hash(right))
+ })
+
+ t.Run("dictionary", func(t *testing.T) {
+ dictA := NewData(arrow.PrimitiveTypes.Int32, 1,
[]*memory.Buffer{
+ nil,
+ memory.NewBufferBytes([]byte{1, 0, 0, 0}),
+ }, nil, 0, 0)
+ defer dictA.Release()
+ dictB := NewData(arrow.PrimitiveTypes.Int32, 1,
[]*memory.Buffer{
+ nil,
+ memory.NewBufferBytes([]byte{2, 0, 0, 0}),
+ }, nil, 0, 0)
+ defer dictB.Release()
+
+ dt := &arrow.DictionaryType{IndexType:
arrow.PrimitiveTypes.Uint8, ValueType: arrow.PrimitiveTypes.Int32}
+ indices := []*memory.Buffer{nil,
memory.NewBufferBytes([]byte{0})}
+ left := NewDataWithDictionary(dt, 1, indices, 0, 0, dictA)
+ defer left.Release()
+ right := NewDataWithDictionary(dt, 1, indices, 0, 0, dictB)
+ defer right.Release()
+
+ assert.NotEqual(t, hash(left), hash(right))
+ })
+}
+
func TestSizeInBytes(t *testing.T) {
var buffers1 = make([]*memory.Buffer, 0, 3)
