This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 69c2fd50 fix(array): validate binary_view and string_view payloads
(#994)
69c2fd50 is described below
commit 69c2fd50fa2fc1f122b7abce66c6863e4302a6e8
Author: Minh Vu <[email protected]>
AuthorDate: Fri Jul 24 20:16:44 2026 +0200
fix(array): validate binary_view and string_view payloads (#994)
## Summary
- add BinaryView and StringView validators
- validate view header ranges, referenced buffers, prefixes, and inline
padding during ValidateFull
- validate UTF-8 for StringView payloads and add regressions for
malformed headers
## Why
BinaryView and StringView accessors currently trust header metadata
directly. Malformed view headers can pass validation today even when
they reference missing data or carry invalid payload metadata.
## Validation
- go test ./arrow/array
- go test ./arrow/compute/...
---
arrow/array/binary.go | 94 ++++++++++++++++++++
arrow/array/string.go | 16 ++++
arrow/array/validate_test.go | 207 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 317 insertions(+)
diff --git a/arrow/array/binary.go b/arrow/array/binary.go
index a8e77ae9..d23e2a0b 100644
--- a/arrow/array/binary.go
+++ b/arrow/array/binary.go
@@ -482,6 +482,17 @@ func (a *BinaryView) ValueLen(i int) int {
return s.Len()
}
+func (a *BinaryView) Validate() error {
+ return validateViewLayout(a, "binary view")
+}
+
+func (a *BinaryView) ValidateFull() error {
+ if err := a.Validate(); err != nil {
+ return err
+ }
+ return validateViewValues(a, a.dataBuffers, nil)
+}
+
// ValueString returns the value at index i as a string instead of
// a byte slice, without copying the underlying data.
func (a *BinaryView) ValueString(i int) string {
@@ -552,6 +563,89 @@ func arrayEqualBinaryView(left, right *BinaryView) bool {
return true
}
+func validateViewLayout(arr ViewLike, kind string) error {
+ data := arr.Data().(*Data)
+ if data.length == 0 {
+ return nil
+ }
+ if data.buffers[1] == nil {
+ return fmt.Errorf("arrow/array: non-empty %s array has no view
buffer", kind)
+ }
+
+ expNumViews := data.offset + data.length
+ if len(data.buffers[1].Bytes())/arrow.ViewHeaderSizeBytes < expNumViews
{
+ return fmt.Errorf("arrow/array: %s buffer must have at least %d
view values", kind, expNumViews)
+ }
+ return nil
+}
+
+func validateViewValues(arr ViewLike, dataBuffers []*memory.Buffer,
validateValue func(int, []byte) error) error {
+ data := arr.Data().(*Data)
+ if data.length == 0 {
+ return nil
+ }
+ rawViews := data.buffers[1].Bytes()
+ for i := 0; i < data.length; i++ {
+ if arr.IsNull(i) {
+ continue
+ }
+
+ view := arr.ValueHeader(i)
+ if view.Len() < 0 {
+ return fmt.Errorf("arrow/array: view at slot %d has
negative size %d", i, view.Len())
+ }
+
+ if view.IsInline() {
+ rawOffset := (data.offset + i) *
arrow.ViewHeaderSizeBytes
+ raw := rawViews[rawOffset :
rawOffset+arrow.ViewHeaderSizeBytes]
+ for _, b := range raw[4+view.Len() :
arrow.ViewHeaderSizeBytes] {
+ if b != 0 {
+ return fmt.Errorf("arrow/array: view at
slot %d was inline with size %d but its padding bytes were not all zero", i,
view.Len())
+ }
+ }
+ if validateValue != nil {
+ if err := validateValue(i, view.InlineBytes());
err != nil {
+ return err
+ }
+ }
+ continue
+ }
+
+ if view.BufferIndex() < 0 {
+ return fmt.Errorf("arrow/array: view at slot %d has
negative buffer index %d", i, view.BufferIndex())
+ }
+ if view.BufferOffset() < 0 {
+ return fmt.Errorf("arrow/array: view at slot %d has
negative offset %d", i, view.BufferOffset())
+ }
+ if int(view.BufferIndex()) >= len(dataBuffers) {
+ return fmt.Errorf("arrow/array: view at slot %d
references buffer %d but there are only %d data buffers", i,
view.BufferIndex(), len(dataBuffers))
+ }
+
+ buf := dataBuffers[view.BufferIndex()]
+ if buf == nil {
+ return fmt.Errorf("arrow/array: view at slot %d
references nil data buffer %d", i, view.BufferIndex())
+ }
+
+ offset := int(view.BufferOffset())
+ end := offset + view.Len()
+ if end > buf.Len() {
+ return fmt.Errorf("arrow/array: view at slot %d
references range %d-%d of buffer %d but that buffer is only %d bytes long", i,
offset, end, view.BufferIndex(), buf.Len())
+ }
+
+ value := buf.Bytes()[offset:end]
+ prefix := view.Prefix()
+ if !bytes.Equal(value[:arrow.ViewPrefixLen], prefix[:]) {
+ return fmt.Errorf("arrow/array: view at slot %d has
inlined prefix %x but the out-of-line data begins with %x", i, prefix,
value[:arrow.ViewPrefixLen])
+ }
+ if validateValue != nil {
+ if err := validateValue(i, value); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
var (
_ arrow.Array = (*Binary)(nil)
_ arrow.Array = (*LargeBinary)(nil)
diff --git a/arrow/array/string.go b/arrow/array/string.go
index 7c2ab074..519b2cae 100644
--- a/arrow/array/string.go
+++ b/arrow/array/string.go
@@ -501,6 +501,22 @@ func (a *StringView) ValueLen(i int) int {
return s.Len()
}
+func (a *StringView) Validate() error {
+ return validateViewLayout(a, "string view")
+}
+
+func (a *StringView) ValidateFull() error {
+ if err := a.Validate(); err != nil {
+ return err
+ }
+ return validateViewValues(a, a.dataBuffers, func(i int, value []byte)
error {
+ if !utf8.Valid(value) {
+ return fmt.Errorf("arrow/array: string view at slot %d
is not valid utf8", i)
+ }
+ return nil
+ })
+}
+
func (a *StringView) String() string {
var o strings.Builder
o.WriteString("[")
diff --git a/arrow/array/validate_test.go b/arrow/array/validate_test.go
index 7b83f7a4..ac641787 100644
--- a/arrow/array/validate_test.go
+++ b/arrow/array/validate_test.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/bitutil"
+ "github.com/apache/arrow-go/v18/arrow/endian"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -77,6 +78,42 @@ func makeInt32ArrayRaw(t *testing.T, values []int32,
validity []byte, nulls, len
return arr
}
+func makeBinaryViewArrayRaw(t *testing.T, headerBytes []byte, dataBuffers
[]*memory.Buffer, validity []byte, nulls, length, offset int) *BinaryView {
+ t.Helper()
+ var validityBuf *memory.Buffer
+ if validity != nil {
+ validityBuf = memory.NewBufferBytes(validity)
+ }
+ viewBuf := memory.NewBufferBytes(headerBytes)
+ buffers := append([]*memory.Buffer{validityBuf, viewBuf},
dataBuffers...)
+ data := NewData(arrow.BinaryTypes.BinaryView, length, buffers, nil,
nulls, offset)
+ arr := NewBinaryViewData(data)
+ data.Release()
+ return arr
+}
+
+func makeStringViewArrayRaw(t *testing.T, headerBytes []byte, dataBuffers
[]*memory.Buffer, validity []byte, nulls, length, offset int) *StringView {
+ t.Helper()
+ var validityBuf *memory.Buffer
+ if validity != nil {
+ validityBuf = memory.NewBufferBytes(validity)
+ }
+ viewBuf := memory.NewBufferBytes(headerBytes)
+ buffers := append([]*memory.Buffer{validityBuf, viewBuf},
dataBuffers...)
+ data := NewData(arrow.BinaryTypes.StringView, length, buffers, nil,
nulls, offset)
+ arr := NewStringViewData(data)
+ data.Release()
+ return arr
+}
+
+func setViewHeaderBufferIndex(raw []byte, idx int32) {
+ endian.Native.PutUint32(raw[8:12], uint32(idx))
+}
+
+func setViewHeaderOffset(raw []byte, offset int32) {
+ endian.Native.PutUint32(raw[12:16], uint32(offset))
+}
+
func TestBinaryValidate(t *testing.T) {
t.Run("valid array passes", func(t *testing.T) {
// offsets [0,3,6,9], data "abcdefghi" — 3 elements of 3 bytes
each
@@ -201,6 +238,176 @@ func TestLargeStringValidate(t *testing.T) {
})
}
+func TestBinaryViewValidate(t *testing.T) {
+ t.Run("empty arrays pass top level validation", func(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ binaryArr := NewBinaryViewBuilder(mem).NewBinaryViewArray()
+ defer binaryArr.Release()
+ stringArr := NewStringViewBuilder(mem).NewStringViewArray()
+ defer stringArr.Release()
+
+ assert.NoError(t, Validate(binaryArr))
+ assert.NoError(t, ValidateFull(binaryArr))
+ assert.NoError(t, Validate(stringArr))
+ assert.NoError(t, ValidateFull(stringArr))
+ })
+
+ t.Run("valid array passes", func(t *testing.T) {
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes([]byte("hello"))
+ arr := makeBinaryViewArrayRaw(t,
arrow.ViewHeaderTraits.CastToBytes(headers[:]), nil, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, arr.Validate())
+ assert.NoError(t, arr.ValidateFull())
+ })
+
+ t.Run("out of line values pass top level validation", func(t
*testing.T) {
+ value := []byte("this payload is out of line")
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes(value)
+ headers[0].SetIndexOffset(0, 0)
+ dataBuf := memory.NewBufferBytes(value)
+ arr := makeBinaryViewArrayRaw(t,
arrow.ViewHeaderTraits.CastToBytes(headers[:]), []*memory.Buffer{dataBuf}, nil,
0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ assert.NoError(t, ValidateFull(arr))
+ })
+
+ t.Run("offset arrays use the correct view slot", func(t *testing.T) {
+ headers := [2]arrow.ViewHeader{}
+ headers[0].SetBytes([]byte("skip"))
+ headers[1].SetBytes([]byte("keep"))
+ arr := makeBinaryViewArrayRaw(t,
arrow.ViewHeaderTraits.CastToBytes(headers[:]), nil, nil, 0, 1, 1)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ assert.NoError(t, ValidateFull(arr))
+ })
+
+ t.Run("negative size passes Validate but fails ValidateFull", func(t
*testing.T) {
+ headerBytes := make([]byte, arrow.ViewHeaderSizeBytes)
+ endian.Native.PutUint32(headerBytes[:4], ^uint32(0))
+ arr := makeBinaryViewArrayRaw(t, headerBytes, nil, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, arr.Validate())
+ err := arr.ValidateFull()
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "negative size")
+ })
+
+ t.Run("missing referenced buffer passes Validate but fails
ValidateFull", func(t *testing.T) {
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes([]byte("this is longer than twelve"))
+ headers[0].SetIndexOffset(0, 0)
+ arr := makeBinaryViewArrayRaw(t,
arrow.ViewHeaderTraits.CastToBytes(headers[:]), nil, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, arr.Validate())
+ err := arr.ValidateFull()
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "references buffer 0")
+ })
+
+ t.Run("prefix mismatch passes Validate but fails ValidateFull", func(t
*testing.T) {
+ value := []byte("this payload is out of line")
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes(value)
+ headers[0].SetIndexOffset(0, 0)
+ headerBytes := append([]byte(nil),
arrow.ViewHeaderTraits.CastToBytes(headers[:])...)
+ headerBytes[4] ^= 0xff
+ dataBuf := memory.NewBufferBytes(value)
+ arr := makeBinaryViewArrayRaw(t, headerBytes,
[]*memory.Buffer{dataBuf}, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ err := ValidateFull(arr)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "out-of-line data begins with")
+ })
+
+ t.Run("negative buffer offset passes Validate but fails ValidateFull",
func(t *testing.T) {
+ value := []byte("this payload is out of line")
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes(value)
+ headers[0].SetIndexOffset(0, 0)
+ headerBytes := append([]byte(nil),
arrow.ViewHeaderTraits.CastToBytes(headers[:])...)
+ setViewHeaderOffset(headerBytes, -1)
+ dataBuf := memory.NewBufferBytes(value)
+ arr := makeBinaryViewArrayRaw(t, headerBytes,
[]*memory.Buffer{dataBuf}, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ err := ValidateFull(arr)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "negative offset")
+ })
+
+ t.Run("negative buffer index passes Validate but fails ValidateFull",
func(t *testing.T) {
+ value := []byte("this payload is out of line")
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes(value)
+ headers[0].SetIndexOffset(0, 0)
+ headerBytes := append([]byte(nil),
arrow.ViewHeaderTraits.CastToBytes(headers[:])...)
+ setViewHeaderBufferIndex(headerBytes, -1)
+ dataBuf := memory.NewBufferBytes(value)
+ arr := makeBinaryViewArrayRaw(t, headerBytes,
[]*memory.Buffer{dataBuf}, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ err := ValidateFull(arr)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "negative buffer index")
+ })
+
+ t.Run("referenced range beyond buffer length passes Validate but fails
ValidateFull", func(t *testing.T) {
+ value := []byte("this payload is out of line")
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes(value)
+ headers[0].SetIndexOffset(0, 2)
+ dataBuf := memory.NewBufferBytes(value)
+ arr := makeBinaryViewArrayRaw(t,
arrow.ViewHeaderTraits.CastToBytes(headers[:]), []*memory.Buffer{dataBuf}, nil,
0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, Validate(arr))
+ err := ValidateFull(arr)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "references range")
+ })
+
+ t.Run("inline padding bytes fail ValidateFull", func(t *testing.T) {
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes([]byte("x"))
+ headerBytes := append([]byte(nil),
arrow.ViewHeaderTraits.CastToBytes(headers[:])...)
+ headerBytes[8] = 1
+ arr := makeBinaryViewArrayRaw(t, headerBytes, nil, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, arr.Validate())
+ err := arr.ValidateFull()
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "padding bytes were not all
zero")
+ })
+}
+
+func TestStringViewValidate(t *testing.T) {
+ t.Run("invalid utf8 passes Validate but fails ValidateFull", func(t
*testing.T) {
+ var headers [1]arrow.ViewHeader
+ headers[0].SetBytes([]byte{0xff})
+ arr := makeStringViewArrayRaw(t,
arrow.ViewHeaderTraits.CastToBytes(headers[:]), nil, nil, 0, 1, 0)
+ defer arr.Release()
+
+ assert.NoError(t, arr.Validate())
+ err := arr.ValidateFull()
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "not valid utf8")
+ })
+}
+
func TestTopLevelValidate(t *testing.T) {
t.Run("Validate dispatches to Validator", func(t *testing.T) {
// non-monotonic string array: passes setData but ValidateFull
must fail
