This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new afb7ad0e fix(arrow/scalar): use wide offsets for large binary scalars
(#1108)
afb7ad0e is described below
commit afb7ad0e370d0ba6fdbd004115ee5f874e04ad5e
Author: Minh Vu <[email protected]>
AuthorDate: Mon Aug 10 18:52:22 2026 +0200
fix(arrow/scalar): use wide offsets for large binary scalars (#1108)
### Rationale for this change
MakeArrayFromScalar currently uses 32-bit offsets for every binary-like
scalar. LargeBinary and LargeString require 64-bit offsets, so their
arrays can have the wrong physical layout.
### What changes are included in this PR?
Select the offset width from the scalar type and test ordinary and large
binary values, including validation and buffer size.
### Are these changes tested?
- `go test ./arrow/scalar`
### Are there any user-facing changes?
No API changes. This corrects the reported behavior while preserving the
existing ownership and compatibility contracts.
---
arrow/scalar/scalar.go | 52 +++++++++--------------------
arrow/scalar/scalar_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 94 insertions(+), 37 deletions(-)
diff --git a/arrow/scalar/scalar.go b/arrow/scalar/scalar.go
index 8cb31764..535eb367 100644
--- a/arrow/scalar/scalar.go
+++ b/arrow/scalar/scalar.go
@@ -756,37 +756,7 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) {
//
// Deprecated: Use array.MakeArrayOfNull
func MakeArrayOfNull(dt arrow.DataType, length int, mem memory.Allocator)
arrow.Array {
- var (
- buffers = []*memory.Buffer{nil}
- children []arrow.ArrayData
- )
-
- buffers[0] = memory.NewResizableBuffer(mem)
- buffers[0].Resize(int(bitutil.BytesForBits(int64(length))))
- defer buffers[0].Release()
-
- switch t := dt.(type) {
- case arrow.NestedType:
- fieldList := t.Fields()
- children = make([]arrow.ArrayData, len(fieldList))
- for i, f := range fieldList {
- arr := MakeArrayOfNull(f.Type, length, mem)
- defer arr.Release()
- children[i] = arr.Data()
- }
- case arrow.FixedWidthDataType:
- buffers = append(buffers, memory.NewResizableBuffer(mem))
-
buffers[1].Resize(int(bitutil.BytesForBits(int64(t.BitWidth()))) * length)
- defer buffers[1].Release()
- case arrow.BinaryDataType:
- buffers = append(buffers, memory.NewResizableBuffer(mem), nil)
- buffers[1].Resize(arrow.Int32Traits.BytesRequired(length + 1))
- defer buffers[1].Release()
- }
-
- data := array.NewData(dt, length, buffers, children, length, 0)
- defer data.Release()
- return array.MakeFromData(data)
+ return array.MakeArrayOfNull(mem, dt, length)
}
// MakeArrayFromScalar returns an array filled with the scalar value repeated
length times.
@@ -800,12 +770,20 @@ func MakeArrayFromScalar(sc Scalar, length int, mem
memory.Allocator) (arrow.Arr
return MakeArrayOfNull(sc.DataType(), length, mem), nil
}
- createOffsets := func(valLength int32) *memory.Buffer {
+ createOffsets := func(valLength int64) *memory.Buffer {
buffer := memory.NewResizableBuffer(mem)
- buffer.Resize(arrow.Int32Traits.BytesRequired(length + 1))
+ offsetTraits :=
sc.DataType().(arrow.OffsetsDataType).OffsetTypeTraits()
+ buffer.Resize(offsetTraits.BytesRequired(length + 1))
+ if offsetTraits.BytesRequired(1) == arrow.Int64SizeBytes {
+ out := arrow.Int64Traits.CastFromBytes(buffer.Bytes())
+ for i, offset := 0, int64(0); i < length+1; i, offset =
i+1, offset+valLength {
+ out[i] = offset
+ }
+ return buffer
+ }
out := arrow.Int32Traits.CastFromBytes(buffer.Bytes())
- for i, offset := 0, int32(0); i < length+1; i, offset = i+1,
offset+valLength {
+ for i, offset := 0, int32(0); i < length+1; i, offset = i+1,
offset+int32(valLength) {
out[i] = offset
}
return buffer
@@ -848,7 +826,7 @@ func MakeArrayFromScalar(sc Scalar, length int, mem
memory.Allocator) (arrow.Arr
}
valuesBuf := createBuffer(s.Data())
- offsetsBuf := createOffsets(int32(len(s.Data())))
+ offsetsBuf := createOffsets(int64(len(s.Data())))
data := array.NewData(sc.DataType(), length,
[]*memory.Buffer{nil, offsetsBuf, valuesBuf}, nil, 0, 0)
defer func() {
valuesBuf.Release()
@@ -880,7 +858,7 @@ func MakeArrayFromScalar(sc Scalar, length int, mem
memory.Allocator) (arrow.Arr
}
defer valueArray.Release()
- offsetsBuf := createOffsets(int32(s.Value.Len()))
+ offsetsBuf := createOffsets(int64(s.Value.Len()))
defer offsetsBuf.Release()
data := array.NewData(s.DataType(), length,
[]*memory.Buffer{nil, offsetsBuf}, []arrow.ArrayData{valueArray.Data()}, 0, 0)
defer data.Release()
@@ -935,7 +913,7 @@ func MakeArrayFromScalar(sc Scalar, length int, mem
memory.Allocator) (arrow.Arr
}
defer valueArr.Release()
- offsetsBuf := createOffsets(int32(structArr.Len()))
+ offsetsBuf := createOffsets(int64(structArr.Len()))
outStructArr := array.NewData(structArr.DataType(),
keyArr.Len(), []*memory.Buffer{nil}, []arrow.ArrayData{keyArr.Data(),
valueArr.Data()}, 0, 0)
data := array.NewData(s.DataType(), length,
[]*memory.Buffer{nil, offsetsBuf}, []arrow.ArrayData{outStructArr}, 0, 0)
defer func() {
diff --git a/arrow/scalar/scalar_test.go b/arrow/scalar/scalar_test.go
index 3cec8f97..1627aef9 100644
--- a/arrow/scalar/scalar_test.go
+++ b/arrow/scalar/scalar_test.go
@@ -1253,6 +1253,85 @@ func TestMakeArrayFromScalar(t *testing.T) {
}
}
+func TestMakeArrayFromScalarUsesCorrectBinaryOffsetWidth(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ buf := memory.NewBufferBytes([]byte("abc"))
+ defer buf.Release()
+
+ tests := []struct {
+ name string
+ sc scalar.Scalar
+ bytes int
+ offsets []int64
+ }{
+ {name: "binary", sc: scalar.NewBinaryScalar(buf,
arrow.BinaryTypes.Binary), bytes: 4, offsets: []int64{0, 3, 6, 9}},
+ {name: "string", sc: scalar.NewStringScalar("abc"), bytes: 4,
offsets: []int64{0, 3, 6, 9}},
+ {name: "large binary", sc: scalar.NewLargeBinaryScalar(buf),
bytes: 8, offsets: []int64{0, 3, 6, 9}},
+ {name: "large string", sc: scalar.NewLargeStringScalar("abc"),
bytes: 8, offsets: []int64{0, 3, 6, 9}},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ if releasable, ok := tc.sc.(scalar.Releasable); ok {
+ defer releasable.Release()
+ }
+
+ arr, err := scalar.MakeArrayFromScalar(tc.sc, 3, mem)
+ require.NoError(t, err)
+ defer arr.Release()
+ require.NoError(t, array.ValidateFull(arr))
+ require.Len(t, arr.Data().Buffers()[1].Bytes(),
4*tc.bytes)
+
+ switch offsets := arr.(type) {
+ case *array.Binary:
+ assert.Equal(t, tc.offsets,
slicesToInt64(offsets.ValueOffsets()))
+ case *array.String:
+ assert.Equal(t, tc.offsets,
slicesToInt64(offsets.ValueOffsets()))
+ case *array.LargeBinary:
+ assert.Equal(t, tc.offsets,
offsets.ValueOffsets())
+ case *array.LargeString:
+ assert.Equal(t, tc.offsets,
offsets.ValueOffsets())
+ default:
+ t.Fatalf("unexpected array type %T", arr)
+ }
+ })
+ }
+}
+
+func slicesToInt64(values []int32) []int64 {
+ out := make([]int64, len(values))
+ for i, value := range values {
+ out[i] = int64(value)
+ }
+ return out
+}
+
+func TestMakeArrayFromScalarUsesCorrectBinaryOffsetWidthForNulls(t *testing.T)
{
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ const length = 3
+ for _, dt := range []arrow.DataType{arrow.BinaryTypes.LargeBinary,
arrow.BinaryTypes.LargeString} {
+ t.Run(dt.Name(), func(t *testing.T) {
+ sc := scalar.MakeNullScalar(dt)
+ if releasable, ok := sc.(scalar.Releasable); ok {
+ defer releasable.Release()
+ }
+
+ arr, err := scalar.MakeArrayFromScalar(sc, length, mem)
+ require.NoError(t, err)
+ defer arr.Release()
+
+ require.NoError(t, array.ValidateFull(arr))
+ assert.Equal(t, length, arr.Len())
+ assert.Equal(t, length, arr.NullN())
+ assert.Equal(t,
arrow.Int64Traits.BytesRequired(length+1), arr.Data().Buffers()[1].Len())
+ })
+ }
+}
+
func TestMakeArrayFromScalarRejectsNegativeLength(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
