This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new b1d20a37 fix(arrow/array): validate fixed-size list child lengths
(#1019)
b1d20a37 is described below
commit b1d20a373fc8a30152ad9cca78657882fa5b929a
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 23:07:14 2026 +0200
fix(arrow/array): validate fixed-size list child lengths (#1019)
### Rationale for this change
`FixedSizeListBuilder` can finalize an array when the child value count
does not equal the parent length multiplied by the list size. Too few
values produce an invalid array, while extra values become silently
inaccessible.
### What changes are included in this PR?
* Require the exact child value count before finalizing a fixed-size
list array.
* Preserve the builder state when validation fails.
* Correct existing Arrow and Parquet null-list fixtures so every parent
slot has the required physical child slots.
### Are these changes tested?
Yes. The tests cover both missing and extra child values and verify
valid null-list layouts. The full `arrow/array` and `parquet/pqarrow`
packages pass with assertion checks enabled.
---
arrow/array/fixed_size_list.go | 5 ++++
arrow/array/fixed_size_list_test.go | 50 +++++++++++++++++++++++-------------
parquet/pqarrow/path_builder_test.go | 6 ++++-
3 files changed, 42 insertions(+), 19 deletions(-)
diff --git a/arrow/array/fixed_size_list.go b/arrow/array/fixed_size_list.go
index ca9210ab..6a4f75ca 100644
--- a/arrow/array/fixed_size_list.go
+++ b/arrow/array/fixed_size_list.go
@@ -313,6 +313,11 @@ func (b *FixedSizeListBuilder) NewListArray() (a
*FixedSizeList) {
}
func (b *FixedSizeListBuilder) newData() (data *Data) {
+ want := int64(b.length) * int64(b.n)
+ if int64(b.values.Len()) != want {
+ panic(fmt.Errorf("%w: arrow/array: fixed-size list value count
must equal list length times list size (values=%d, want=%d)",
+ arrow.ErrInvalid, b.values.Len(), want))
+ }
values := b.values.NewArray()
defer values.Release()
diff --git a/arrow/array/fixed_size_list_test.go
b/arrow/array/fixed_size_list_test.go
index 44f0b39f..b899647a 100644
--- a/arrow/array/fixed_size_list_test.go
+++ b/arrow/array/fixed_size_list_test.go
@@ -17,6 +17,7 @@
package array_test
import (
+ "fmt"
"reflect"
"testing"
@@ -31,26 +32,22 @@ func TestFixedSizeListArray(t *testing.T) {
defer pool.AssertSize(t, 0)
var (
- vs = []int32{0, 1, 2, 3, 4, 5, 6}
- lengths = []int{3, 0, 4}
+ vs = []int32{0, 1, 2, 0, 0, 0, 3, 4, 5}
isValid = []bool{true, false, true}
)
- lb := array.NewFixedSizeListBuilder(pool, int32(len(vs)),
arrow.PrimitiveTypes.Int32)
+ lb := array.NewFixedSizeListBuilder(pool, 3, arrow.PrimitiveTypes.Int32)
defer lb.Release()
for i := 0; i < 10; i++ {
vb := lb.ValueBuilder().(*array.Int32Builder)
vb.Reserve(len(vs))
- pos := 0
- for i, length := range lengths {
- lb.Append(isValid[i])
- for j := 0; j < length; j++ {
- vb.Append(vs[pos])
- pos++
- }
- }
+ lb.Append(true)
+ vb.AppendValues(vs[:3], nil)
+ lb.AppendNull()
+ lb.Append(true)
+ vb.AppendValues(vs[6:], nil)
arr := lb.NewArray().(*array.FixedSizeList)
defer arr.Release()
@@ -66,11 +63,11 @@ func TestFixedSizeListArray(t *testing.T) {
t.Fatalf("got=%d, want=%d", got, want)
}
- for i := range lengths {
+ for i := range isValid {
if got, want := arr.IsValid(i), isValid[i]; got != want
{
t.Fatalf("got[%d]=%v, want[%d]=%v", i, got, i,
want)
}
- if got, want := arr.IsNull(i), lengths[i] == 0; got !=
want {
+ if got, want := arr.IsNull(i), !isValid[i]; got != want
{
t.Fatalf("got[%d]=%v, want[%d]=%v", i, got, i,
want)
}
}
@@ -100,12 +97,11 @@ func TestFixedSizeListArrayBulkAppend(t *testing.T) {
defer pool.AssertSize(t, 0)
var (
- vs = []int32{0, 1, 2, 3, 4, 5, 6}
- lengths = []int{3, 0, 4}
+ vs = []int32{0, 1, 2, 0, 0, 0, 3, 4, 5}
isValid = []bool{true, false, true}
)
- lb := array.NewFixedSizeListBuilder(pool, int32(len(vs)),
arrow.PrimitiveTypes.Int32)
+ lb := array.NewFixedSizeListBuilder(pool, 3, arrow.PrimitiveTypes.Int32)
defer lb.Release()
vb := lb.ValueBuilder().(*array.Int32Builder)
vb.Reserve(len(vs))
@@ -126,11 +122,11 @@ func TestFixedSizeListArrayBulkAppend(t *testing.T) {
t.Fatalf("got=%d, want=%d", got, want)
}
- for i := range lengths {
+ for i := range isValid {
if got, want := arr.IsValid(i), isValid[i]; got != want {
t.Fatalf("got[%d]=%v, want[%d]=%v", i, got, i, want)
}
- if got, want := arr.IsNull(i), lengths[i] == 0; got != want {
+ if got, want := arr.IsNull(i), !isValid[i]; got != want {
t.Fatalf("got[%d]=%v, want[%d]=%v", i, got, i, want)
}
}
@@ -217,6 +213,24 @@ func TestFixedSizeListArraySlice(t *testing.T) {
}
}
+func TestFixedSizeListBuilderRejectsMismatchedValueLength(t *testing.T) {
+ for _, valueLen := range []int{1, 3} {
+ t.Run(fmt.Sprintf("values_%d", valueLen), func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewFixedSizeListBuilder(mem, 2,
arrow.PrimitiveTypes.Int32)
+ defer b.Release()
+ b.Append(true)
+
b.ValueBuilder().(*array.Int32Builder).AppendValues(make([]int32, valueLen),
nil)
+
+ assert.PanicsWithError(t,
+ fmt.Sprintf("invalid: arrow/array: fixed-size
list value count must equal list length times list size (values=%d, want=2)",
valueLen),
+ func() { b.NewListArray() })
+ })
+ }
+}
+
func TestFixedSizeListStringRoundTrip(t *testing.T) {
// 1. create array
pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
diff --git a/parquet/pqarrow/path_builder_test.go
b/parquet/pqarrow/path_builder_test.go
index d736ee90..42b748a0 100644
--- a/parquet/pqarrow/path_builder_test.go
+++ b/parquet/pqarrow/path_builder_test.go
@@ -588,7 +588,9 @@ func TestFixedSizeListNullableElems(t *testing.T) {
vb := bldr.ValueBuilder().(*array.Int64Builder)
bldr.AppendValues([]bool{false, true, true, false})
+ vb.AppendNulls(2)
vb.AppendValues([]int64{2, 3, 4, 5}, nil)
+ vb.AppendNulls(2)
// produce: [null, [2, 3], [4, 5], null]
@@ -621,7 +623,9 @@ func TestFixedSizeListMissingMiddleTwoVisitedRanges(t
*testing.T) {
vb := bldr.ValueBuilder().(*array.Int64Builder)
bldr.AppendValues([]bool{true, false, true})
- vb.AppendValues([]int64{0, 1, 2, 3}, nil)
+ vb.AppendValues([]int64{0, 1}, nil)
+ vb.AppendNulls(2)
+ vb.AppendValues([]int64{2, 3}, nil)
// produce: [[0, 1], null, [2, 3]]
