This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 6459eca5 fix(parquet/pqarrow): honor offset and null slots when
writing fixed-size lists (#1382)
6459eca5 is described below
commit 6459eca56e210f1201d27762d7633070ed9997ab
Author: Matt Topol <[email protected]>
AuthorDate: Fri Oct 23 17:04:18 2026 +0100
fix(parquet/pqarrow): honor offset and null slots when writing fixed-size
lists (#1382)
### Rationale for this change
`pqarrow` wrote incorrect data for `FixedSizeList` columns without
returning an error:
- `fixedSizeRangeSelector` computed child ranges as `idx * listSize`,
ignoring the parent array's offset. A sliced `FixedSizeList` (e.g.
`NewSlice(arr, 1, 2)` selecting `[[20, 21]]`) was written as the leading
rows of the child array (`[[10, 11]]`). Because the writer slices
columns per row group, any unsliced `FixedSizeList` column spanning more
than one row group was affected too: `[[10, 11], [20, 21]]` with
row-group size 1 read back as `[[10, 11], [10, 11]]`.
- Null `FixedSizeList` slots still occupy child storage, so a null list
between non-null lists produces several disjoint visited leaf ranges.
The writer rejected this with `lists with non-zero length null
components are not supported`.
Thanks to ChipLab (Donmakino LLC) for reporting the offset issue.
### What changes are included in this PR?
- `fixedSizeRangeSelector` carries the parent offset and computes
`(offset + idx) * listSize`.
- `arrowColumnWriter.Write` concatenates multiple visited leaf ranges
into the values passed to `WriteArrowToColumn` (the single-range case
remains a zero-copy slice).
### Are these changes tested?
Yes. `TestWriteFixedSizeListHonorsOffset` round-trips flat,
element-null, list-null, nested, and nested-with-nulls `FixedSizeList`
columns, both sliced and unsliced, with one row per row group and a
single row group. A file containing null lists written by Go was also
read back correctly with PyArrow.
### Are there any user-facing changes?
Yes: sliced or multi-row-group `FixedSizeList` columns are now written
with the correct values, and `FixedSizeList` columns with null lists
between non-null lists can now be written.
**This PR contains a "Critical Fix".** Previously, writing such columns
silently produced incorrect data.
---
parquet/pqarrow/encode_arrow.go | 37 +++++++++---
parquet/pqarrow/fixed_size_list_test.go | 99 +++++++++++++++++++++++++++++++++
parquet/pqarrow/path_builder.go | 5 +-
3 files changed, 132 insertions(+), 9 deletions(-)
diff --git a/parquet/pqarrow/encode_arrow.go b/parquet/pqarrow/encode_arrow.go
index dd865c02..36106c79 100644
--- a/parquet/pqarrow/encode_arrow.go
+++ b/parquet/pqarrow/encode_arrow.go
@@ -304,13 +304,7 @@ func (acw *arrowColumnWriter) Write(ctx context.Context)
error {
}
defer res.Release()
- if len(res.postListVisitedElems) != 1 {
- return errors.New("lists with non-zero length
null components are not supported")
- }
- rng := res.postListVisitedElems[0]
- values := array.NewSlice(res.leafArr, rng.start,
rng.end)
- defer values.Release()
- if err = WriteArrowToColumn(ctx, cw, values,
res.defLevels, res.repLevels, res.leafIsNullable); err != nil {
+ if err = writeVisitedLeafValues(ctx, cw, &res,
arrCtx.props.mem); err != nil {
return err
}
}
@@ -318,6 +312,35 @@ func (acw *arrowColumnWriter) Write(ctx context.Context)
error {
return nil
}
+// writeVisitedLeafValues writes the visited leaf values of res to cw. Its
+// deferred Release frees a concatenated copy as soon as this chunk is written,
+// including when the write panics.
+func writeVisitedLeafValues(ctx context.Context, cw file.ColumnChunkWriter,
res *multipathLevelResult, mem memory.Allocator) error {
+ values, err := visitedLeafValues(res, mem)
+ if err != nil {
+ return err
+ }
+ defer values.Release()
+ return WriteArrowToColumn(ctx, cw, values, res.defLevels,
res.repLevels, res.leafIsNullable)
+}
+
+// visitedLeafValues returns the leaf values covered by the visited element
+// ranges. Null fixed-size-list slots still occupy child storage, so a column
+// can produce several disjoint ranges; their values are concatenated in order.
+func visitedLeafValues(res *multipathLevelResult, mem memory.Allocator)
(arrow.Array, error) {
+ if len(res.postListVisitedElems) == 1 {
+ rng := res.postListVisitedElems[0]
+ return array.NewSlice(res.leafArr, rng.start, rng.end), nil
+ }
+
+ slices := make([]arrow.Array, len(res.postListVisitedElems))
+ for i, rng := range res.postListVisitedElems {
+ slices[i] = array.NewSlice(res.leafArr, rng.start, rng.end)
+ defer slices[i].Release()
+ }
+ return array.Concatenate(slices, mem)
+}
+
// WriteArrowToColumn writes apache arrow columnar data directly to a
ColumnWriter.
// Returns non-nil error if the array data type is not compatible with the
concrete
// writer type.
diff --git a/parquet/pqarrow/fixed_size_list_test.go
b/parquet/pqarrow/fixed_size_list_test.go
new file mode 100644
index 00000000..a8a412b6
--- /dev/null
+++ b/parquet/pqarrow/fixed_size_list_test.go
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pqarrow_test
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/apache/arrow-go/v18/parquet/pqarrow"
+ "github.com/stretchr/testify/require"
+)
+
+// TestWriteFixedSizeListHonorsOffset checks that the writer selects child
+// values relative to the parent's offset, both for explicitly sliced arrays
+// and for the implicit slices taken when a column spans several row groups.
+func TestWriteFixedSizeListHonorsOffset(t *testing.T) {
+ fsl := arrow.FixedSizeListOf(2, arrow.PrimitiveTypes.Int64)
+ nested := arrow.FixedSizeListOf(2, fsl)
+
+ tests := []struct {
+ name string
+ dtype arrow.DataType
+ json string
+ sliceStart int64
+ sliceEnd int64
+ }{
+ {"flat", fsl, `[[10, 11], [20, 21], [30, 31], [40, 41]]`, 1, 3},
+ {"null-elements", fsl, `[[10, 11], [null, 21], [30, null],
[null, null], [50, 51]]`, 1, 4},
+ {"null-lists", fsl, `[[10, 11], null, [null, 31], [40, null],
null, [60, 61]]`, 2, 6},
+ {"nested", nested, `[[[1, 2], [3, 4]], [[5, 6], [7, null]],
[[9, 10], [11, 12]], [[13, 14], [15, 16]]]`, 1, 4},
+ {"nested-nulls", nested, `[[[1, 2], [3, 4]], [[5, 6], null],
null, [[13, 14], [15, 16]]]`, 1, 4},
+ }
+
+ for _, tt := range tests {
+ full, _, err := array.FromJSON(memory.DefaultAllocator,
tt.dtype, strings.NewReader(tt.json))
+ require.NoError(t, err)
+ defer full.Release()
+ sliced := array.NewSlice(full, tt.sliceStart, tt.sliceEnd)
+ defer sliced.Release()
+
+ for _, input := range []struct {
+ name string
+ arr arrow.Array
+ }{{"unsliced", full}, {"sliced", sliced}} {
+ for _, rowGroupSize := range []int64{1,
int64(input.arr.Len())} {
+ t.Run(fmt.Sprintf("%s/%s/rowgroup-%d", tt.name,
input.name, rowGroupSize), func(t *testing.T) {
+ got := roundTripFixedSizeList(t,
input.arr, rowGroupSize)
+ defer got.Release()
+ require.Truef(t, array.Equal(input.arr,
got), "want %v, got %v", input.arr, got)
+ })
+ }
+ }
+ }
+}
+
+func roundTripFixedSizeList(t *testing.T, arr arrow.Array, rowGroupSize int64)
arrow.Array {
+ t.Helper()
+ mem := memory.DefaultAllocator
+ field := arrow.Field{Name: "value", Type: arr.DataType(), Nullable:
true}
+ sch := arrow.NewSchema([]arrow.Field{field}, nil)
+ col := arrow.NewColumnFromArr(field, arr)
+ defer col.Release()
+ tbl := array.NewTable(sch, []arrow.Column{col}, int64(arr.Len()))
+ defer tbl.Release()
+
+ var buf bytes.Buffer
+ require.NoError(t, pqarrow.WriteTable(tbl, &buf, rowGroupSize,
parquet.NewWriterProperties(),
+ pqarrow.NewArrowWriterProperties(pqarrow.WithStoreSchema())))
+
+ out, err := pqarrow.ReadTable(context.Background(),
bytes.NewReader(buf.Bytes()),
+ parquet.NewReaderProperties(mem),
pqarrow.ArrowReadProperties{}, mem)
+ require.NoError(t, err)
+ defer out.Release()
+
+ got, err := array.Concatenate(out.Column(0).Data().Chunks(), mem)
+ require.NoError(t, err)
+ return got
+}
diff --git a/parquet/pqarrow/path_builder.go b/parquet/pqarrow/path_builder.go
index b10318a4..6f25bf50 100644
--- a/parquet/pqarrow/path_builder.go
+++ b/parquet/pqarrow/path_builder.go
@@ -66,11 +66,12 @@ func (v largeVarRangeSelector) GetRange(idx int64)
elemRange {
}
type fixedSizeRangeSelector struct {
+ offset int64
listSize int32
}
func (f fixedSizeRangeSelector) GetRange(idx int64) elemRange {
- start := idx * int64(f.listSize)
+ start := (f.offset + idx) * int64(f.listSize)
return elemRange{start, start + int64(f.listSize)}
}
@@ -423,7 +424,7 @@ func (p *pathBuilder) Visit(arr arrow.Array) error {
larr := arr.(*array.FixedSizeList)
listSize := larr.DataType().(*arrow.FixedSizeListType).Len()
return p.visitListLike(arr,
- fixedSizeRangeSelector{listSize},
+ fixedSizeRangeSelector{int64(larr.Data().Offset()),
listSize},
0, // defLevelIfEmpty = maxDefLevel (after all
increments)
larr.ListValues())
case arrow.DICTIONARY:
