This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 97f780be fix(arrow/extensions): avoid child values for missing variant
lists (#1293)
97f780be is described below
commit 97f780beb5a9129cbcf2c14193bc5830cf4ad035
Author: matanper <[email protected]>
AuthorDate: Tue Sep 8 20:21:09 2026 +0300
fix(arrow/extensions): avoid child values for missing variant lists (#1293)
### Rationale for this change
Writing shredded VARIANT values to Parquet fails with `lists with
non-zero length null components are not supported` when a nested list
has a null, missing, or scalar parent between populated rows. For
example:
```json
{"obj":{"items":[1]}}
{"obj":null}
{"obj":{"items":[2]}}
```
`shreddedObjBuilder.AppendMissing` marks the parent struct null, but
recursively calling `shreddedArrayBuilder.AppendMissing` adds a valid
list and a placeholder child element. The Parquet writer rejects the
resulting skipped child range.
Fixes #1292.
### What changes are included in this PR?
Make `shreddedArrayBuilder.AppendMissing` append a null list without
adding child elements. The production change is two added lines
(including a comment) and four removed lines.
Add a self-contained Parquet write/read regression for null, missing,
and scalar parent values between populated lists. It verifies all
decoded JSON values, row count, and checked-allocator cleanup. No
external fixtures are required.
### Are these changes tested?
All three regression cases reproduce the original write error before the
fix and pass afterward.
- `go test ./parquet/pqarrow -run
TestShreddedVariantListsUnderMissingParent -count=1`
- `go test ./arrow/extensions ./parquet/variant`
### Are there any user-facing changes?
These valid shredded VARIANT values can now be written to Parquet while
preserving null, missing, and scalar parent values. No public API
changes.
---
arrow/extensions/variant.go | 6 ++---
parquet/pqarrow/variant_test.go | 55 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 57 insertions(+), 4 deletions(-)
diff --git a/arrow/extensions/variant.go b/arrow/extensions/variant.go
index f4474133..270f1db3 100644
--- a/arrow/extensions/variant.go
+++ b/arrow/extensions/variant.go
@@ -1456,10 +1456,8 @@ type shreddedArrayBuilder struct {
}
func (s *shreddedArrayBuilder) AppendMissing() {
- s.listBldr.Append(true)
- s.elemBldr.Append(true)
- s.valueBldr.AppendNull()
- s.typedBldr.AppendMissing()
+ // Missing lists must not add child elements beneath a null parent
struct.
+ s.listBldr.AppendNull()
}
func (b *shreddedArrayBuilder) tryTyped(v variant.Value) (residual []byte) {
diff --git a/parquet/pqarrow/variant_test.go b/parquet/pqarrow/variant_test.go
index 81fa246b..078a9dc4 100644
--- a/parquet/pqarrow/variant_test.go
+++ b/parquet/pqarrow/variant_test.go
@@ -17,6 +17,7 @@
package pqarrow_test
import (
+ "bytes"
"context"
"fmt"
"io"
@@ -29,6 +30,7 @@ import (
"unsafe"
"github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/endian"
"github.com/apache/arrow-go/v18/arrow/extensions"
"github.com/apache/arrow-go/v18/arrow/memory"
@@ -36,6 +38,7 @@ import (
"github.com/apache/arrow-go/v18/parquet"
"github.com/apache/arrow-go/v18/parquet/pqarrow"
"github.com/apache/arrow-go/v18/parquet/variant"
+ "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@@ -324,3 +327,55 @@ func (s *ShreddedVariantTestSuite) TestErrorCases() {
func TestShreddedVariantExamples(t *testing.T) {
suite.Run(t, &ShreddedVariantTestSuite{generate: false})
}
+
+// A missing list below a null struct must not introduce skipped child values.
+func TestShreddedVariantListsUnderMissingParent(t *testing.T) {
+ for name, middle := range map[string]string{
+ "null_parent": `{"obj":null}`,
+ "missing_parent": `{}`,
+ "scalar_parent": `{"obj":"other"}`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ alloc :=
memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer alloc.AssertSize(t, 0)
+ typ := extensions.NewShreddedVariantType(arrow.StructOf(
+ arrow.Field{Name: "obj", Nullable: true, Type:
arrow.StructOf(
+ arrow.Field{Name: "items", Nullable:
true, Type: arrow.ListOf(arrow.PrimitiveTypes.Int64)},
+ )},
+ ))
+ builder := extensions.NewVariantBuilder(alloc, typ)
+ defer builder.Release()
+ input := []string{`{"obj":{"items":[1]}}`, middle,
`{"obj":{"items":[2]}}`}
+ require.NoError(t,
builder.UnmarshalJSON([]byte("["+strings.Join(input, ",")+"]")))
+ values := builder.NewArray()
+ defer values.Release()
+ schema := arrow.NewSchema([]arrow.Field{{Name: "event",
Type: typ, Nullable: true}}, nil)
+ rec := array.NewRecordBatch(schema,
[]arrow.Array{values}, int64(len(input)))
+ defer rec.Release()
+ tbl := array.NewTableFromRecords(schema,
[]arrow.RecordBatch{rec})
+ defer tbl.Release()
+ var encoded bytes.Buffer
+ require.NoError(t, pqarrow.WriteTable(tbl, &encoded,
1024, parquet.NewWriterProperties(parquet.WithAllocator(alloc)),
pqarrow.DefaultWriterProps()))
+ result, err := pqarrow.ReadTable(t.Context(),
bytes.NewReader(encoded.Bytes()), nil, pqarrow.ArrowReadProperties{}, alloc)
+ require.NoError(t, err)
+ defer result.Release()
+ require.EqualValues(t, len(input), result.NumRows())
+ reader := array.NewTableReader(result, 1024)
+ defer reader.Release()
+ row := 0
+ for reader.Next() {
+ variants :=
reader.RecordBatch().Column(0).(*extensions.VariantArray)
+ for i := 0; i < variants.Len(); i++ {
+ value, err := variants.Value(i)
+ require.NoError(t, err)
+ decoded, err := value.MarshalJSON()
+ require.NoError(t, err)
+ require.JSONEq(t, input[row],
string(decoded))
+ row++
+ }
+ }
+ require.NoError(t, reader.Err())
+ require.Len(t, input, row)
+ })
+ }
+}
