This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ccfae42b66 ARROW-16502: [Go] Accept missing optional fields when
unmarshalling JSON in StructBuilder
ccfae42b66 is described below
commit ccfae42b66bce8f342eedc2f1c274e45331defc6
Author: Przemysław Kowolik <[email protected]>
AuthorDate: Thu May 12 13:47:56 2022 -0400
ARROW-16502: [Go] Accept missing optional fields when unmarshalling JSON in
StructBuilder
When calling array.StructBuilder.UnmarshalJSON with a JSON object that has
missing optional fields, it fails to decode the JSON object properly and will
panic - but overall it's a common behavior to drop empty/null fields from the
JSON
Fix this by filling all missing optional fields with null values to prevent
builder from panic
Closes #13097 from Kowol/bufix/struct-unmarshal-optional-fields
Authored-by: Przemysław Kowolik <[email protected]>
Signed-off-by: Matthew Topol <[email protected]>
---
go/arrow/array/struct.go | 12 +++++++
go/arrow/array/struct_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 93 insertions(+)
diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go
index 8abf9d0af8..052fabb7fd 100644
--- a/go/arrow/array/struct.go
+++ b/go/arrow/array/struct.go
@@ -334,6 +334,18 @@ func (b *StructBuilder) unmarshalOne(dec *json.Decoder)
error {
return err
}
}
+
+ // Append null values to all optional fields that were not
presented in the json input
+ for _, field := range b.dtype.(*arrow.StructType).Fields() {
+ if !field.Nullable {
+ continue
+ }
+ idx, _ :=
b.dtype.(*arrow.StructType).FieldIdx(field.Name)
+ if _, hasKey := keylist[field.Name]; !hasKey {
+ b.fields[idx].AppendNull()
+ }
+ }
+
// consume '}'
_, err := dec.Token()
return err
diff --git a/go/arrow/array/struct_test.go b/go/arrow/array/struct_test.go
index 70c18a5b18..d029da1f28 100644
--- a/go/arrow/array/struct_test.go
+++ b/go/arrow/array/struct_test.go
@@ -404,3 +404,84 @@ func TestStructArrayNullBitmap(t *testing.T) {
t.Fatalf("invalid string representation:\ngot = %q\nwant= %q",
got, want)
}
}
+
+func TestStructArrayUnmarshalJSONMissingFields(t *testing.T) {
+ pool := memory.NewGoAllocator()
+
+ var (
+ fields = []arrow.Field{
+ {Name: "f1", Type: arrow.PrimitiveTypes.Float64,
Nullable: true},
+ {Name: "f2", Type: arrow.PrimitiveTypes.Int32},
+ {
+ Name: "f3", Type: arrow.StructOf(
+ []arrow.Field{
+ {Name: "f3_1", Type:
arrow.BinaryTypes.String, Nullable: true},
+ {Name: "f3_2", Type:
arrow.BinaryTypes.String, Nullable: true},
+ {Name: "f3_3", Type:
arrow.BinaryTypes.String, Nullable: false},
+ }...,
+ ),
+ },
+ }
+ dtype = arrow.StructOf(fields...)
+ )
+
+ sb := array.NewStructBuilder(pool, dtype)
+ defer sb.Release()
+
+ tests := map[string]struct {
+ jsonInput string
+ want string
+ panic bool
+ }{
+ "missing optional fields": {
+ jsonInput: `[{"f2": 3, "f3": {"f3_3": "test"}}]`,
+ panic: false,
+ want: `{[(null)] [3] {[(null)] [(null)]
["test"]}}`,
+ },
+ "missing required field": {
+ jsonInput: `[{"f2": 3, "f3": {"f3_1": "test"}}]`,
+ panic: true,
+ want: "",
+ },
+ }
+
+ for name, tc := range tests {
+ t.Run(
+ name, func(t *testing.T) {
+
+ var val bool
+
+ if tc.panic {
+ defer func() {
+ e := recover()
+ if e == nil {
+ t.Fatalf("this should
have panicked, but did not; slice value %v", val)
+ }
+ if got, want := e.(string),
"arrow/array: index out of range"; got != want {
+ t.Fatalf("invalid
error. got=%q, want=%q", got, want)
+ }
+ }()
+ } else {
+ defer func() {
+ if e := recover(); e != nil {
+ t.Fatalf("unexpected
panic: %v", e)
+ }
+ }()
+ }
+
+ err := sb.UnmarshalJSON([]byte(tc.jsonInput))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ arr := sb.NewArray().(*array.Struct)
+ defer arr.Release()
+
+ got := arr.String()
+ if got != tc.want {
+ t.Fatalf("invalid string
representation:\ngot = %q\nwant= %q", got, tc.want)
+ }
+ },
+ )
+ }
+}
