This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 1a28af36 fix: add nullability to struct type String() (#700)
1a28af36 is described below
commit 1a28af36a6328fc3b98f2917f48dadc3f0641ab2
Author: Willem Jan <[email protected]>
AuthorDate: Tue Mar 10 20:59:20 2026 +0100
fix: add nullability to struct type String() (#700)
### Rationale for this change
We frequently run into panic:
panic: arrow/array: column "colName" type mismatch, got=.., want=...
The got and want will be exactly the same schema, the only difference is
in the nullability of certain struct fields. The only way to figure out
which fields are the problem is to go through the debugger.
The nullability is rendered for other types like map and list, so it
seems like it should be there for struct fields as well.
### What changes are included in this PR?
- Add nullable in String() representation of StructType fields
- Corrected tests that were affected by this
- Some were just adding nullable where it's now added to the string and
it previously was not
- Some appear to be actual fixes to the tests, where the schema's did
not match, but that was not detected in the schema string comparison
because nullability was not included for StructType.
### Are these changes tested?
ran make test, didn't have the parquet test files, so didn't run those.
### Are there any user-facing changes?
Yes, the String() representation for data types is user facing.
Co-authored-by: Willem Jan Noort <[email protected]>
---
arrow/avro/reader_test.go | 5 ++---
arrow/avro/schema_test.go | 10 ++++------
arrow/datatype_nested.go | 3 +++
arrow/util/protobuf_reflect_test.go | 32 ++++++++++++++++----------------
4 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/arrow/avro/reader_test.go b/arrow/avro/reader_test.go
index 9e4f36eb..0eb513b3 100644
--- a/arrow/avro/reader_test.go
+++ b/arrow/avro/reader_test.go
@@ -101,9 +101,8 @@ func TestReader(t *testing.T) {
),
},
arrow.Field{
- Name: "mapfield",
- Type:
arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64),
- Nullable: true,
+ Name: "mapfield",
+ Type:
arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64),
},
arrow.Field{
Name: "arrayField",
diff --git a/arrow/avro/schema_test.go b/arrow/avro/schema_test.go
index d813da00..921e5e81 100644
--- a/arrow/avro/schema_test.go
+++ b/arrow/avro/schema_test.go
@@ -83,9 +83,8 @@ func TestSchemaStringEqual(t *testing.T) {
Name: "person",
Type: arrow.StructOf(
arrow.Field{
- Name: "lastname",
- Type:
arrow.BinaryTypes.String,
- Nullable: true,
+ Name: "lastname",
+ Type:
arrow.BinaryTypes.String,
},
arrow.Field{
Name: "address",
@@ -101,9 +100,8 @@ func TestSchemaStringEqual(t *testing.T) {
),
},
arrow.Field{
- Name: "mapfield",
- Type:
arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64),
- Nullable: true,
+ Name: "mapfield",
+ Type:
arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int64),
},
arrow.Field{
Name: "arrayField",
diff --git a/arrow/datatype_nested.go b/arrow/datatype_nested.go
index 6664c505..fd855d67 100644
--- a/arrow/datatype_nested.go
+++ b/arrow/datatype_nested.go
@@ -444,6 +444,9 @@ func (t *StructType) String() string {
o.WriteString(", ")
}
o.WriteString(fmt.Sprintf("%s: %v", f.Name, f.Type))
+ if f.Nullable {
+ o.WriteString(" nullable")
+ }
}
o.WriteString(">")
return o.String()
diff --git a/arrow/util/protobuf_reflect_test.go
b/arrow/util/protobuf_reflect_test.go
index 38297bb2..35ef1569 100644
--- a/arrow/util/protobuf_reflect_test.go
+++ b/arrow/util/protobuf_reflect_test.go
@@ -119,14 +119,14 @@ func AllTheTypesFixture() Fixture {
- bytes: type=binary, nullable
- double: type=float64, nullable
- enum: type=dictionary<values=utf8, indices=int32, ordered=false>,
nullable
- - message: type=struct<field1: utf8>, nullable
+ - message: type=struct<field1: utf8 nullable>, nullable
- oneofstring: type=utf8, nullable
- - oneofmessage: type=struct<field1: utf8>, nullable
- - any: type=struct<field1: utf8>, nullable
+ - oneofmessage: type=struct<field1: utf8 nullable>, nullable
+ - any: type=struct<field1: utf8 nullable>, nullable
- simple_map: type=map<int32, utf8, items_nullable>, nullable
- - complex_map: type=map<utf8, struct<field1: utf8>, items_nullable>,
nullable
+ - complex_map: type=map<utf8, struct<field1: utf8 nullable>,
items_nullable>, nullable
- simple_list: type=list<item: utf8, nullable>, nullable
- - complex_list: type=list<item: struct<field1: utf8>, nullable>, nullable`
+ - complex_list: type=list<item: struct<field1: utf8 nullable>, nullable>,
nullable`
return Fixture{
msg: &msg,
@@ -255,13 +255,13 @@ func TestGetSchema(t *testing.T) {
- bytes: type=binary, nullable
- double: type=float64, nullable
- enum: type=dictionary<values=utf8, indices=int32, ordered=false>,
nullable
- - message: type=struct<field1: utf8>, nullable
- - oneof: type=dense_union<oneofstring: type=utf8, nullable=0,
oneofmessage: type=struct<field1: utf8>, nullable=1>, nullable
- - any: type=struct<field1: utf8>, nullable
+ - message: type=struct<field1: utf8 nullable>, nullable
+ - oneof: type=dense_union<oneofstring: type=utf8, nullable=0,
oneofmessage: type=struct<field1: utf8 nullable>, nullable=1>, nullable
+ - any: type=struct<field1: utf8 nullable>, nullable
- simple_map: type=map<int32, utf8, items_nullable>, nullable
- - complex_map: type=map<utf8, struct<field1: utf8>, items_nullable>,
nullable
+ - complex_map: type=map<utf8, struct<field1: utf8 nullable>,
items_nullable>, nullable
- simple_list: type=list<item: utf8, nullable>, nullable
- - complex_list: type=list<item: struct<field1: utf8>, nullable>, nullable`
+ - complex_list: type=list<item: struct<field1: utf8 nullable>, nullable>,
nullable`
CheckSchema(t, pmr, want)
excludeComplex := func(pfr *ProtobufFieldReflection) bool {
@@ -386,8 +386,8 @@ func TestExcludedNested(t *testing.T) {
}
schema := `schema:
fields: 2
- - simple_a: type=list<item: struct<field1: utf8>, nullable>, nullable
- - simple_b: type=list<item: struct<field1: utf8>, nullable>, nullable`
+ - simple_a: type=list<item: struct<field1: utf8 nullable>, nullable>,
nullable
+ - simple_b: type=list<item: struct<field1: utf8 nullable>, nullable>,
nullable`
simpleNested := util_message.SimpleNested{
SimpleA: []*util_message.ExampleMessage{&msg},
@@ -415,8 +415,8 @@ func TestExcludedNested(t *testing.T) {
f := AllTheTypesNoAnyFixture()
schema = `schema:
fields: 2
- - all_the_types_no_any_a: type=list<item: struct<str: utf8, int32: int32,
int64: int64, sint32: int32, sin64: int64, uint32: uint32, uint64: uint64,
fixed32: uint32, fixed64: uint64, sfixed32: int32, bool: bool, bytes: binary,
double: float64, enum: dictionary<values=utf8, indices=int32, ordered=false>,
message: struct<field1: utf8>, oneofstring: utf8, oneofmessage: struct<field1:
utf8>, simple_map: map<int32, utf8, items_nullable>, complex_map: map<utf8,
struct<field1: utf8>, items_ [...]
- - all_the_types_no_any_b: type=list<item: struct<str: utf8, int32: int32,
int64: int64, sint32: int32, sin64: int64, uint32: uint32, uint64: uint64,
fixed32: uint32, fixed64: uint64, sfixed32: int32, bool: bool, bytes: binary,
double: float64, enum: dictionary<values=utf8, indices=int32, ordered=false>,
message: struct<field1: utf8>, oneofstring: utf8, oneofmessage: struct<field1:
utf8>, simple_map: map<int32, utf8, items_nullable>, complex_map: map<utf8,
struct<field1: utf8>, items_ [...]
+ - all_the_types_no_any_a: type=list<item: struct<str: utf8 nullable,
int32: int32 nullable, int64: int64 nullable, sint32: int32 nullable, sin64:
int64 nullable, uint32: uint32 nullable, uint64: uint64 nullable, fixed32:
uint32 nullable, fixed64: uint64 nullable, sfixed32: int32 nullable, bool: bool
nullable, bytes: binary nullable, double: float64 nullable, enum:
dictionary<values=utf8, indices=int32, ordered=false> nullable, message:
struct<field1: utf8 nullable> nullable, oneofstr [...]
+ - all_the_types_no_any_b: type=list<item: struct<str: utf8 nullable,
int32: int32 nullable, int64: int64 nullable, sint32: int32 nullable, sin64:
int64 nullable, uint32: uint32 nullable, uint64: uint64 nullable, fixed32:
uint32 nullable, fixed64: uint64 nullable, sfixed32: int32 nullable, bool: bool
nullable, bytes: binary nullable, double: float64 nullable, enum:
dictionary<values=utf8, indices=int32, ordered=false> nullable, message:
struct<field1: utf8 nullable> nullable, oneofstr [...]
complexNested := util_message.ComplexNested{
AllTheTypesNoAnyA:
[]*util_message.AllTheTypesNoAny{f.msg.(*util_message.AllTheTypesNoAny)},
@@ -445,8 +445,8 @@ func TestExcludedNested(t *testing.T) {
schema = `schema:
fields: 2
- - complex_nested: type=struct<all_the_types_no_any_a: list<item:
struct<str: utf8, int32: int32, int64: int64, sint32: int32, sin64: int64,
uint32: uint32, uint64: uint64, fixed32: uint32, fixed64: uint64, sfixed32:
int32, bool: bool, bytes: binary, double: float64, enum:
dictionary<values=utf8, indices=int32, ordered=false>, message: struct<field1:
utf8>, oneofstring: utf8, oneofmessage: struct<field1: utf8>, simple_map:
map<int32, utf8, items_nullable>, complex_map: map<utf8, struc [...]
- - simple_nested: type=struct<simple_a: list<item: struct<field1: utf8>,
nullable>, simple_b: list<item: struct<field1: utf8>, nullable>>, nullable`
+ - complex_nested: type=struct<all_the_types_no_any_a: list<item:
struct<str: utf8 nullable, int32: int32 nullable, int64: int64 nullable,
sint32: int32 nullable, sin64: int64 nullable, uint32: uint32 nullable, uint64:
uint64 nullable, fixed32: uint32 nullable, fixed64: uint64 nullable, sfixed32:
int32 nullable, bool: bool nullable, bytes: binary nullable, double: float64
nullable, enum: dictionary<values=utf8, indices=int32, ordered=false> nullable,
message: struct<field1: utf8 nulla [...]
+ - simple_nested: type=struct<simple_a: list<item: struct<field1: utf8
nullable>, nullable> nullable, simple_b: list<item: struct<field1: utf8
nullable>, nullable> nullable>, nullable`
deepNested := util_message.DeepNested{
ComplexNested: &complexNested,
