This is an automated email from the ASF dual-hosted git repository. kszucs pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
commit e3bf93a1beb8b1f2671872cddf223570b8f7f52f Author: Sebastien Binet <bi...@cern.ch> AuthorDate: Wed Oct 2 10:32:42 2019 +0200 ARROW-6752: [Go] make Null array implement Stringer, add tests for Nu… …ll arrays Closes #5555 from sbinet/issue-6752 and squashes the following commits: 18b5387cc <Sebastien Binet> ARROW-6752: make Null array implement Stringer, add tests for Null arrays Authored-by: Sebastien Binet <bi...@cern.ch> Signed-off-by: Sebastien Binet <bi...@cern.ch> --- go/arrow/array/compare_test.go | 14 +++++----- go/arrow/array/null.go | 14 ++++++++++ go/arrow/internal/arrdata/arrdata.go | 52 ++++++++++++++++++++++++++++++++++++ go/arrow/internal/arrjson/arrjson.go | 13 +++++++++ 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go index e9927f0..3ed326b 100644 --- a/go/arrow/array/compare_test.go +++ b/go/arrow/array/compare_test.go @@ -44,7 +44,7 @@ func TestArrayEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArrayEqual(sub1, sub2) { + if array.ArrayEqual(sub1, sub2) && name != "nulls" { t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -73,7 +73,7 @@ func TestArraySliceEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) { + if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) && name != "nulls" { t.Fatalf("non-identical slices should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -99,7 +99,7 @@ func TestArrayApproxEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArrayApproxEqual(sub1, sub2) { + if array.ArrayApproxEqual(sub1, sub2) && name != "nulls" { t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -489,7 +489,7 @@ func TestRecordEqual(t *testing.T) { t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0) } - if array.RecordEqual(rec0, rec1) { + if array.RecordEqual(rec0, rec1) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1) } @@ -498,7 +498,7 @@ func TestRecordEqual(t *testing.T) { sub01 := rec0.NewSlice(1, recs[0].NumRows()) defer sub01.Release() - if array.RecordEqual(sub00, sub01) { + if array.RecordEqual(sub00, sub01) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01) } }) @@ -514,7 +514,7 @@ func TestRecordApproxEqual(t *testing.T) { t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0) } - if array.RecordApproxEqual(rec0, rec1) { + if array.RecordApproxEqual(rec0, rec1) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1) } @@ -523,7 +523,7 @@ func TestRecordApproxEqual(t *testing.T) { sub01 := rec0.NewSlice(1, recs[0].NumRows()) defer sub01.Release() - if array.RecordApproxEqual(sub00, sub01) { + if array.RecordApproxEqual(sub00, sub01) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01) } }) diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go index 346445d..56d7fa4 100644 --- a/go/arrow/array/null.go +++ b/go/arrow/array/null.go @@ -17,6 +17,7 @@ package array import ( + "strings" "sync/atomic" "github.com/apache/arrow/go/arrow" @@ -53,6 +54,19 @@ func NewNullData(data *Data) *Null { return a } +func (a *Null) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + o.WriteString("(null)") + } + o.WriteString("]") + return o.String() +} + func (a *Null) setData(data *Data) { a.array.setData(data) a.array.nullBitmapBytes = nil diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index c249236..40dee3b 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -34,6 +34,7 @@ var ( ) func init() { + Records["nulls"] = makeNullRecords() Records["primitives"] = makePrimitiveRecords() Records["structs"] = makeStructsRecords() Records["lists"] = makeListsRecords() @@ -51,6 +52,49 @@ func init() { sort.Strings(RecordNames) } +func makeNullRecords() []array.Record { + mem := memory.NewGoAllocator() + + meta := arrow.NewMetadata( + []string{"k1", "k2", "k3"}, + []string{"v1", "v2", "v3"}, + ) + + schema := arrow.NewSchema( + []arrow.Field{ + arrow.Field{Name: "nulls", Type: arrow.Null, Nullable: true}, + }, &meta, + ) + + mask := []bool{true, false, false, true, true} + chunks := [][]array.Interface{ + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + } + + defer func() { + for _, chunk := range chunks { + for _, col := range chunk { + col.Release() + } + } + }() + + recs := make([]array.Record, len(chunks)) + for i, chunk := range chunks { + recs[i] = array.NewRecord(schema, chunk, -1) + } + + return recs +} + func makePrimitiveRecords() []array.Record { mem := memory.NewGoAllocator() @@ -362,6 +406,7 @@ func makeStringsRecords() []array.Record { } type ( + nullT struct{} time32s arrow.Time32 time32ms arrow.Time32 time64ns arrow.Time64 @@ -372,6 +417,10 @@ type ( timestamp_ns arrow.Timestamp ) +var ( + null nullT +) + func makeFixedWidthTypesRecords() []array.Record { mem := memory.NewGoAllocator() schema := arrow.NewSchema( @@ -650,6 +699,9 @@ func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface } switch a := a.(type) { + case []nullT: + return array.NewNull(len(a)) + case []bool: bldr := array.NewBooleanBuilder(mem) defer bldr.Release() diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index 11cc3d0..2b2bcaf 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -65,6 +65,8 @@ type dataType struct { func dtypeToJSON(dt arrow.DataType) dataType { switch dt := dt.(type) { + case *arrow.NullType: + return dataType{Name: "null"} case *arrow.BooleanType: return dataType{Name: "bool"} case *arrow.Int8Type: @@ -155,6 +157,8 @@ func dtypeToJSON(dt arrow.DataType) dataType { func dtypeFromJSON(dt dataType, children []Field) arrow.DataType { switch dt.Name { + case "null": + return arrow.Null case "bool": return arrow.FixedWidthTypes.Boolean case "int": @@ -364,6 +368,9 @@ func arraysToJSON(schema *arrow.Schema, arrs []array.Interface) []Array { func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Interface { switch dt := dt.(type) { + case *arrow.NullType: + return array.NewNull(arr.Count) + case *arrow.BooleanType: bldr := array.NewBooleanBuilder(mem) defer bldr.Release() @@ -618,6 +625,12 @@ func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Int func arrayToJSON(field arrow.Field, arr array.Interface) Array { switch arr := arr.(type) { + case *array.Null: + return Array{ + Name: field.Name, + Count: arr.Len(), + } + case *array.Boolean: return Array{ Name: field.Name,