This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new c604adb ARROW-3672 & ARROW-3673: [Go] add support for time32 and time64 array c604adb is described below commit c604adbb3ba9eefa9f24093a9b652b9bc66fcd72 Author: alexandreyc <alexandre.crays...@gmail.com> AuthorDate: Thu Nov 15 08:27:21 2018 -0500 ARROW-3672 & ARROW-3673: [Go] add support for time32 and time64 array Hello everyone, My attempt at adding support for time32 and time64 array. Need review because I'm not sure I added all that is needed. Thanks, Alexandre Author: alexandreyc <alexandre.crays...@gmail.com> Closes #2944 from alexandreyc/master and squashes the following commits: e686b19b5 <alexandreyc> update mu to u for microseconds dd2a5d0d1 <alexandreyc> add tests for time 64 array and fix bug ea68cd50f <alexandreyc> add tests for time 32 array 83eb1310f <alexandreyc> add tests for time32 and time64 builders e79db3348 <alexandreyc> add time32 and time64 array 3d6ddcf64 <alexandreyc> Fix missing import in numeric builder template --- go/arrow/array/array.go | 4 +- go/arrow/array/numeric.gen.go | 90 ++++++++++ go/arrow/array/numeric_test.go | 260 ++++++++++++++++++++++++++++ go/arrow/array/numericbuilder.gen.go | 272 ++++++++++++++++++++++++++++++ go/arrow/array/numericbuilder.gen.go.tmpl | 1 + go/arrow/array/numericbuilder_test.go | 229 +++++++++++++++++++++++++ go/arrow/datatype_fixedwidth.go | 34 +++- go/arrow/datatype_fixedwidth_test.go | 2 +- go/arrow/numeric.tmpldata | 24 +++ go/arrow/type_traits_numeric.gen.go | 98 +++++++++++ 10 files changed, 1008 insertions(+), 6 deletions(-) diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index d1dd31d..a225693 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -183,8 +183,8 @@ func init() { arrow.DATE32: unsupportedArrayType, arrow.DATE64: unsupportedArrayType, arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) }, - arrow.TIME32: unsupportedArrayType, - arrow.TIME64: unsupportedArrayType, + arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) }, + arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) }, arrow.INTERVAL: unsupportedArrayType, arrow.DECIMAL: unsupportedArrayType, arrow.LIST: func(data *Data) Interface { return NewListData(data) }, diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go index 6f633ea..1f734c0 100644 --- a/go/arrow/array/numeric.gen.go +++ b/go/arrow/array/numeric.gen.go @@ -519,3 +519,93 @@ func (a *Timestamp) setData(data *Data) { a.values = a.values[beg:end] } } + +// A type which represents an immutable sequence of arrow.Time32 values. +type Time32 struct { + array + values []arrow.Time32 +} + +func NewTime32Data(data *Data) *Time32 { + a := &Time32{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Time32) Value(i int) arrow.Time32 { return a.values[i] } +func (a *Time32) Time32Values() []arrow.Time32 { return a.values } + +func (a *Time32) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Time32) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Time32Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +// A type which represents an immutable sequence of arrow.Time64 values. +type Time64 struct { + array + values []arrow.Time64 +} + +func NewTime64Data(data *Data) *Time64 { + a := &Time64{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Time64) Value(i int) arrow.Time64 { return a.values[i] } +func (a *Time64) Time64Values() []arrow.Time64 { return a.values } + +func (a *Time64) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Time64) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Time64Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go index 352ccd1..9e8267a 100644 --- a/go/arrow/array/numeric_test.go +++ b/go/arrow/array/numeric_test.go @@ -134,3 +134,263 @@ func TestFloat64SliceDataWithNull(t *testing.T) { t.Fatalf("got=%v, want=%v", got, want) } } + +func TestNewTime32Data(t *testing.T) { + data := []arrow.Time32{ + arrow.Time32(1), + arrow.Time32(2), + arrow.Time32(4), + arrow.Time32(8), + arrow.Time32(16), + } + + dtype := arrow.FixedWidthTypes.Time32s + ad := array.NewData(dtype, len(data), + []*memory.Buffer{nil, memory.NewBufferBytes(arrow.Time32Traits.CastToBytes(data))}, + nil, 0, 0, + ) + t32a := array.NewTime32Data(ad) + + assert.Equal(t, len(data), t32a.Len(), "unexpected Len()") + assert.Equal(t, data, t32a.Time32Values(), "unexpected Float64Values()") +} + +func TestTime32SliceData(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer pool.AssertSize(t, 0) + + const ( + beg = 2 + end = 4 + ) + + var ( + vs = []arrow.Time32{ + arrow.Time32(1), + arrow.Time32(2), + arrow.Time32(4), + arrow.Time32(8), + arrow.Time32(16), + } + sub = vs[beg:end] + ) + + dtype := arrow.FixedWidthTypes.Time32s + b := array.NewTime32Builder(pool, dtype.(*arrow.Time32Type)) + defer b.Release() + + for _, v := range vs { + b.Append(v) + } + + arr := b.NewArray().(*array.Time32) + defer arr.Release() + + if got, want := arr.Len(), len(vs); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := arr.Time32Values(), vs; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } + + slice := array.NewSlice(arr, beg, end).(*array.Time32) + defer slice.Release() + + if got, want := slice.Len(), len(sub); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := slice.Time32Values(), sub; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } +} + +func TestTime32SliceDataWithNull(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer pool.AssertSize(t, 0) + + const ( + beg = 2 + end = 5 + ) + + var ( + valids = []bool{true, true, true, false, true, true} + vs = []arrow.Time32{ + arrow.Time32(1), + arrow.Time32(2), + arrow.Time32(3), + arrow.Time32(0), + arrow.Time32(4), + arrow.Time32(5), + } + sub = vs[beg:end] + ) + + dtype := arrow.FixedWidthTypes.Time32s + b := array.NewTime32Builder(pool, dtype.(*arrow.Time32Type)) + defer b.Release() + + b.AppendValues(vs, valids) + + arr := b.NewArray().(*array.Time32) + defer arr.Release() + + if got, want := arr.Len(), len(valids); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := arr.NullN(), 1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := arr.Time32Values(), vs; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } + + slice := array.NewSlice(arr, beg, end).(*array.Time32) + defer slice.Release() + + if got, want := slice.NullN(), 1; got != want { + t.Errorf("got=%d, want=%d", got, want) + } + + if got, want := slice.Len(), len(sub); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := slice.Time32Values(), sub; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } +} + +func TestNewTime64Data(t *testing.T) { + data := []arrow.Time64{ + arrow.Time64(1), + arrow.Time64(2), + arrow.Time64(4), + arrow.Time64(8), + arrow.Time64(16), + } + + dtype := arrow.FixedWidthTypes.Time64us + ad := array.NewData(dtype, len(data), + []*memory.Buffer{nil, memory.NewBufferBytes(arrow.Time64Traits.CastToBytes(data))}, + nil, 0, 0, + ) + t64a := array.NewTime64Data(ad) + + assert.Equal(t, len(data), t64a.Len(), "unexpected Len()") + assert.Equal(t, data, t64a.Time64Values(), "unexpected Float64Values()") +} + +func TestTime64SliceData(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer pool.AssertSize(t, 0) + + const ( + beg = 2 + end = 4 + ) + + var ( + vs = []arrow.Time64{ + arrow.Time64(1), + arrow.Time64(2), + arrow.Time64(4), + arrow.Time64(8), + arrow.Time64(16), + } + sub = vs[beg:end] + ) + + dtype := arrow.FixedWidthTypes.Time64us + b := array.NewTime64Builder(pool, dtype.(*arrow.Time64Type)) + defer b.Release() + + for _, v := range vs { + b.Append(v) + } + + arr := b.NewArray().(*array.Time64) + defer arr.Release() + + if got, want := arr.Len(), len(vs); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := arr.Time64Values(), vs; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } + + slice := array.NewSlice(arr, beg, end).(*array.Time64) + defer slice.Release() + + if got, want := slice.Len(), len(sub); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := slice.Time64Values(), sub; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } +} + +func TestTime64SliceDataWithNull(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer pool.AssertSize(t, 0) + + const ( + beg = 2 + end = 5 + ) + + var ( + valids = []bool{true, true, true, false, true, true} + vs = []arrow.Time64{ + arrow.Time64(1), + arrow.Time64(2), + arrow.Time64(3), + arrow.Time64(0), + arrow.Time64(4), + arrow.Time64(5), + } + sub = vs[beg:end] + ) + + dtype := arrow.FixedWidthTypes.Time64us + b := array.NewTime64Builder(pool, dtype.(*arrow.Time64Type)) + defer b.Release() + + b.AppendValues(vs, valids) + + arr := b.NewArray().(*array.Time64) + defer arr.Release() + + if got, want := arr.Len(), len(valids); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := arr.NullN(), 1; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := arr.Time64Values(), vs; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } + + slice := array.NewSlice(arr, beg, end).(*array.Time64) + defer slice.Release() + + if got, want := slice.NullN(), 1; got != want { + t.Errorf("got=%d, want=%d", got, want) + } + + if got, want := slice.Len(), len(sub); got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + + if got, want := slice.Time64Values(), sub; !reflect.DeepEqual(got, want) { + t.Fatalf("got=%v, want=%v", got, want) + } +} diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index 6585474..3a7dc16 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -1502,6 +1502,276 @@ func (b *TimestampBuilder) newData() (data *Data) { return } +type Time32Builder struct { + builder + + dtype *arrow.Time32Type + data *memory.Buffer + rawData []arrow.Time32 +} + +func NewTime32Builder(mem memory.Allocator, dtype *arrow.Time32Type) *Time32Builder { + return &Time32Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Time32Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Time32Builder) Append(v arrow.Time32) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Time32Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Time32Builder) UnsafeAppend(v arrow.Time32) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Time32Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Time32Builder) AppendValues(v []arrow.Time32, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + b.Reserve(len(v)) + if len(v) > 0 { + arrow.Time32Traits.Copy(b.rawData[b.length:], v) + } + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Time32Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Time32Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Time32Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Time32Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Time32Traits.BytesRequired(n)) + b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder +// so it can be used to build a new array. +func (b *Time32Builder) NewArray() Interface { + return b.NewTime32Array() +} + +// NewTime32Array creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder +// so it can be used to build a new array. +func (b *Time32Builder) NewTime32Array() (a *Time32) { + data := b.newData() + a = NewTime32Data(data) + data.Release() + return +} + +func (b *Time32Builder) newData() (data *Data) { + bytesRequired := arrow.Time32Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Time64Builder struct { + builder + + dtype *arrow.Time64Type + data *memory.Buffer + rawData []arrow.Time64 +} + +func NewTime64Builder(mem memory.Allocator, dtype *arrow.Time64Type) *Time64Builder { + return &Time64Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Time64Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Time64Builder) Append(v arrow.Time64) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Time64Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Time64Builder) UnsafeAppend(v arrow.Time64) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Time64Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Time64Builder) AppendValues(v []arrow.Time64, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + b.Reserve(len(v)) + if len(v) > 0 { + arrow.Time64Traits.Copy(b.rawData[b.length:], v) + } + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Time64Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Time64Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Time64Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Time64Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Time64Traits.BytesRequired(n)) + b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder +// so it can be used to build a new array. +func (b *Time64Builder) NewArray() Interface { + return b.NewTime64Array() +} + +// NewTime64Array creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder +// so it can be used to build a new array. +func (b *Time64Builder) NewTime64Array() (a *Time64) { + data := b.newData() + a = NewTime64Data(data) + data.Release() + return +} + +func (b *Time64Builder) newData() (data *Data) { + bytesRequired := arrow.Time64Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + var ( _ Builder = (*Int64Builder)(nil) _ Builder = (*Uint64Builder)(nil) @@ -1514,4 +1784,6 @@ var ( _ Builder = (*Int8Builder)(nil) _ Builder = (*Uint8Builder)(nil) _ Builder = (*TimestampBuilder)(nil) + _ Builder = (*Time32Builder)(nil) + _ Builder = (*Time64Builder)(nil) ) diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl index 7a3a311..5ae3737 100644 --- a/go/arrow/array/numericbuilder.gen.go.tmpl +++ b/go/arrow/array/numericbuilder.gen.go.tmpl @@ -18,6 +18,7 @@ package array import ( "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/numericbuilder_test.go b/go/arrow/array/numericbuilder_test.go index eb60569..65f3c86 100644 --- a/go/arrow/array/numericbuilder_test.go +++ b/go/arrow/array/numericbuilder_test.go @@ -19,6 +19,7 @@ package array_test import ( "testing" + "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" "github.com/apache/arrow/go/arrow/memory" "github.com/stretchr/testify/assert" @@ -133,3 +134,231 @@ func TestFloat64Builder_Resize(t *testing.T) { ab.Release() } + +func TestNewTime32Builder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time32Type{Unit: arrow.Second} + ab := array.NewTime32Builder(mem, dtype) + + ab.Append(1) + ab.Append(2) + ab.Append(3) + ab.AppendNull() + ab.Append(5) + ab.Append(6) + ab.AppendNull() + ab.Append(8) + ab.Append(9) + ab.Append(10) + + // check state of builder before NewTime32Array + assert.Equal(t, 10, ab.Len(), "unexpected Len()") + assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") + + a := ab.NewTime32Array() + + // check state of builder after NewTime32Array + assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewTime32Array did not reset state") + assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewTime32Array did not reset state") + assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewTime32Array did not reset state") + + // check state of array + assert.Equal(t, 2, a.NullN(), "unexpected null count") + assert.Equal(t, []arrow.Time32{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.Time32Values(), "unexpected Time32Values") + assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity + assert.Len(t, a.Time32Values(), 10, "unexpected length of Time32Values") + + a.Release() + + ab.Append(7) + ab.Append(8) + + a = ab.NewTime32Array() + + assert.Equal(t, 0, a.NullN()) + assert.Equal(t, []arrow.Time32{7, 8}, a.Time32Values()) + assert.Len(t, a.Time32Values(), 2) + + a.Release() +} + +func TestTime32Builder_AppendValues(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time32Type{Unit: arrow.Second} + ab := array.NewTime32Builder(mem, dtype) + + exp := []arrow.Time32{0, 1, 2, 3} + ab.AppendValues(exp, nil) + a := ab.NewTime32Array() + assert.Equal(t, exp, a.Time32Values()) + + a.Release() + ab.Release() +} + +func TestTime32Builder_Empty(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time32Type{Unit: arrow.Second} + ab := array.NewTime32Builder(mem, dtype) + + exp := []arrow.Time32{0, 1, 2, 3} + ab.AppendValues(exp, nil) + a := ab.NewTime32Array() + assert.Equal(t, exp, a.Time32Values()) + a.Release() + + a = ab.NewTime32Array() + assert.Zero(t, a.Len()) + a.Release() + + ab.Release() +} + +func TestTime32Builder_Resize(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time32Type{Unit: arrow.Second} + ab := array.NewTime32Builder(mem, dtype) + + assert.Equal(t, 0, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + ab.Reserve(63) + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + for i := 0; i < 63; i++ { + ab.Append(0) + } + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 63, ab.Len()) + + ab.Resize(5) + assert.Equal(t, 5, ab.Len()) + + ab.Resize(32) + assert.Equal(t, 5, ab.Len()) + + ab.Release() +} + +func TestNewTime64Builder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time64Type{Unit: arrow.Second} + ab := array.NewTime64Builder(mem, dtype) + + ab.Append(1) + ab.Append(2) + ab.Append(3) + ab.AppendNull() + ab.Append(5) + ab.Append(6) + ab.AppendNull() + ab.Append(8) + ab.Append(9) + ab.Append(10) + + // check state of builder before NewTime64Array + assert.Equal(t, 10, ab.Len(), "unexpected Len()") + assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") + + a := ab.NewTime64Array() + + // check state of builder after NewTime64Array + assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewTime64Array did not reset state") + assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewTime64Array did not reset state") + assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewTime64Array did not reset state") + + // check state of array + assert.Equal(t, 2, a.NullN(), "unexpected null count") + assert.Equal(t, []arrow.Time64{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.Time64Values(), "unexpected Time64Values") + assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity + assert.Len(t, a.Time64Values(), 10, "unexpected length of Time64Values") + + a.Release() + + ab.Append(7) + ab.Append(8) + + a = ab.NewTime64Array() + + assert.Equal(t, 0, a.NullN()) + assert.Equal(t, []arrow.Time64{7, 8}, a.Time64Values()) + assert.Len(t, a.Time64Values(), 2) + + a.Release() +} + +func TestTime64Builder_AppendValues(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time64Type{Unit: arrow.Second} + ab := array.NewTime64Builder(mem, dtype) + + exp := []arrow.Time64{0, 1, 2, 3} + ab.AppendValues(exp, nil) + a := ab.NewTime64Array() + assert.Equal(t, exp, a.Time64Values()) + + a.Release() + ab.Release() +} + +func TestTime64Builder_Empty(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time64Type{Unit: arrow.Second} + ab := array.NewTime64Builder(mem, dtype) + + exp := []arrow.Time64{0, 1, 2, 3} + ab.AppendValues(exp, nil) + a := ab.NewTime64Array() + assert.Equal(t, exp, a.Time64Values()) + a.Release() + + a = ab.NewTime64Array() + assert.Zero(t, a.Len()) + a.Release() + + ab.Release() +} + +func TestTime64Builder_Resize(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + dtype := &arrow.Time64Type{Unit: arrow.Second} + ab := array.NewTime64Builder(mem, dtype) + + assert.Equal(t, 0, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + ab.Reserve(63) + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + for i := 0; i < 63; i++ { + ab.Append(0) + } + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 63, ab.Len()) + + ab.Resize(5) + assert.Equal(t, 5, ab.Len()) + + ab.Resize(32) + assert.Equal(t, 5, ab.Len()) + + ab.Release() +} diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index de96ccb..cbddcef 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -26,6 +26,8 @@ func (t *BooleanType) BitWidth() int { return 1 } type ( Timestamp int64 + Time32 int32 + Time64 int64 TimeUnit int ) @@ -36,7 +38,7 @@ const ( Second ) -func (u TimeUnit) String() string { return [...]string{"ns", "µs", "ms", "s"}[uint(u)&3] } +func (u TimeUnit) String() string { return [...]string{"ns", "us", "ms", "s"}[uint(u)&3] } // TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z). // The zero-value is a nanosecond and time zone neutral. Time zone neutral can be @@ -52,10 +54,36 @@ func (*TimestampType) Name() string { return "timestamp" } // BitWidth returns the number of bits required to store a single element of this data type in memory. func (*TimestampType) BitWidth() int { return 64 } +// Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight. +type Time32Type struct { + Unit TimeUnit +} + +func (*Time32Type) ID() Type { return TIME32 } +func (*Time32Type) Name() string { return "time32" } +func (*Time32Type) BitWidth() int { return 32 } + +// Time64Type is encoded as a 64-bit signed integer, representing either microseconds or nanoseconds since midnight. +type Time64Type struct { + Unit TimeUnit +} + +func (*Time64Type) ID() Type { return TIME64 } +func (*Time64Type) Name() string { return "time64" } +func (*Time64Type) BitWidth() int { return 64 } + var ( FixedWidthTypes = struct { - Boolean FixedWidthDataType + Boolean FixedWidthDataType + Time32s FixedWidthDataType + Time32ms FixedWidthDataType + Time64us FixedWidthDataType + Time64ns FixedWidthDataType }{ - Boolean: &BooleanType{}, + Boolean: &BooleanType{}, + Time32s: &Time32Type{Unit: Second}, + Time32ms: &Time32Type{Unit: Millisecond}, + Time64us: &Time64Type{Unit: Microsecond}, + Time64ns: &Time64Type{Unit: Nanosecond}, } ) diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go index 44fbdd1..865f0ae 100644 --- a/go/arrow/datatype_fixedwidth_test.go +++ b/go/arrow/datatype_fixedwidth_test.go @@ -30,7 +30,7 @@ func TestTimeUnit_String(t *testing.T) { exp string }{ {arrow.Nanosecond, "ns"}, - {arrow.Microsecond, "µs"}, + {arrow.Microsecond, "us"}, {arrow.Millisecond, "ms"}, {arrow.Second, "s"}, } diff --git a/go/arrow/numeric.tmpldata b/go/arrow/numeric.tmpldata index ea08d8b..b9e976e 100644 --- a/go/arrow/numeric.tmpldata +++ b/go/arrow/numeric.tmpldata @@ -83,5 +83,29 @@ "Opt": { "Parametric": true } + }, + { + "Name": "Time32", + "name": "time32", + "Type": "Time32", + "QualifiedType": "arrow.Time32", + "InternalType": "int32", + "Default": "0", + "Size": "4", + "Opt": { + "Parametric": true + } + }, + { + "Name": "Time64", + "name": "time64", + "Type": "Time64", + "QualifiedType": "arrow.Time64", + "InternalType": "int64", + "Default": "0", + "Size": "8", + "Opt": { + "Parametric": true + } } ] \ No newline at end of file diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go index b51960b..59ed13f 100644 --- a/go/arrow/type_traits_numeric.gen.go +++ b/go/arrow/type_traits_numeric.gen.go @@ -36,6 +36,8 @@ var ( Int8Traits int8Traits Uint8Traits uint8Traits TimestampTraits timestampTraits + Time32Traits time32Traits + Time64Traits time64Traits ) // Int64 traits @@ -565,3 +567,99 @@ func (timestampTraits) CastToBytes(b []Timestamp) []byte { // Copy copies src to dst. func (timestampTraits) Copy(dst, src []Timestamp) { copy(dst, src) } + +// Time32 traits + +const ( + // Time32SizeBytes specifies the number of bytes required to store a single Time32 in memory + Time32SizeBytes = int(unsafe.Sizeof(Time32(0))) +) + +type time32Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (time32Traits) BytesRequired(n int) int { return Time32SizeBytes * n } + +// PutValue +func (time32Traits) PutValue(b []byte, v Time32) { + binary.LittleEndian.PutUint32(b, uint32(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Time32. +// +// NOTE: len(b) must be a multiple of Time32SizeBytes. +func (time32Traits) CastFromBytes(b []byte) []Time32 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Time32 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Time32SizeBytes + s.Cap = h.Cap / Time32SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (time32Traits) CastToBytes(b []Time32) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Time32SizeBytes + s.Cap = h.Cap * Time32SizeBytes + + return res +} + +// Copy copies src to dst. +func (time32Traits) Copy(dst, src []Time32) { copy(dst, src) } + +// Time64 traits + +const ( + // Time64SizeBytes specifies the number of bytes required to store a single Time64 in memory + Time64SizeBytes = int(unsafe.Sizeof(Time64(0))) +) + +type time64Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (time64Traits) BytesRequired(n int) int { return Time64SizeBytes * n } + +// PutValue +func (time64Traits) PutValue(b []byte, v Time64) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Time64. +// +// NOTE: len(b) must be a multiple of Time64SizeBytes. +func (time64Traits) CastFromBytes(b []byte) []Time64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Time64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Time64SizeBytes + s.Cap = h.Cap / Time64SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (time64Traits) CastToBytes(b []Time64) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Time64SizeBytes + s.Cap = h.Cap * Time64SizeBytes + + return res +} + +// Copy copies src to dst. +func (time64Traits) Copy(dst, src []Time64) { copy(dst, src) }