This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new bcaf453 chore(arrow): remove most lock copies (#362)
bcaf453 is described below
commit bcaf45378899f6bec22b7c37675afad9a172b3a5
Author: Matt Topol <[email protected]>
AuthorDate: Wed Apr 30 10:52:18 2025 -0400
chore(arrow): remove most lock copies (#362)
### Rationale for this change
#326 moved to using `atomic.Int64{]` for the refcount, as a result we
found many places that we were copying the value. For linting reasons we
should remove those.
### What changes are included in this PR?
Utilizing generics to clean up the dictionary array code significantly,
along with removing most of the cases where we were copying the refcount
which contains a lock.
### Are these changes tested?
Yes, the existing unit tests cover it all.
### Are there any user-facing changes?
Nope.
---
arrow/array/dictionary.go | 720 ++++++++++-------------------------------
arrow/array/extension.go | 2 +-
arrow/array/interval.go | 3 +-
arrow/array/timestamp.go | 4 +-
arrow/array/union.go | 8 +-
ci/docker/debian-12.dockerfile | 4 +-
6 files changed, 185 insertions(+), 556 deletions(-)
diff --git a/arrow/array/dictionary.go b/arrow/array/dictionary.go
index 88df0cb..4ddb5d4 100644
--- a/arrow/array/dictionary.go
+++ b/arrow/array/dictionary.go
@@ -27,8 +27,6 @@ import (
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/bitutil"
"github.com/apache/arrow-go/v18/arrow/decimal"
- "github.com/apache/arrow-go/v18/arrow/decimal128"
- "github.com/apache/arrow-go/v18/arrow/decimal256"
"github.com/apache/arrow-go/v18/arrow/float16"
"github.com/apache/arrow-go/v18/arrow/internal/debug"
"github.com/apache/arrow-go/v18/arrow/memory"
@@ -425,6 +423,73 @@ type dictionaryBuilder struct {
idxBuilder IndexBuilder
}
+func createDictBuilder[T arrow.ValueType](mem memory.Allocator, idxbldr
IndexBuilder, memo hashing.MemoTable, dt *arrow.DictionaryType, init
arrow.Array) DictionaryBuilder {
+ ret := &dictBuilder[T]{
+ dictionaryBuilder: dictionaryBuilder{
+ builder: builder{mem: mem},
+ idxBuilder: idxbldr,
+ memoTable: memo,
+ dt: dt,
+ },
+ }
+ ret.builder.refCount.Add(1)
+
+ if init != nil {
+ if err := ret.InsertDictValues(init.(arrValues[T])); err != nil
{
+ panic(err)
+ }
+ }
+ return ret
+}
+
+func createBinaryDictBuilder(mem memory.Allocator, idxbldr IndexBuilder, memo
hashing.MemoTable, dt *arrow.DictionaryType, init arrow.Array)
DictionaryBuilder {
+ ret := &BinaryDictionaryBuilder{
+ dictionaryBuilder: dictionaryBuilder{
+ builder: builder{mem: mem},
+ idxBuilder: idxbldr,
+ memoTable: memo,
+ dt: dt,
+ },
+ }
+ ret.builder.refCount.Add(1)
+
+ if init != nil {
+ switch v := init.(type) {
+ case *String:
+ if err := ret.InsertStringDictValues(v); err != nil {
+ panic(err)
+ }
+ case *Binary:
+ if err := ret.InsertDictValues(v); err != nil {
+ panic(err)
+ }
+ }
+ }
+ return ret
+}
+
+func createFixedSizeDictBuilder[T fsbType](mem memory.Allocator, idxbldr
IndexBuilder, memo hashing.MemoTable, dt *arrow.DictionaryType, init
arrow.Array) DictionaryBuilder {
+ var z T
+ ret := &fixedSizeDictionaryBuilder[T]{
+ dictionaryBuilder: dictionaryBuilder{
+ builder: builder{mem: mem},
+ idxBuilder: idxbldr,
+ memoTable: memo,
+ dt: dt,
+ },
+ byteWidth: int(unsafe.Sizeof(z)),
+ }
+ ret.builder.refCount.Add(1)
+
+ if init != nil {
+ if err := ret.InsertDictValues(init.(arrValues[T])); err != nil
{
+ panic(err)
+ }
+ }
+
+ return ret
+}
+
// NewDictionaryBuilderWithDict initializes a dictionary builder and inserts
the values from `init` as the first
// values in the dictionary, but does not insert them as values into the array.
func NewDictionaryBuilderWithDict(mem memory.Allocator, dt
*arrow.DictionaryType, init arrow.Array) DictionaryBuilder {
@@ -442,128 +507,55 @@ func NewDictionaryBuilderWithDict(mem memory.Allocator,
dt *arrow.DictionaryType
panic(fmt.Errorf("arrow/array: unsupported builder for value
type of %T", dt))
}
- bldr := dictionaryBuilder{
- builder: builder{mem: mem},
- idxBuilder: idxbldr,
- memoTable: memo,
- dt: dt,
- }
-
- bldr.builder.refCount.Add(1)
-
switch dt.ValueType.ID() {
case arrow.NULL:
- ret := &NullDictionaryBuilder{bldr}
+ ret := &NullDictionaryBuilder{
+ dictionaryBuilder: dictionaryBuilder{
+ builder: builder{mem: mem},
+ idxBuilder: idxbldr,
+ memoTable: memo,
+ dt: dt,
+ },
+ }
+ ret.builder.refCount.Add(1)
debug.Assert(init == nil, "arrow/array: doesn't make sense to
init a null dictionary")
return ret
case arrow.UINT8:
- ret := &Uint8DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Uint8)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[uint8](mem, idxbldr, memo, dt, init)
case arrow.INT8:
- ret := &Int8DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Int8)); err != nil
{
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[int8](mem, idxbldr, memo, dt, init)
case arrow.UINT16:
- ret := &Uint16DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Uint16)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[uint16](mem, idxbldr, memo, dt, init)
case arrow.INT16:
- ret := &Int16DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Int16)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[int16](mem, idxbldr, memo, dt, init)
case arrow.UINT32:
- ret := &Uint32DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Uint32)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[uint32](mem, idxbldr, memo, dt, init)
case arrow.INT32:
- ret := &Int32DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Int32)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[int32](mem, idxbldr, memo, dt, init)
case arrow.UINT64:
- ret := &Uint64DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Uint64)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[uint64](mem, idxbldr, memo, dt, init)
case arrow.INT64:
- ret := &Int64DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Int64)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[int64](mem, idxbldr, memo, dt, init)
case arrow.FLOAT16:
- ret := &Float16DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Float16)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[float16.Num](mem, idxbldr, memo, dt,
init)
case arrow.FLOAT32:
- ret := &Float32DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Float32)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[float32](mem, idxbldr, memo, dt, init)
case arrow.FLOAT64:
- ret := &Float64DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Float64)); err !=
nil {
- panic(err)
- }
- }
- return ret
- case arrow.STRING:
- ret := &BinaryDictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertStringDictValues(init.(*String));
err != nil {
- panic(err)
- }
- }
- return ret
- case arrow.BINARY:
- ret := &BinaryDictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Binary)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[float64](mem, idxbldr, memo, dt, init)
+ case arrow.STRING, arrow.BINARY:
+ return createBinaryDictBuilder(mem, idxbldr, memo, dt, init)
case arrow.FIXED_SIZE_BINARY:
ret := &FixedSizeBinaryDictionaryBuilder{
- bldr,
dt.ValueType.(*arrow.FixedSizeBinaryType).ByteWidth,
+ dictionaryBuilder: dictionaryBuilder{
+ builder: builder{mem: mem},
+ idxBuilder: idxbldr,
+ memoTable: memo,
+ dt: dt,
+ },
+ byteWidth:
dt.ValueType.(*arrow.FixedSizeBinaryType).ByteWidth,
}
+ ret.builder.refCount.Add(1)
+
if init != nil {
if err = ret.InsertDictValues(init.(*FixedSizeBinary));
err != nil {
panic(err)
@@ -571,93 +563,27 @@ func NewDictionaryBuilderWithDict(mem memory.Allocator,
dt *arrow.DictionaryType
}
return ret
case arrow.DATE32:
- ret := &Date32DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Date32)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.Date32](mem, idxbldr, memo, dt,
init)
case arrow.DATE64:
- ret := &Date64DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Date64)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.Date64](mem, idxbldr, memo, dt,
init)
case arrow.TIMESTAMP:
- ret := &TimestampDictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Timestamp)); err
!= nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.Timestamp](mem, idxbldr, memo,
dt, init)
case arrow.TIME32:
- ret := &Time32DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Time32)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.Time32](mem, idxbldr, memo, dt,
init)
case arrow.TIME64:
- ret := &Time64DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Time64)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.Time64](mem, idxbldr, memo, dt,
init)
case arrow.INTERVAL_MONTHS:
- ret := &MonthIntervalDictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*MonthInterval));
err != nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.MonthInterval](mem, idxbldr,
memo, dt, init)
case arrow.INTERVAL_DAY_TIME:
- ret := &DayTimeDictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*DayTimeInterval));
err != nil {
- panic(err)
- }
- }
- return ret
+ return createFixedSizeDictBuilder[arrow.DayTimeInterval](mem,
idxbldr, memo, dt, init)
case arrow.DECIMAL32:
- ret := &Decimal32DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Decimal32)); err
!= nil {
- panic(err)
- }
- }
- return ret
+ return createFixedSizeDictBuilder[decimal.Decimal32](mem,
idxbldr, memo, dt, init)
case arrow.DECIMAL64:
- ret := &Decimal64DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Decimal64)); err
!= nil {
- panic(err)
- }
- }
- return ret
+ return createFixedSizeDictBuilder[decimal.Decimal64](mem,
idxbldr, memo, dt, init)
case arrow.DECIMAL128:
- ret := &Decimal128DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Decimal128)); err
!= nil {
- panic(err)
- }
- }
- return ret
+ return createFixedSizeDictBuilder[decimal.Decimal128](mem,
idxbldr, memo, dt, init)
case arrow.DECIMAL256:
- ret := &Decimal256DictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Decimal256)); err
!= nil {
- panic(err)
- }
- }
- return ret
+ return createFixedSizeDictBuilder[decimal.Decimal256](mem,
idxbldr, memo, dt, init)
case arrow.LIST:
case arrow.STRUCT:
case arrow.SPARSE_UNION:
@@ -667,24 +593,12 @@ func NewDictionaryBuilderWithDict(mem memory.Allocator,
dt *arrow.DictionaryType
case arrow.EXTENSION:
case arrow.FIXED_SIZE_LIST:
case arrow.DURATION:
- ret := &DurationDictionaryBuilder{bldr}
- if init != nil {
- if err = ret.InsertDictValues(init.(*Duration)); err !=
nil {
- panic(err)
- }
- }
- return ret
+ return createDictBuilder[arrow.Duration](mem, idxbldr, memo,
dt, init)
case arrow.LARGE_STRING:
case arrow.LARGE_BINARY:
case arrow.LARGE_LIST:
case arrow.INTERVAL_MONTH_DAY_NANO:
- ret := &MonthDayNanoDictionaryBuilder{bldr}
- if init != nil {
- if err =
ret.InsertDictValues(init.(*MonthDayNanoInterval)); err != nil {
- panic(err)
- }
- }
- return ret
+ return
createFixedSizeDictBuilder[arrow.MonthDayNanoInterval](mem, idxbldr, memo, dt,
init)
}
panic("arrow/array: unimplemented dictionary key type")
@@ -1072,55 +986,20 @@ func (b *NullDictionaryBuilder) AppendArray(arr
arrow.Array) error {
return nil
}
-type Int8DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Int8DictionaryBuilder) Append(v int8) error { return b.appendValue(v)
}
-func (b *Int8DictionaryBuilder) InsertDictValues(arr *Int8) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type Uint8DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Uint8DictionaryBuilder) Append(v uint8) error { return
b.appendValue(v) }
-func (b *Uint8DictionaryBuilder) InsertDictValues(arr *Uint8) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type Int16DictionaryBuilder struct {
+type dictBuilder[T arrow.ValueType] struct {
dictionaryBuilder
}
-func (b *Int16DictionaryBuilder) Append(v int16) error { return
b.appendValue(v) }
-func (b *Int16DictionaryBuilder) InsertDictValues(arr *Int16) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
+func (b *dictBuilder[T]) Append(v T) error {
+ return b.appendValue(v)
}
-type Uint16DictionaryBuilder struct {
- dictionaryBuilder
+type arrValues[T arrow.ValueType] interface {
+ Values() []T
}
-func (b *Uint16DictionaryBuilder) Append(v uint16) error { return
b.appendValue(v) }
-func (b *Uint16DictionaryBuilder) InsertDictValues(arr *Uint16) (err error) {
- for _, v := range arr.values {
+func (b *dictBuilder[T]) InsertDictValues(arr arrValues[T]) (err error) {
+ for _, v := range arr.Values() {
if err = b.insertDictValue(v); err != nil {
break
}
@@ -1128,204 +1007,30 @@ func (b *Uint16DictionaryBuilder) InsertDictValues(arr
*Uint16) (err error) {
return
}
-type Int32DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Int32DictionaryBuilder) Append(v int32) error { return
b.appendValue(v) }
-func (b *Int32DictionaryBuilder) InsertDictValues(arr *Int32) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type Uint32DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Uint32DictionaryBuilder) Append(v uint32) error { return
b.appendValue(v) }
-func (b *Uint32DictionaryBuilder) InsertDictValues(arr *Uint32) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type Int64DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Int64DictionaryBuilder) Append(v int64) error { return
b.appendValue(v) }
-func (b *Int64DictionaryBuilder) InsertDictValues(arr *Int64) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type Uint64DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Uint64DictionaryBuilder) Append(v uint64) error { return
b.appendValue(v) }
-func (b *Uint64DictionaryBuilder) InsertDictValues(arr *Uint64) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type DurationDictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *DurationDictionaryBuilder) Append(v arrow.Duration) error { return
b.appendValue(int64(v)) }
-func (b *DurationDictionaryBuilder) InsertDictValues(arr *Duration) (err
error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int64(v)); err != nil {
- break
- }
- }
- return
-}
-
-type TimestampDictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *TimestampDictionaryBuilder) Append(v arrow.Timestamp) error { return
b.appendValue(int64(v)) }
-func (b *TimestampDictionaryBuilder) InsertDictValues(arr *Timestamp) (err
error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int64(v)); err != nil {
- break
- }
- }
- return
-}
-
-type Time32DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Time32DictionaryBuilder) Append(v arrow.Time32) error { return
b.appendValue(int32(v)) }
-func (b *Time32DictionaryBuilder) InsertDictValues(arr *Time32) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int32(v)); err != nil {
- break
- }
- }
- return
-}
-
-type Time64DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Time64DictionaryBuilder) Append(v arrow.Time64) error { return
b.appendValue(int64(v)) }
-func (b *Time64DictionaryBuilder) InsertDictValues(arr *Time64) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int64(v)); err != nil {
- break
- }
- }
- return
-}
-
-type Date32DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Date32DictionaryBuilder) Append(v arrow.Date32) error { return
b.appendValue(int32(v)) }
-func (b *Date32DictionaryBuilder) InsertDictValues(arr *Date32) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int32(v)); err != nil {
- break
- }
- }
- return
-}
-
-type Date64DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Date64DictionaryBuilder) Append(v arrow.Date64) error { return
b.appendValue(int64(v)) }
-func (b *Date64DictionaryBuilder) InsertDictValues(arr *Date64) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int64(v)); err != nil {
- break
- }
- }
- return
-}
-
-type MonthIntervalDictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *MonthIntervalDictionaryBuilder) Append(v arrow.MonthInterval) error {
- return b.appendValue(int32(v))
-}
-
-func (b *MonthIntervalDictionaryBuilder) InsertDictValues(arr *MonthInterval)
(err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(int32(v)); err != nil {
- break
- }
- }
- return
-}
-
-type Float16DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Float16DictionaryBuilder) Append(v float16.Num) error { return
b.appendValue(v.Uint16()) }
-func (b *Float16DictionaryBuilder) InsertDictValues(arr *Float16) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v.Uint16()); err != nil {
- break
- }
- }
- return
-}
-
-type Float32DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Float32DictionaryBuilder) Append(v float32) error { return
b.appendValue(v) }
-func (b *Float32DictionaryBuilder) InsertDictValues(arr *Float32) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
-
-type Float64DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Float64DictionaryBuilder) Append(v float64) error { return
b.appendValue(v) }
-func (b *Float64DictionaryBuilder) InsertDictValues(arr *Float64) (err error) {
- for _, v := range arr.values {
- if err = b.insertDictValue(v); err != nil {
- break
- }
- }
- return
-}
+type Int8DictionaryBuilder = dictBuilder[int8]
+type Uint8DictionaryBuilder = dictBuilder[uint8]
+type Int16DictionaryBuilder = dictBuilder[int16]
+type Uint16DictionaryBuilder = dictBuilder[uint16]
+type Int32DictionaryBuilder = dictBuilder[int32]
+type Uint32DictionaryBuilder = dictBuilder[uint32]
+type Int64DictionaryBuilder = dictBuilder[int64]
+type Uint64DictionaryBuilder = dictBuilder[uint64]
+type Float16DictionaryBuilder = dictBuilder[float16.Num]
+type Float32DictionaryBuilder = dictBuilder[float32]
+type Float64DictionaryBuilder = dictBuilder[float64]
+type DurationDictionaryBuilder = dictBuilder[arrow.Duration]
+type TimestampDictionaryBuilder = dictBuilder[arrow.Timestamp]
+type Time32DictionaryBuilder = dictBuilder[arrow.Time32]
+type Time64DictionaryBuilder = dictBuilder[arrow.Time64]
+type Date32DictionaryBuilder = dictBuilder[arrow.Date32]
+type Date64DictionaryBuilder = dictBuilder[arrow.Date64]
+type MonthIntervalDictionaryBuilder = dictBuilder[arrow.MonthInterval]
+type DayTimeDictionaryBuilder =
fixedSizeDictionaryBuilder[arrow.DayTimeInterval]
+type Decimal32DictionaryBuilder = fixedSizeDictionaryBuilder[decimal.Decimal32]
+type Decimal64DictionaryBuilder = fixedSizeDictionaryBuilder[decimal.Decimal64]
+type Decimal128DictionaryBuilder =
fixedSizeDictionaryBuilder[decimal.Decimal128]
+type Decimal256DictionaryBuilder =
fixedSizeDictionaryBuilder[decimal.Decimal256]
+type MonthDayNanoDictionaryBuilder =
fixedSizeDictionaryBuilder[arrow.MonthDayNanoInterval]
type BinaryDictionaryBuilder struct {
dictionaryBuilder
@@ -1402,140 +1107,61 @@ func (b *BinaryDictionaryBuilder) ValueStr(i int)
string {
return string(b.Value(i))
}
-type FixedSizeBinaryDictionaryBuilder struct {
- dictionaryBuilder
- byteWidth int
-}
-
-func (b *FixedSizeBinaryDictionaryBuilder) Append(v []byte) error {
- return b.appendValue(v[:b.byteWidth])
-}
-
-func (b *FixedSizeBinaryDictionaryBuilder) InsertDictValues(arr
*FixedSizeBinary) (err error) {
- var (
- beg = arr.array.data.offset * b.byteWidth
- end = (arr.array.data.offset + arr.data.length) * b.byteWidth
- )
- data := arr.valueBytes[beg:end]
- for len(data) > 0 {
- if err = b.insertDictValue(data[:b.byteWidth]); err != nil {
- break
- }
- data = data[b.byteWidth:]
- }
- return
+type fsbType interface {
+ arrow.DayTimeInterval | arrow.MonthDayNanoInterval |
+ decimal.Decimal32 | decimal.Decimal64 | decimal.Decimal128 |
decimal.Decimal256
}
-type Decimal32DictionaryBuilder struct {
+type fixedSizeDictionaryBuilder[T fsbType] struct {
dictionaryBuilder
+ byteWidth int
}
-func (b *Decimal32DictionaryBuilder) Append(v decimal.Decimal32) error {
- return
b.appendValue((*(*[arrow.Decimal32SizeBytes]byte)(unsafe.Pointer(&v)))[:])
-}
-
-func (b *Decimal32DictionaryBuilder) InsertDictValues(arr *Decimal32) (err
error) {
- data := arrow.Decimal32Traits.CastToBytes(arr.values)
- for len(data) > 0 {
- if err = b.insertDictValue(data[:arrow.Decimal32SizeBytes]);
err != nil {
- break
- }
- data = data[arrow.Decimal32SizeBytes:]
- }
- return
-}
-
-type Decimal64DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Decimal64DictionaryBuilder) Append(v decimal.Decimal64) error {
- return
b.appendValue((*(*[arrow.Decimal64SizeBytes]byte)(unsafe.Pointer(&v)))[:])
-}
-
-func (b *Decimal64DictionaryBuilder) InsertDictValues(arr *Decimal64) (err
error) {
- data := arrow.Decimal64Traits.CastToBytes(arr.values)
- for len(data) > 0 {
- if err = b.insertDictValue(data[:arrow.Decimal64SizeBytes]);
err != nil {
- break
- }
- data = data[arrow.Decimal64SizeBytes:]
- }
- return
-}
-
-type Decimal128DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Decimal128DictionaryBuilder) Append(v decimal128.Num) error {
- return
b.appendValue((*(*[arrow.Decimal128SizeBytes]byte)(unsafe.Pointer(&v)))[:])
-}
-
-func (b *Decimal128DictionaryBuilder) InsertDictValues(arr *Decimal128) (err
error) {
- data := arrow.Decimal128Traits.CastToBytes(arr.values)
- for len(data) > 0 {
- if err = b.insertDictValue(data[:arrow.Decimal128SizeBytes]);
err != nil {
- break
- }
- data = data[arrow.Decimal128SizeBytes:]
- }
- return
-}
-
-type Decimal256DictionaryBuilder struct {
- dictionaryBuilder
-}
-
-func (b *Decimal256DictionaryBuilder) Append(v decimal256.Num) error {
- return
b.appendValue((*(*[arrow.Decimal256SizeBytes]byte)(unsafe.Pointer(&v)))[:])
-}
-
-func (b *Decimal256DictionaryBuilder) InsertDictValues(arr *Decimal256) (err
error) {
- data := arrow.Decimal256Traits.CastToBytes(arr.values)
- for len(data) > 0 {
- if err = b.insertDictValue(data[:arrow.Decimal256SizeBytes]);
err != nil {
- break
- }
- data = data[arrow.Decimal256SizeBytes:]
+func (b *fixedSizeDictionaryBuilder[T]) Append(v T) error {
+ if v, ok := any(v).([]byte); ok {
+ return b.appendBytes(v[:b.byteWidth])
}
- return
-}
-
-type MonthDayNanoDictionaryBuilder struct {
- dictionaryBuilder
-}
-func (b *MonthDayNanoDictionaryBuilder) Append(v arrow.MonthDayNanoInterval)
error {
- return
b.appendValue((*(*[arrow.MonthDayNanoIntervalSizeBytes]byte)(unsafe.Pointer(&v)))[:])
+ sliceHdr := struct {
+ Addr *T
+ Len int
+ Cap int
+ }{&v, b.byteWidth, b.byteWidth}
+ slice := *(*[]byte)(unsafe.Pointer(&sliceHdr))
+ return b.appendValue(slice)
}
-func (b *MonthDayNanoDictionaryBuilder) InsertDictValues(arr
*MonthDayNanoInterval) (err error) {
- data := arrow.MonthDayNanoIntervalTraits.CastToBytes(arr.values)
+func (b *fixedSizeDictionaryBuilder[T]) InsertDictValues(arr arrValues[T])
(err error) {
+ data := arrow.GetBytes(arr.Values())
for len(data) > 0 {
- if err =
b.insertDictValue(data[:arrow.MonthDayNanoIntervalSizeBytes]); err != nil {
+ if err = b.insertDictBytes(data[:b.byteWidth]); err != nil {
break
}
- data = data[arrow.MonthDayNanoIntervalSizeBytes:]
+ data = data[b.byteWidth:]
}
return
}
-type DayTimeDictionaryBuilder struct {
+type FixedSizeBinaryDictionaryBuilder struct {
dictionaryBuilder
+ byteWidth int
}
-func (b *DayTimeDictionaryBuilder) Append(v arrow.DayTimeInterval) error {
- return
b.appendValue((*(*[arrow.DayTimeIntervalSizeBytes]byte)(unsafe.Pointer(&v)))[:])
+func (b *FixedSizeBinaryDictionaryBuilder) Append(v []byte) error {
+ return b.appendValue(v[:b.byteWidth])
}
-func (b *DayTimeDictionaryBuilder) InsertDictValues(arr *DayTimeInterval) (err
error) {
- data := arrow.DayTimeIntervalTraits.CastToBytes(arr.values)
+func (b *FixedSizeBinaryDictionaryBuilder) InsertDictValues(arr
*FixedSizeBinary) (err error) {
+ var (
+ beg = arr.array.data.offset * b.byteWidth
+ end = (arr.array.data.offset + arr.data.length) * b.byteWidth
+ )
+ data := arr.valueBytes[beg:end]
for len(data) > 0 {
- if err =
b.insertDictValue(data[:arrow.DayTimeIntervalSizeBytes]); err != nil {
+ if err = b.insertDictValue(data[:b.byteWidth]); err != nil {
break
}
- data = data[arrow.DayTimeIntervalSizeBytes:]
+ data = data[b.byteWidth:]
}
return
}
diff --git a/arrow/array/extension.go b/arrow/array/extension.go
index 4177303..e509b5e 100644
--- a/arrow/array/extension.go
+++ b/arrow/array/extension.go
@@ -173,7 +173,7 @@ func (e *ExtensionArrayBase) ValueStr(i int) string {
}
// no-op function that exists simply to force embedding this in any extension
array types.
-func (ExtensionArrayBase) mustEmbedExtensionArrayBase() {}
+func (*ExtensionArrayBase) mustEmbedExtensionArrayBase() {}
// ExtensionBuilder is a convenience builder so that NewBuilder and such will
still work
// with extension types properly. Depending on preference it may be cleaner or
easier to just use
diff --git a/arrow/array/interval.go b/arrow/array/interval.go
index 1f16d71..54915cd 100644
--- a/arrow/array/interval.go
+++ b/arrow/array/interval.go
@@ -62,7 +62,8 @@ func (a *MonthInterval) ValueStr(i int) string {
}
return fmt.Sprintf("%v", a.Value(i))
}
-func (a *MonthInterval) MonthIntervalValues() []arrow.MonthInterval { return
a.values }
+func (a *MonthInterval) MonthIntervalValues() []arrow.MonthInterval { return
a.Values() }
+func (a *MonthInterval) Values() []arrow.MonthInterval { return
a.values }
func (a *MonthInterval) String() string {
o := new(strings.Builder)
diff --git a/arrow/array/timestamp.go b/arrow/array/timestamp.go
index 164caac..9f8ca47 100644
--- a/arrow/array/timestamp.go
+++ b/arrow/array/timestamp.go
@@ -52,8 +52,10 @@ func (a *Timestamp) Reset(data *Data) {
// Value returns the value at the specified index.
func (a *Timestamp) Value(i int) arrow.Timestamp { return a.values[i] }
+func (a *Timestamp) Values() []arrow.Timestamp { return a.values }
+
// TimestampValues returns the values.
-func (a *Timestamp) TimestampValues() []arrow.Timestamp { return a.values }
+func (a *Timestamp) TimestampValues() []arrow.Timestamp { return a.Values() }
// String returns a string representation of the array.
func (a *Timestamp) String() string {
diff --git a/arrow/array/union.go b/arrow/array/union.go
index edb3625..9c13af0 100644
--- a/arrow/array/union.go
+++ b/arrow/array/union.go
@@ -735,7 +735,7 @@ type unionBuilder struct {
typesBuilder *int8BufferBuilder
}
-func newUnionBuilder(mem memory.Allocator, children []Builder, typ
arrow.UnionType) unionBuilder {
+func newUnionBuilder(mem memory.Allocator, children []Builder, typ
arrow.UnionType) *unionBuilder {
if children == nil {
children = make([]Builder, 0)
}
@@ -767,7 +767,7 @@ func newUnionBuilder(mem memory.Allocator, children
[]Builder, typ arrow.UnionTy
b.typeIDtoBuilder[typeID] = c
}
- return b
+ return &b
}
func (b *unionBuilder) NumChildren() int {
@@ -878,7 +878,7 @@ func (b *unionBuilder) newData() *Data {
// that they have the correct number of preceding elements that have been
// added to the builder beforehand.
type SparseUnionBuilder struct {
- unionBuilder
+ *unionBuilder
}
// NewEmptySparseUnionBuilder is a helper to construct a SparseUnionBuilder
@@ -1108,7 +1108,7 @@ func (b *SparseUnionBuilder) UnmarshalOne(dec
*json.Decoder) error {
// methods. You can also add new types to the union on the fly by using
// AppendChild.
type DenseUnionBuilder struct {
- unionBuilder
+ *unionBuilder
offsetsBuilder *int32BufferBuilder
}
diff --git a/ci/docker/debian-12.dockerfile b/ci/docker/debian-12.dockerfile
index bcf0984..415b1d2 100644
--- a/ci/docker/debian-12.dockerfile
+++ b/ci/docker/debian-12.dockerfile
@@ -16,9 +16,9 @@
# under the License.
ARG arch=amd64
-ARG go=1.22.6
+ARG go=1.23
FROM ${arch}/golang:${go}-bookworm
# Copy the go.mod and go.sum over and pre-download all the dependencies
COPY . /arrow-go
-RUN cd /arrow-go && go mod download
+RUN cd /arrow-go && go mod download github.com/apache/arrow-go/v18@latest
