This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d30669b ARROW-14430: [Go] Basic Expression, Field Reference and Datum
handling
d30669b is described below
commit d30669b7f27f60b20f02c45f1e4ca22f4abd06e1
Author: Matthew Topol <[email protected]>
AuthorDate: Wed Jan 5 15:33:29 2022 -0500
ARROW-14430: [Go] Basic Expression, Field Reference and Datum handling
This is for preparation of connecting to the Compute API, building out the
support for Datums, Field Referencing and Field Paths, and Expression handling.
The actual calls into the C++ and connection there will be part of a subsequent
change in order to keep the size down.
Closes #11514 from zeroshade/arrow-fieldrefs
Lead-authored-by: Matthew Topol <[email protected]>
Co-authored-by: Matt Topol <[email protected]>
Signed-off-by: Matthew Topol <[email protected]>
---
dev/release/rat_exclude_files.txt | 2 +
go/arrow/_tools/tools.go | 23 +
go/arrow/array/boolean.go | 4 +-
go/arrow/array/struct.go | 1 -
go/arrow/cdata/cdata.go | 8 +
go/arrow/cdata/cdata_test_framework.go | 4 -
go/arrow/cdata/interface.go | 6 +
go/arrow/compute/datum.go | 346 ++++++++++++
go/arrow/compute/datumkind_string.go | 29 +
go/arrow/compute/expression.go | 963 +++++++++++++++++++++++++++++++++
go/arrow/compute/expression_test.go | 257 +++++++++
go/arrow/compute/fieldref.go | 603 +++++++++++++++++++++
go/arrow/compute/fieldref_test.go | 316 +++++++++++
go/arrow/compute/hash_util.go | 24 +
go/arrow/compute/no_exec.go | 45 ++
go/arrow/compute/utils.go | 83 +++
go/arrow/compute/valueshape_string.go | 25 +
go/arrow/datatype_extension.go | 7 +
go/arrow/datatype_fixedwidth.go | 10 +-
go/arrow/datatype_nested.go | 12 +
go/arrow/scalar/nested.go | 12 +
go/arrow/scalar/parse.go | 338 ++++++++++++
go/arrow/scalar/scalar.go | 38 +-
go/arrow/scalar/scalar_test.go | 71 +++
go/go.mod | 4 +-
go/go.sum | 9 -
26 files changed, 3213 insertions(+), 27 deletions(-)
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
index 7157669..4829600 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -131,6 +131,8 @@ go/arrow/flight/Flight_grpc.pb.go
go/arrow/internal/cpu/*
go/arrow/type_string.go
go/arrow/cdata/test/go.sum
+go/arrow/compute/datumkind_string.go
+go/arrow/compute/valueshape_string.go
go/*.tmpldata
go/*.s
go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go
diff --git a/go/arrow/_tools/tools.go b/go/arrow/_tools/tools.go
new file mode 100644
index 0000000..6c494bb
--- /dev/null
+++ b/go/arrow/_tools/tools.go
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build tools
+
+package _tools
+
+import (
+ _ "golang.org/x/tools/cmd/stringer"
+)
diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go
index 36e6b12..072d8c6 100644
--- a/go/arrow/array/boolean.go
+++ b/go/arrow/array/boolean.go
@@ -36,7 +36,9 @@ type Boolean struct {
// The nullBitmap buffer can be nil of there are no null values.
// If nulls is not known, use UnknownNullCount to calculate the value of NullN
at runtime from the nullBitmap buffer.
func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer,
nulls int) *Boolean {
- return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length,
[]*memory.Buffer{nullBitmap, data}, nil, nulls, 0))
+ arrdata := NewData(arrow.FixedWidthTypes.Boolean, length,
[]*memory.Buffer{nullBitmap, data}, nil, nulls, 0)
+ defer arrdata.Release()
+ return NewBooleanData(arrdata)
}
func NewBooleanData(data *Data) *Boolean {
diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go
index 89baf4c..36d1058 100644
--- a/go/arrow/array/struct.go
+++ b/go/arrow/array/struct.go
@@ -293,7 +293,6 @@ func (b *StructBuilder) newData() (data *Data) {
b.dtype, b.length,
[]*memory.Buffer{
b.nullBitmap,
- nil, // FIXME(sbinet)
},
fields,
b.nulls,
diff --git a/go/arrow/cdata/cdata.go b/go/arrow/cdata/cdata.go
index ba15943..de88b71 100644
--- a/go/arrow/cdata/cdata.go
+++ b/go/arrow/cdata/cdata.go
@@ -577,3 +577,11 @@ func (n *nativeCRecordBatchReader) Read() (array.Record,
error) {
return ImportCRecordBatchWithSchema(arr, n.schema)
}
+
+func releaseArr(arr *CArrowArray) {
+ C.ArrowArrayRelease(arr)
+}
+
+func releaseSchema(schema *CArrowSchema) {
+ C.ArrowSchemaRelease(schema)
+}
diff --git a/go/arrow/cdata/cdata_test_framework.go
b/go/arrow/cdata/cdata_test_framework.go
index 04d7cf7..3580b2a 100644
--- a/go/arrow/cdata/cdata_test_framework.go
+++ b/go/arrow/cdata/cdata_test_framework.go
@@ -76,10 +76,6 @@ func releaseStream(s *CArrowArrayStream) {
C.ArrowArrayStreamRelease(s)
}
-func releaseSchema(s *CArrowSchema) {
- C.ArrowSchemaRelease(s)
-}
-
func schemaIsReleased(s *CArrowSchema) bool {
return C.ArrowSchemaIsReleased(s) == 1
}
diff --git a/go/arrow/cdata/interface.go b/go/arrow/cdata/interface.go
index 54d4aa5..be533f5 100644
--- a/go/arrow/cdata/interface.go
+++ b/go/arrow/cdata/interface.go
@@ -223,3 +223,9 @@ func ExportArrowRecordBatch(rb array.Record, out
*CArrowArray, outSchema *CArrow
func ExportArrowArray(arr array.Interface, out *CArrowArray, outSchema
*CArrowSchema) {
exportArray(arr, out, outSchema)
}
+
+// ReleaseCArrowArray calls ArrowArrayRelease on the passed in cdata array
+func ReleaseCArrowArray(arr *CArrowArray) { releaseArr(arr) }
+
+// ReleaseCArrowSchema calls ArrowSchemaRelease on the passed in cdata schema
+func ReleaseCArrowSchema(schema *CArrowSchema) { releaseSchema(schema) }
diff --git a/go/arrow/compute/datum.go b/go/arrow/compute/datum.go
new file mode 100644
index 0000000..82d7e49
--- /dev/null
+++ b/go/arrow/compute/datum.go
@@ -0,0 +1,346 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+ "github.com/apache/arrow/go/v7/arrow/scalar"
+)
+
+//go:generate go run golang.org/x/tools/cmd/stringer -type=ValueShape
-linecomment
+//go:generate go run golang.org/x/tools/cmd/stringer -type=DatumKind
-linecomment
+
+// ValueShape is a brief description of the shape of a value (array, scalar or
otherwise)
+type ValueShape int8
+
+const (
+ // either Array or Scalar
+ ShapeAny ValueShape = iota // any
+ ShapeArray // array
+ ShapeScalar // scalar
+)
+
+// ValueDescr is a descriptor type giving both the shape and the datatype of a
value
+// but without the data.
+type ValueDescr struct {
+ Shape ValueShape
+ Type arrow.DataType
+}
+
+func (v *ValueDescr) String() string {
+ return fmt.Sprintf("%s [%s]", v.Shape, v.Type)
+}
+
+// DatumKind is an enum used for denoting which kind of type a datum is
encapsulating
+type DatumKind int
+
+const (
+ KindNone DatumKind = iota // none
+ KindScalar // scalar
+ KindArray // array
+ KindChunked // chunked_array
+ KindRecord // record_batch
+ KindTable // table
+ KindCollection // collection
+)
+
+const UnknownLength int64 = -1
+
+// Datum is a variant interface for wrapping the various Arrow data structures
+// for now the various Datum types just hold a Value which is the type they
+// are wrapping, but it might make sense in the future for those types
+// to actually be aliases or embed their types instead. Not sure yet.
+type Datum interface {
+ fmt.Stringer
+ Kind() DatumKind
+ Len() int64
+ Equals(Datum) bool
+ Release()
+}
+
+// ArrayLikeDatum is an interface for treating a Datum similarly to an Array,
+// so that it is easy to differentiate between Record/Table/Collection and
Scalar,
+// Array/ChunkedArray for ease of use. Chunks will return an empty slice for
Scalar,
+// a slice with 1 element for Array, and the slice of chunks for a chunked
array.
+type ArrayLikeDatum interface {
+ Datum
+ Shape() ValueShape
+ Descr() ValueDescr
+ NullN() int64
+ Type() arrow.DataType
+ Chunks() []array.Interface
+}
+
+// TableLikeDatum is an interface type for specifying either a RecordBatch or a
+// Table as both contain a schema as opposed to just a single data type.
+type TableLikeDatum interface {
+ Datum
+ Schema() *arrow.Schema
+}
+
+// EmptyDatum is the null case, a Datum with nothing in it.
+type EmptyDatum struct{}
+
+func (EmptyDatum) String() string { return "nullptr" }
+func (EmptyDatum) Kind() DatumKind { return KindNone }
+func (EmptyDatum) Len() int64 { return UnknownLength }
+func (EmptyDatum) Release() {}
+func (EmptyDatum) Equals(other Datum) bool {
+ _, ok := other.(EmptyDatum)
+ return ok
+}
+
+// ScalarDatum contains a scalar value
+type ScalarDatum struct {
+ Value scalar.Scalar
+}
+
+func (ScalarDatum) Kind() DatumKind { return KindScalar }
+func (ScalarDatum) Shape() ValueShape { return ShapeScalar }
+func (ScalarDatum) Len() int64 { return 1 }
+func (ScalarDatum) Chunks() []array.Interface { return nil }
+func (d *ScalarDatum) Type() arrow.DataType { return d.Value.DataType() }
+func (d *ScalarDatum) String() string { return d.Value.String() }
+func (d *ScalarDatum) Descr() ValueDescr { return ValueDescr{ShapeScalar,
d.Value.DataType()} }
+func (d *ScalarDatum) ToScalar() (scalar.Scalar, error) {
+ return d.Value, nil
+}
+
+func (d *ScalarDatum) NullN() int64 {
+ if d.Value.IsValid() {
+ return 0
+ }
+ return 1
+}
+
+type releasable interface {
+ Release()
+}
+
+func (d *ScalarDatum) Release() {
+ if v, ok := d.Value.(releasable); ok {
+ v.Release()
+ }
+}
+
+func (d *ScalarDatum) Equals(other Datum) bool {
+ if rhs, ok := other.(*ScalarDatum); ok {
+ return scalar.Equals(d.Value, rhs.Value)
+ }
+ return false
+}
+
+// ArrayDatum references an array.Data object which can be used to create
+// array instances from if needed.
+type ArrayDatum struct {
+ Value *array.Data
+}
+
+func (ArrayDatum) Kind() DatumKind { return KindArray }
+func (ArrayDatum) Shape() ValueShape { return ShapeArray }
+func (d *ArrayDatum) Type() arrow.DataType { return d.Value.DataType() }
+func (d *ArrayDatum) Len() int64 { return int64(d.Value.Len())
}
+func (d *ArrayDatum) NullN() int64 { return
int64(d.Value.NullN()) }
+func (d *ArrayDatum) Descr() ValueDescr { return
ValueDescr{ShapeArray, d.Value.DataType()} }
+func (d *ArrayDatum) String() string { return
fmt.Sprintf("Array:{%s}", d.Value.DataType()) }
+func (d *ArrayDatum) MakeArray() array.Interface { return
array.MakeFromData(d.Value) }
+func (d *ArrayDatum) Chunks() []array.Interface { return
[]array.Interface{d.MakeArray()} }
+func (d *ArrayDatum) ToScalar() (scalar.Scalar, error) {
+ return scalar.NewListScalarData(d.Value), nil
+}
+func (d *ArrayDatum) Release() {
+ d.Value.Release()
+ d.Value = nil
+}
+
+func (d *ArrayDatum) Equals(other Datum) bool {
+ rhs, ok := other.(*ArrayDatum)
+ if !ok {
+ return false
+ }
+
+ left := d.MakeArray()
+ defer left.Release()
+ right := rhs.MakeArray()
+ defer right.Release()
+
+ return array.ArrayEqual(left, right)
+}
+
+// ChunkedDatum contains a chunked array for use with expressions and compute.
+type ChunkedDatum struct {
+ Value *array.Chunked
+}
+
+func (ChunkedDatum) Kind() DatumKind { return KindChunked }
+func (ChunkedDatum) Shape() ValueShape { return ShapeArray }
+func (d *ChunkedDatum) Type() arrow.DataType { return d.Value.DataType() }
+func (d *ChunkedDatum) Len() int64 { return
int64(d.Value.Len()) }
+func (d *ChunkedDatum) NullN() int64 { return
int64(d.Value.NullN()) }
+func (d *ChunkedDatum) Descr() ValueDescr { return
ValueDescr{ShapeArray, d.Value.DataType()} }
+func (d *ChunkedDatum) String() string { return
fmt.Sprintf("Array:{%s}", d.Value.DataType()) }
+func (d *ChunkedDatum) Chunks() []array.Interface { return d.Value.Chunks() }
+
+func (d *ChunkedDatum) Release() {
+ d.Value.Release()
+ d.Value = nil
+}
+
+func (d *ChunkedDatum) Equals(other Datum) bool {
+ if rhs, ok := other.(*ChunkedDatum); ok {
+ return array.ChunkedEqual(d.Value, rhs.Value)
+ }
+ return false
+}
+
+// RecordDatum contains an array.Record for passing a full record to an
expression
+// or to compute.
+type RecordDatum struct {
+ Value array.Record
+}
+
+func (RecordDatum) Kind() DatumKind { return KindRecord }
+func (RecordDatum) String() string { return "RecordBatch" }
+func (r *RecordDatum) Len() int64 { return r.Value.NumRows() }
+func (r *RecordDatum) Schema() *arrow.Schema { return r.Value.Schema() }
+
+func (r *RecordDatum) Release() {
+ r.Value.Release()
+ r.Value = nil
+}
+
+func (r *RecordDatum) Equals(other Datum) bool {
+ if rhs, ok := other.(*RecordDatum); ok {
+ return array.RecordEqual(r.Value, rhs.Value)
+ }
+ return false
+}
+
+// TableDatum contains a table so that multiple record batches can be worked
with
+// together as a single table for being passed to compute and expression
handling.
+type TableDatum struct {
+ Value array.Table
+}
+
+func (TableDatum) Kind() DatumKind { return KindTable }
+func (TableDatum) String() string { return "Table" }
+func (d *TableDatum) Len() int64 { return d.Value.NumRows() }
+func (d *TableDatum) Schema() *arrow.Schema { return d.Value.Schema() }
+
+func (d *TableDatum) Release() {
+ d.Value.Release()
+ d.Value = nil
+}
+
+func (d *TableDatum) Equals(other Datum) bool {
+ if rhs, ok := other.(*TableDatum); ok {
+ return array.TableEqual(d.Value, rhs.Value)
+ }
+ return false
+}
+
+// CollectionDatum is a slice of Datums
+type CollectionDatum []Datum
+
+func (CollectionDatum) Kind() DatumKind { return KindCollection }
+func (c CollectionDatum) Len() int64 { return int64(len(c)) }
+func (c CollectionDatum) String() string {
+ var b strings.Builder
+ b.WriteString("Collection(")
+ for i, d := range c {
+ if i > 0 {
+ b.WriteString(", ")
+ }
+ b.WriteString(d.String())
+ }
+ b.WriteByte(')')
+ return b.String()
+}
+
+func (c CollectionDatum) Release() {
+ for _, v := range c {
+ v.Release()
+ }
+}
+
+func (c CollectionDatum) Equals(other Datum) bool {
+ rhs, ok := other.(CollectionDatum)
+ if !ok {
+ return false
+ }
+
+ if len(c) != len(rhs) {
+ return false
+ }
+
+ for i := range c {
+ if !c[i].Equals(rhs[i]) {
+ return false
+ }
+ }
+ return true
+}
+
+// NewDatum will construct the appropriate Datum type based on what is passed
in
+// as the argument.
+//
+// An array.Interface gets an ArrayDatum
+// An array.Chunked gets a ChunkedDatum
+// An array.Record gets a RecordDatum
+// an array.Table gets a TableDatum
+// a []Datum gets a CollectionDatum
+// a scalar.Scalar gets a ScalarDatum
+//
+// Anything else is passed to scalar.MakeScalar and recieves a scalar
+// datum of that appropriate type.
+func NewDatum(value interface{}) Datum {
+ switch v := value.(type) {
+ case Datum:
+ return v
+ case array.Interface:
+ v.Data().Retain()
+ return &ArrayDatum{v.Data()}
+ case *array.Chunked:
+ v.Retain()
+ return &ChunkedDatum{v}
+ case array.Record:
+ v.Retain()
+ return &RecordDatum{v}
+ case array.Table:
+ v.Retain()
+ return &TableDatum{v}
+ case []Datum:
+ return CollectionDatum(v)
+ case scalar.Scalar:
+ return &ScalarDatum{v}
+ default:
+ return &ScalarDatum{scalar.MakeScalar(value)}
+ }
+}
+
+var (
+ _ ArrayLikeDatum = (*ScalarDatum)(nil)
+ _ ArrayLikeDatum = (*ArrayDatum)(nil)
+ _ ArrayLikeDatum = (*ChunkedDatum)(nil)
+ _ TableLikeDatum = (*RecordDatum)(nil)
+ _ TableLikeDatum = (*TableDatum)(nil)
+ _ Datum = (CollectionDatum)(nil)
+)
diff --git a/go/arrow/compute/datumkind_string.go
b/go/arrow/compute/datumkind_string.go
new file mode 100644
index 0000000..56cef31
--- /dev/null
+++ b/go/arrow/compute/datumkind_string.go
@@ -0,0 +1,29 @@
+// Code generated by "stringer -type=DatumKind -linecomment"; DO NOT EDIT.
+
+package compute
+
+import "strconv"
+
+func _() {
+ // An "invalid array index" compiler error signifies that the constant
values have changed.
+ // Re-run the stringer command to generate them again.
+ var x [1]struct{}
+ _ = x[KindNone-0]
+ _ = x[KindScalar-1]
+ _ = x[KindArray-2]
+ _ = x[KindChunked-3]
+ _ = x[KindRecord-4]
+ _ = x[KindTable-5]
+ _ = x[KindCollection-6]
+}
+
+const _DatumKind_name =
"nonescalararraychunked_arrayrecord_batchtablecollection"
+
+var _DatumKind_index = [...]uint8{0, 4, 10, 15, 28, 40, 45, 55}
+
+func (i DatumKind) String() string {
+ if i < 0 || i >= DatumKind(len(_DatumKind_index)-1) {
+ return "DatumKind(" + strconv.FormatInt(int64(i), 10) + ")"
+ }
+ return _DatumKind_name[_DatumKind_index[i]:_DatumKind_index[i+1]]
+}
diff --git a/go/arrow/compute/expression.go b/go/arrow/compute/expression.go
new file mode 100644
index 0000000..8d560b5
--- /dev/null
+++ b/go/arrow/compute/expression.go
@@ -0,0 +1,963 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+ "bytes"
+ "context"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "hash/maphash"
+ "reflect"
+ "strconv"
+ "strings"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+ "github.com/apache/arrow/go/v7/arrow/internal/debug"
+ "github.com/apache/arrow/go/v7/arrow/ipc"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/apache/arrow/go/v7/arrow/scalar"
+)
+
+var hashSeed = maphash.MakeSeed()
+
+// Expression is an interface for mapping one datum to another. An expression
+// is one of:
+// A literal Datum
+// A reference to a single (potentially nested) field of an input Datum
+// A call to a compute function, with arguments specified by other
Expressions
+type Expression interface {
+ fmt.Stringer
+ // IsBound returns true if this expression has been bound to a
particular
+ // Datum and/or Schema.
+ IsBound() bool
+ // IsScalarExpr returns true if this expression is composed only of
scalar
+ // literals, field references and calls to scalar functions.
+ IsScalarExpr() bool
+ // IsNullLiteral returns true if this expression is a literal and
entirely
+ // null.
+ IsNullLiteral() bool
+ // IsSatisfiable returns true if this expression could evaluate to true
+ IsSatisfiable() bool
+ // FieldRef returns a pointer to the underlying field reference, or nil
if
+ // this expression is not a field reference.
+ FieldRef() *FieldRef
+ // Descr returns the shape of this expression will evaluate to
including the type
+ // and whether it will be an Array, Scalar, or either.
+ Descr() ValueDescr
+ // Type returns the datatype this expression will evaluate to.
+ Type() arrow.DataType
+
+ Hash() uint64
+ Equals(Expression) bool
+
+ // Bind binds this expression to the given input schema, looking up
appropriate
+ // underlying implementations and some expression simplification may be
performed
+ // along with implicit casts being inserted.
+ // Any state necessary for execution will be initialized.
+ //
+ // This only works in conjunction with cgo and being able to link
against the
+ // C++ libarrow.so compute library. If this was not built with the
libarrow compute
+ // support, this will panic.
+ Bind(context.Context, memory.Allocator, *arrow.Schema) (Expression,
error)
+
+ // Release releases the underlying bound C++ memory that is allocated
when
+ // a Bind is performed. Any bound expression should get released to
ensure
+ // no memory leaks.
+ Release()
+
+ boundExpr() boundRef
+}
+
+func printDatum(datum Datum) string {
+ switch datum := datum.(type) {
+ case *ScalarDatum:
+ if !datum.Value.IsValid() {
+ return "null"
+ }
+
+ switch datum.Type().ID() {
+ case arrow.STRING, arrow.LARGE_STRING:
+ return
strconv.Quote(datum.Value.(scalar.BinaryScalar).String())
+ case arrow.BINARY, arrow.FIXED_SIZE_BINARY, arrow.LARGE_BINARY:
+ return `"` +
strings.ToUpper(hex.EncodeToString(datum.Value.(scalar.BinaryScalar).Data())) +
`"`
+ }
+
+ return datum.Value.String()
+ default:
+ return datum.String()
+ }
+}
+
+// Literal is an expression denoting a literal Datum which could be any value
+// as a scalar, an array, or so on.
+type Literal struct {
+ Literal Datum
+
+ bound boundRef
+}
+
+func (Literal) FieldRef() *FieldRef { return nil }
+func (l *Literal) String() string { return printDatum(l.Literal) }
+func (l *Literal) boundExpr() boundRef { return l.bound }
+func (l *Literal) Type() arrow.DataType { return
l.Literal.(ArrayLikeDatum).Type() }
+func (l *Literal) IsBound() bool { return l.Type() != nil }
+func (l *Literal) IsScalarExpr() bool { return l.Literal.Kind() ==
KindScalar }
+
+func (l *Literal) Equals(other Expression) bool {
+ if rhs, ok := other.(*Literal); ok {
+ return l.Literal.Equals(rhs.Literal)
+ }
+ return false
+}
+
+func (l *Literal) IsNullLiteral() bool {
+ if ad, ok := l.Literal.(ArrayLikeDatum); ok {
+ return ad.NullN() == ad.Len()
+ }
+ return true
+}
+
+func (l *Literal) IsSatisfiable() bool {
+ if l.IsNullLiteral() {
+ return false
+ }
+
+ if sc, ok := l.Literal.(*ScalarDatum); ok && sc.Type().ID() ==
arrow.BOOL {
+ return sc.Value.(*scalar.Boolean).Value
+ }
+
+ return true
+}
+
+func (l *Literal) Descr() ValueDescr {
+ if ad, ok := l.Literal.(ArrayLikeDatum); ok {
+ return ad.Descr()
+ }
+
+ return ValueDescr{ShapeAny, nil}
+}
+
+func (l *Literal) Hash() uint64 {
+ if l.IsScalarExpr() {
+ return scalar.Hash(hashSeed, l.Literal.(*ScalarDatum).Value)
+ }
+ return 0
+}
+
+func (l *Literal) Bind(ctx context.Context, mem memory.Allocator, schema
*arrow.Schema) (Expression, error) {
+ bound, _, _, _, err := bindExprSchema(ctx, mem, l, schema)
+ if err != nil {
+ return nil, err
+ }
+
+ return &Literal{l.Literal, bound}, nil
+}
+
+func (l *Literal) Release() {
+ l.Literal.Release()
+ if l.bound != 0 {
+ l.bound.release()
+ }
+}
+
+// Parameter represents a field reference and needs to be bound in order to
determine
+// its type and shape.
+type Parameter struct {
+ ref *FieldRef
+
+ // post bind props
+ descr ValueDescr
+ index int
+
+ bound boundRef
+}
+
+func (Parameter) IsNullLiteral() bool { return false }
+func (p *Parameter) boundExpr() boundRef { return p.bound }
+func (p *Parameter) Type() arrow.DataType { return p.descr.Type }
+func (p *Parameter) IsBound() bool { return p.Type() != nil }
+func (p *Parameter) IsScalarExpr() bool { return p.ref != nil }
+func (p *Parameter) IsSatisfiable() bool { return p.Type() == nil ||
p.Type().ID() != arrow.NULL }
+func (p *Parameter) FieldRef() *FieldRef { return p.ref }
+func (p *Parameter) Descr() ValueDescr { return p.descr }
+func (p *Parameter) Hash() uint64 { return p.ref.Hash(hashSeed) }
+
+func (p *Parameter) String() string {
+ switch {
+ case p.ref.IsName():
+ return p.ref.Name()
+ case p.ref.IsFieldPath():
+ return p.ref.FieldPath().String()
+ default:
+ return p.ref.String()
+ }
+}
+
+func (p *Parameter) Equals(other Expression) bool {
+ if rhs, ok := other.(*Parameter); ok {
+ return p.ref.Equals(*rhs.ref)
+ }
+
+ return false
+}
+
+func (p *Parameter) Bind(ctx context.Context, mem memory.Allocator, schema
*arrow.Schema) (Expression, error) {
+ bound, descr, index, _, err := bindExprSchema(ctx, mem, p, schema)
+ if err != nil {
+ return nil, err
+ }
+
+ return &Parameter{
+ ref: p.ref,
+ index: index,
+ descr: descr,
+ bound: bound,
+ }, nil
+}
+
+func (p *Parameter) Release() {
+ if p.bound != 0 {
+ p.bound.release()
+ }
+}
+
+type comparisonType int8
+
+const (
+ compNA comparisonType = 0
+ compEQ comparisonType = 1
+ compLT comparisonType = 2
+ compGT comparisonType = 4
+ compNE comparisonType = compLT | compGT
+ compLE comparisonType = compLT | compEQ
+ compGE comparisonType = compGT | compEQ
+)
+
+func (c comparisonType) name() string {
+ switch c {
+ case compEQ:
+ return "equal"
+ case compLT:
+ return "less"
+ case compGT:
+ return "greater"
+ case compNE:
+ return "not_equal"
+ case compLE:
+ return "less_equal"
+ case compGE:
+ return "greater_equal"
+ }
+ return "na"
+}
+
+func (c comparisonType) getOp() string {
+ switch c {
+ case compEQ:
+ return "=="
+ case compLT:
+ return "<"
+ case compGT:
+ return ">"
+ case compNE:
+ return "!="
+ case compLE:
+ return "<="
+ case compGE:
+ return ">="
+ }
+ debug.Assert(false, "invalid getop")
+ return ""
+}
+
+var compmap = map[string]comparisonType{
+ "equal": compEQ,
+ "less": compLT,
+ "greater": compGT,
+ "not_equal": compNE,
+ "less_equal": compLE,
+ "greater_equal": compGE,
+}
+
+func optionsToString(fn FunctionOptions) string {
+ if s, ok := fn.(fmt.Stringer); ok {
+ return s.String()
+ }
+
+ var b strings.Builder
+ v := reflect.Indirect(reflect.ValueOf(fn))
+ b.WriteByte('{')
+ for i := 0; i < v.Type().NumField(); i++ {
+ fld := v.Type().Field(i)
+ tag := fld.Tag.Get("compute")
+ if tag == "-" {
+ continue
+ }
+
+ fldVal := v.Field(i)
+ fmt.Fprintf(&b, "%s=%v, ", tag, fldVal.Interface())
+ }
+ ret := b.String()
+ return ret[:len(ret)-2] + "}"
+}
+
+// Call is a function call with specific arguments which are themselves other
+// expressions. A call can also have options that are specific to the function
+// in question. It must be bound to determine the shape and type.
+type Call struct {
+ funcName string
+ args []Expression
+ descr ValueDescr
+ options FunctionOptions
+
+ cachedHash uint64
+ bound boundRef
+}
+
+func (c *Call) boundExpr() boundRef { return c.bound }
+func (c *Call) IsNullLiteral() bool { return false }
+func (c *Call) FieldRef() *FieldRef { return nil }
+func (c *Call) Descr() ValueDescr { return c.descr }
+func (c *Call) Type() arrow.DataType { return c.descr.Type }
+func (c *Call) IsSatisfiable() bool { return c.Type() == nil || c.Type().ID()
!= arrow.NULL }
+
+func (c *Call) String() string {
+ binary := func(op string) string {
+ return "(" + c.args[0].String() + " " + op + " " +
c.args[1].String() + ")"
+ }
+
+ if cmp, ok := compmap[c.funcName]; ok {
+ return binary(cmp.getOp())
+ }
+
+ const kleene = "_kleene"
+ if strings.HasSuffix(c.funcName, kleene) {
+ return binary(strings.TrimSuffix(c.funcName, kleene))
+ }
+
+ if c.funcName == "make_struct" && c.options != nil {
+ opts := c.options.(*MakeStructOptions)
+ out := "{"
+ for i, a := range c.args {
+ out += opts.FieldNames[i] + "=" + a.String() + ", "
+ }
+ return out[:len(out)-2] + "}"
+ }
+
+ var b strings.Builder
+ b.WriteString(c.funcName + "(")
+ for _, a := range c.args {
+ b.WriteString(a.String() + ", ")
+ }
+
+ if c.options != nil {
+ b.WriteString(optionsToString(c.options))
+ b.WriteString(" ")
+ }
+
+ ret := b.String()
+ return ret[:len(ret)-2] + ")"
+}
+
+func (c *Call) Hash() uint64 {
+ if c.cachedHash != 0 {
+ return c.cachedHash
+ }
+
+ var h maphash.Hash
+ h.SetSeed(hashSeed)
+
+ h.WriteString(c.funcName)
+ c.cachedHash = h.Sum64()
+ for _, arg := range c.args {
+ c.cachedHash = hashCombine(c.cachedHash, arg.Hash())
+ }
+ return c.cachedHash
+}
+
+func (c *Call) IsScalarExpr() bool {
+ for _, arg := range c.args {
+ if !arg.IsScalarExpr() {
+ return false
+ }
+ }
+ return isFuncScalar(c.funcName)
+}
+
+func (c *Call) IsBound() bool {
+ if c.Type() == nil {
+ return false
+ }
+
+ return c.bound != 0
+}
+
+func (c *Call) Equals(other Expression) bool {
+ rhs, ok := other.(*Call)
+ if !ok {
+ return false
+ }
+
+ if c.funcName != rhs.funcName || len(c.args) != len(rhs.args) {
+ return false
+ }
+
+ for i := range c.args {
+ if !c.args[i].Equals(rhs.args[i]) {
+ return false
+ }
+ }
+
+ if opt, ok := c.options.(FunctionOptionsEqual); ok {
+ return opt.Equals(rhs.options)
+ }
+ return reflect.DeepEqual(c.options, rhs.options)
+}
+
+func (c *Call) Bind(ctx context.Context, mem memory.Allocator, schema
*arrow.Schema) (Expression, error) {
+ _, _, _, output, err := bindExprSchema(ctx, mem, c, schema)
+ if err != nil {
+ return nil, err
+ }
+ return output, nil
+}
+
+func (c *Call) Release() {
+ for _, a := range c.args {
+ a.Release()
+ }
+ if r, ok := c.options.(releasable); ok {
+ r.Release()
+ }
+ if c.bound != 0 {
+ c.bound.release()
+ }
+}
+
+// FunctionOptions can be any type which has a TypeName function. The fields
+// of the type will be used (via reflection) to determine the information to
+// propagate when serializing to pass to the C++ for execution.
+type FunctionOptions interface {
+ TypeName() string
+}
+
+type FunctionOptionsEqual interface {
+ Equals(FunctionOptions) bool
+}
+
+type MakeStructOptions struct {
+ FieldNames []string `compute:"field_names"`
+ FieldNullability []bool `compute:"field_nullability"`
+ FieldMetadata []*arrow.Metadata `compute:"field_metadata"`
+}
+
+func (MakeStructOptions) TypeName() string { return "MakeStructOptions" }
+
+type NullOptions struct {
+ NanIsNull bool `compute:"nan_is_null"`
+}
+
+func (NullOptions) TypeName() string { return "NullOptions" }
+
+type StrptimeOptions struct {
+ Format string `compute:"format"`
+ Unit arrow.TimeUnit `compute:"unit"`
+}
+
+func (StrptimeOptions) TypeName() string { return "StrptimeOptions" }
+
+type NullSelectionBehavior int8
+
+const (
+ DropNulls NullSelectionBehavior = iota
+ EmitNulls
+)
+
+type FilterOptions struct {
+ NullSelection NullSelectionBehavior `compute:"null_selection_behavior"`
+}
+
+func (FilterOptions) TypeName() string { return "FilterOptions" }
+
+type ArithmeticOptions struct {
+ CheckOverflow bool `compute:"check_overflow"`
+}
+
+func (ArithmeticOptions) TypeName() string { return "ArithmeticOptions" }
+
+type CastOptions struct {
+ ToType arrow.DataType `compute:"to_type"`
+ AllowIntOverflow bool `compute:"allow_int_overflow"`
+ AllowTimeTruncate bool `compute:"allow_time_truncate"`
+ AllowTimeOverflow bool `compute:"allow_time_overflow"`
+ AllowDecimalTruncate bool `compute:"allow_decimal_truncate"`
+ AllowFloatTruncate bool `compute:"allow_float_truncate"`
+ AllowInvalidUtf8 bool `compute:"allow_invalid_utf8"`
+}
+
+func (CastOptions) TypeName() string { return "CastOptions" }
+
+func DefaultCastOptions(safe bool) *CastOptions {
+ if safe {
+ return &CastOptions{}
+ }
+ return &CastOptions{
+ AllowIntOverflow: true,
+ AllowTimeTruncate: true,
+ AllowTimeOverflow: true,
+ AllowDecimalTruncate: true,
+ AllowFloatTruncate: true,
+ AllowInvalidUtf8: true,
+ }
+}
+
+func NewCastOptions(dt arrow.DataType, safe bool) *CastOptions {
+ opts := DefaultCastOptions(safe)
+ if dt != nil {
+ opts.ToType = dt
+ } else {
+ opts.ToType = arrow.Null
+ }
+ return opts
+}
+
+func Cast(ex Expression, dt arrow.DataType) Expression {
+ opts := &CastOptions{}
+ if dt == nil {
+ opts.ToType = arrow.Null
+ } else {
+ opts.ToType = dt
+ }
+
+ return NewCall("cast", []Expression{ex}, opts)
+}
+
+type SetLookupOptions struct {
+ ValueSet Datum `compute:"value_set"`
+ SkipNulls bool `compute:"skip_nulls"`
+}
+
+func (SetLookupOptions) TypeName() string { return "SetLookupOptions" }
+
+func (s *SetLookupOptions) Release() { s.ValueSet.Release() }
+
+func (s *SetLookupOptions) Equals(other FunctionOptions) bool {
+ rhs, ok := other.(*SetLookupOptions)
+ if !ok {
+ return false
+ }
+
+ return s.SkipNulls == rhs.SkipNulls && s.ValueSet.Equals(rhs.ValueSet)
+}
+
+func (s *SetLookupOptions) FromStructScalar(sc *scalar.Struct) error {
+ if v, err := sc.Field("skip_nulls"); err == nil {
+ s.SkipNulls = v.(*scalar.Boolean).Value
+ }
+
+ value, err := sc.Field("value_set")
+ if err != nil {
+ return err
+ }
+
+ if v, ok := value.(scalar.ListScalar); ok {
+ s.ValueSet = NewDatum(v.GetList())
+ return nil
+ }
+
+ return errors.New("set lookup options valueset should be a list")
+}
+
+var (
+ funcOptionsMap map[string]reflect.Type
+ funcOptsTypes = []FunctionOptions{
+ SetLookupOptions{}, ArithmeticOptions{}, CastOptions{},
+ FilterOptions{}, NullOptions{}, StrptimeOptions{},
MakeStructOptions{},
+ }
+)
+
+func init() {
+ funcOptionsMap = make(map[string]reflect.Type)
+ for _, ft := range funcOptsTypes {
+ funcOptionsMap[ft.TypeName()] = reflect.TypeOf(ft)
+ }
+}
+
+// NewLiteral constructs a new literal expression from any value. It is passed
+// to NewDatum which will construct the appropriate Datum and/or scalar
+// value for the type provided.
+func NewLiteral(arg interface{}) Expression {
+ return &Literal{Literal: NewDatum(arg)}
+}
+
+func NullLiteral(dt arrow.DataType) Expression {
+ return &Literal{Literal: NewDatum(scalar.MakeNullScalar(dt))}
+}
+
+// NewRef constructs a parameter expression which refers to a specific field
+func NewRef(ref FieldRef) Expression {
+ return &Parameter{ref: &ref, index: -1}
+}
+
+// NewFieldRef is shorthand for NewRef(FieldRefName(field))
+func NewFieldRef(field string) Expression {
+ return NewRef(FieldRefName(field))
+}
+
+// NewCall constructs an expression that represents a specific function call
with
+// the given arguments and options.
+func NewCall(name string, args []Expression, opts FunctionOptions) Expression {
+ return &Call{funcName: name, args: args, options: opts}
+}
+
+// Project is shorthand for `make_struct` to produce a record batch output
+// from a group of expressions.
+func Project(values []Expression, names []string) Expression {
+ nulls := make([]bool, len(names))
+ for i := range nulls {
+ nulls[i] = true
+ }
+ meta := make([]*arrow.Metadata, len(names))
+ return NewCall("make_struct", values,
+ &MakeStructOptions{FieldNames: names, FieldNullability: nulls,
FieldMetadata: meta})
+}
+
+// Equal is a convenience function for the equal function
+func Equal(lhs, rhs Expression) Expression {
+ return NewCall("equal", []Expression{lhs, rhs}, nil)
+}
+
+// NotEqual creates a call to not_equal
+func NotEqual(lhs, rhs Expression) Expression {
+ return NewCall("not_equal", []Expression{lhs, rhs}, nil)
+}
+
+// Less is shorthand for NewCall("less",....)
+func Less(lhs, rhs Expression) Expression {
+ return NewCall("less", []Expression{lhs, rhs}, nil)
+}
+
+// LessEqual is shorthand for NewCall("less_equal",....)
+func LessEqual(lhs, rhs Expression) Expression {
+ return NewCall("less_equal", []Expression{lhs, rhs}, nil)
+}
+
+// Greater is shorthand for NewCall("greater",....)
+func Greater(lhs, rhs Expression) Expression {
+ return NewCall("greater", []Expression{lhs, rhs}, nil)
+}
+
+// GreaterEqual is shorthand for NewCall("greater_equal",....)
+func GreaterEqual(lhs, rhs Expression) Expression {
+ return NewCall("greater_equal", []Expression{lhs, rhs}, nil)
+}
+
+// IsNull creates an expression that returns true if the passed in expression
is
+// null. Optionally treating NaN as null if desired.
+func IsNull(lhs Expression, nanIsNull bool) Expression {
+ return NewCall("less", []Expression{lhs}, &NullOptions{nanIsNull})
+}
+
+// IsValid is the inverse of IsNull
+func IsValid(lhs Expression) Expression {
+ return NewCall("is_valid", []Expression{lhs}, nil)
+}
+
+type binop func(lhs, rhs Expression) Expression
+
+func foldLeft(op binop, args ...Expression) Expression {
+ switch len(args) {
+ case 0:
+ return nil
+ case 1:
+ return args[0]
+ }
+
+ folded := args[0]
+ for _, a := range args[1:] {
+ folded = op(folded, a)
+ }
+ return folded
+}
+
+func and(lhs, rhs Expression) Expression {
+ return NewCall("and_kleene", []Expression{lhs, rhs}, nil)
+}
+
+// And constructs a tree of calls to and_kleene for boolean And logic taking
+// an arbitrary number of values.
+func And(lhs, rhs Expression, ops ...Expression) Expression {
+ folded := foldLeft(and, append([]Expression{lhs, rhs}, ops...)...)
+ if folded != nil {
+ return folded
+ }
+ return NewLiteral(true)
+}
+
+func or(lhs, rhs Expression) Expression {
+ return NewCall("or_kleene", []Expression{lhs, rhs}, nil)
+}
+
+// Or constructs a tree of calls to or_kleene for boolean Or logic taking
+// an arbitrary number of values.
+func Or(lhs, rhs Expression, ops ...Expression) Expression {
+ folded := foldLeft(or, append([]Expression{lhs, rhs}, ops...)...)
+ if folded != nil {
+ return folded
+ }
+ return NewLiteral(false)
+}
+
+// Not creates a call to "invert" for the value specified.
+func Not(expr Expression) Expression {
+ return NewCall("invert", []Expression{expr}, nil)
+}
+
+func SerializeOptions(opts FunctionOptions, mem memory.Allocator)
(*memory.Buffer, error) {
+ sc, err := scalar.ToScalar(opts, mem)
+ if err != nil {
+ return nil, err
+ }
+ if sc, ok := sc.(releasable); ok {
+ defer sc.Release()
+ }
+
+ arr, err := scalar.MakeArrayFromScalar(sc, 1, mem)
+ if err != nil {
+ return nil, err
+ }
+ defer arr.Release()
+
+ batch := array.NewRecord(arrow.NewSchema([]arrow.Field{{Type:
arr.DataType(), Nullable: true}}, nil), []array.Interface{arr}, 1)
+ defer batch.Release()
+
+ buf := &bufferWriteSeeker{mem: mem}
+ wr, err := ipc.NewFileWriter(buf, ipc.WithSchema(batch.Schema()),
ipc.WithAllocator(mem))
+ if err != nil {
+ return nil, err
+ }
+
+ wr.Write(batch)
+ wr.Close()
+ return buf.buf, nil
+}
+
+// SerializeExpr serializes expressions by converting them to Metadata and
+// storing this in the schema of a Record. Embedded arrays and scalars are
+// stored in its columns. Finally the record is written as an IPC file
+func SerializeExpr(expr Expression, mem memory.Allocator) (*memory.Buffer,
error) {
+ var (
+ cols []array.Interface
+ metaKey []string
+ metaValue []string
+ visit func(Expression) error
+ )
+
+ addScalar := func(s scalar.Scalar) (string, error) {
+ ret := len(cols)
+ arr, err := scalar.MakeArrayFromScalar(s, 1, mem)
+ if err != nil {
+ return "", err
+ }
+ cols = append(cols, arr)
+ return strconv.Itoa(ret), nil
+ }
+
+ visit = func(e Expression) error {
+ switch e := e.(type) {
+ case *Literal:
+ if !e.IsScalarExpr() {
+ return errors.New("not implemented:
serialization of non-scalar literals")
+ }
+ metaKey = append(metaKey, "literal")
+ s, err := addScalar(e.Literal.(*ScalarDatum).Value)
+ if err != nil {
+ return err
+ }
+ metaValue = append(metaValue, s)
+ case *Parameter:
+ if e.ref.Name() == "" {
+ return errors.New("not implemented:
serialization of non-name field_ref")
+ }
+
+ metaKey = append(metaKey, "field_ref")
+ metaValue = append(metaValue, e.ref.Name())
+ case *Call:
+ metaKey = append(metaKey, "call")
+ metaValue = append(metaValue, e.funcName)
+
+ for _, arg := range e.args {
+ visit(arg)
+ }
+
+ if e.options != nil {
+ st, err := scalar.ToScalar(e.options, mem)
+ if err != nil {
+ return err
+ }
+ metaKey = append(metaKey, "options")
+ s, err := addScalar(st)
+ if err != nil {
+ return err
+ }
+ metaValue = append(metaValue, s)
+
+ for _, f := range st.(*scalar.Struct).Value {
+ switch s := f.(type) {
+ case releasable:
+ defer s.Release()
+ }
+ }
+ }
+
+ metaKey = append(metaKey, "end")
+ metaValue = append(metaValue, e.funcName)
+ }
+ return nil
+ }
+
+ if err := visit(expr); err != nil {
+ return nil, err
+ }
+
+ fields := make([]arrow.Field, len(cols))
+ for i, c := range cols {
+ fields[i].Type = c.DataType()
+ defer c.Release()
+ }
+
+ metadata := arrow.NewMetadata(metaKey, metaValue)
+ rec := array.NewRecord(arrow.NewSchema(fields, &metadata), cols, 1)
+ defer rec.Release()
+
+ buf := &bufferWriteSeeker{mem: mem}
+ wr, err := ipc.NewFileWriter(buf, ipc.WithSchema(rec.Schema()),
ipc.WithAllocator(mem))
+ if err != nil {
+ return nil, err
+ }
+
+ wr.Write(rec)
+ wr.Close()
+ return buf.buf, nil
+}
+
+func DeserializeExpr(mem memory.Allocator, buf *memory.Buffer) (Expression,
error) {
+ rdr, err := ipc.NewFileReader(bytes.NewReader(buf.Bytes()),
ipc.WithAllocator(mem))
+ if err != nil {
+ return nil, err
+ }
+ defer rdr.Close()
+
+ batch, err := rdr.Read()
+ if err != nil {
+ return nil, err
+ }
+
+ if !batch.Schema().HasMetadata() {
+ return nil, errors.New("serialized Expression's batch repr had
no metadata")
+ }
+
+ if batch.NumRows() != 1 {
+ return nil, fmt.Errorf("serialized Expression's batch repr was
not a single row - had %d", batch.NumRows())
+ }
+
+ var (
+ getone func() (Expression, error)
+ index int = 0
+ metadata = batch.Schema().Metadata()
+ )
+
+ getscalar := func(i string) (scalar.Scalar, error) {
+ colIndex, err := strconv.ParseInt(i, 10, 32)
+ if err != nil {
+ return nil, err
+ }
+ if colIndex >= batch.NumCols() {
+ return nil, errors.New("column index out of bounds")
+ }
+ return scalar.GetScalar(batch.Column(int(colIndex)), 0)
+ }
+
+ getone = func() (Expression, error) {
+ if index >= metadata.Len() {
+ return nil, errors.New("unterminated serialized
Expression")
+ }
+
+ key, val := metadata.Keys()[index], metadata.Values()[index]
+ index++
+
+ switch key {
+ case "literal":
+ scalar, err := getscalar(val)
+ if err != nil {
+ return nil, err
+ }
+ if r, ok := scalar.(releasable); ok {
+ defer r.Release()
+ }
+ return NewLiteral(scalar), err
+ case "field_ref":
+ return NewFieldRef(val), nil
+ case "call":
+ args := make([]Expression, 0)
+ for metadata.Keys()[index] != "end" {
+ if metadata.Keys()[index] == "options" {
+ optsScalar, err :=
getscalar(metadata.Values()[index])
+ if err != nil {
+ return nil, err
+ }
+ if r, ok := optsScalar.(releasable); ok
{
+ defer r.Release()
+ }
+ var opts FunctionOptions
+ if optsScalar != nil {
+ typname, err :=
optsScalar.(*scalar.Struct).Field("_type_name")
+ if err != nil {
+ return nil, err
+ }
+ if typname.DataType().ID() !=
arrow.BINARY {
+ return nil,
errors.New("options scalar typename must be binary")
+ }
+
+ optionsVal :=
reflect.New(funcOptionsMap[string(typname.(*scalar.Binary).Data())]).Interface()
+ if err :=
scalar.FromScalar(optsScalar.(*scalar.Struct), optionsVal); err != nil {
+ return nil, err
+ }
+ opts =
optionsVal.(FunctionOptions)
+ }
+ index += 2
+ return NewCall(val, args, opts), nil
+ }
+
+ arg, err := getone()
+ if err != nil {
+ return nil, err
+ }
+ args = append(args, arg)
+ }
+ index++
+ return NewCall(val, args, nil), nil
+ default:
+ return nil, fmt.Errorf("unrecognized serialized
Expression key %s", key)
+ }
+ }
+
+ return getone()
+}
diff --git a/go/arrow/compute/expression_test.go
b/go/arrow/compute/expression_test.go
new file mode 100644
index 0000000..6bff7f3
--- /dev/null
+++ b/go/arrow/compute/expression_test.go
@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compute_test
+
+import (
+ "testing"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+ "github.com/apache/arrow/go/v7/arrow/compute"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/apache/arrow/go/v7/arrow/scalar"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestExpressionToString(t *testing.T) {
+ ts, _ := scalar.MakeScalar("1990-10-23
10:23:33.123456").CastTo(arrow.FixedWidthTypes.Timestamp_ns)
+
+ add := compute.NewCall("add",
[]compute.Expression{compute.NewFieldRef("beta"), compute.NewLiteral(3)},
&compute.ArithmeticOptions{})
+
+ tests := []struct {
+ expr compute.Expression
+ expected string
+ }{
+ {compute.NewFieldRef("alpha"), "alpha"},
+ {compute.NewLiteral(3), "3"},
+ {compute.NewLiteral("a"), `"a"`},
+ {compute.NewLiteral("a\nb"), `"a\nb"`},
+ {compute.NewLiteral(&scalar.Boolean{}), "null"},
+ {compute.NewLiteral(&scalar.Int64{}), "null"},
+
{compute.NewLiteral(scalar.NewBinaryScalar(memory.NewBufferBytes([]byte("az")),
+ arrow.BinaryTypes.Binary)), `"617A"`},
+ {compute.NewLiteral(ts), "1990-10-23 10:23:33.123456"},
+ {compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewFieldRef("beta")}, nil),
"add(3, beta)"},
+ {compute.And(compute.NewFieldRef("a"),
compute.NewFieldRef("b")), "(a and b)"},
+ {compute.Or(compute.NewFieldRef("a"),
compute.NewFieldRef("b")), "(a or b)"},
+ {compute.Not(compute.NewFieldRef("a")), "invert(a)"},
+ {compute.Cast(compute.NewFieldRef("a"),
arrow.PrimitiveTypes.Int32),
+ "cast(a, {to_type=int32, allow_int_overflow=false,
allow_time_truncate=false, " +
+ "allow_time_overflow=false,
allow_decimal_truncate=false, " +
+ "allow_float_truncate=false,
allow_invalid_utf8=false})"},
+ {compute.Cast(compute.NewFieldRef("a"), nil),
+ "cast(a, {to_type=null, allow_int_overflow=false,
allow_time_truncate=false, " +
+ "allow_time_overflow=false,
allow_decimal_truncate=false, " +
+ "allow_float_truncate=false,
allow_invalid_utf8=false})"},
+ {compute.Equal(compute.NewFieldRef("a"),
compute.NewLiteral(1)), "(a == 1)"},
+ {compute.Less(compute.NewFieldRef("a"), compute.NewLiteral(2)),
"(a < 2)"},
+ {compute.Greater(compute.NewFieldRef("a"),
compute.NewLiteral(3)), "(a > 3)"},
+ {compute.NotEqual(compute.NewFieldRef("a"),
compute.NewLiteral("a")), `(a != "a")`},
+ {compute.LessEqual(compute.NewFieldRef("a"),
compute.NewLiteral("b")), `(a <= "b")`},
+ {compute.GreaterEqual(compute.NewFieldRef("a"),
compute.NewLiteral("c")), `(a >= "c")`},
+ {compute.Project(
+ []compute.Expression{
+ compute.NewFieldRef("a"),
compute.NewFieldRef("a"), compute.NewLiteral(3), add,
+ }, []string{"a", "renamed_a", "three", "b"}),
+ "{a=a, renamed_a=a, three=3, b=" + add.String() + "}"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.expected, func(t *testing.T) {
+ assert.Equal(t, tt.expected, tt.expr.String())
+ })
+ }
+}
+
+func TestExpressionEquality(t *testing.T) {
+ tests := []struct {
+ exp1 compute.Expression
+ exp2 compute.Expression
+ equal bool
+ }{
+ {compute.NewLiteral(1), compute.NewLiteral(1), true},
+ {compute.NewLiteral(1), compute.NewLiteral(2), false},
+ {compute.NewFieldRef("a"), compute.NewFieldRef("a"), true},
+ {compute.NewFieldRef("a"), compute.NewFieldRef("b"), false},
+ {compute.NewFieldRef("a"), compute.NewLiteral(2), false},
+ {compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")}, nil),
+ compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")}, nil),
true},
+ {compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")}, nil),
+ compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(2), compute.NewLiteral("a")}, nil),
false},
+ {compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")}, nil),
+ compute.NewCall("add",
[]compute.Expression{compute.NewFieldRef("a"), compute.NewLiteral(3)}, nil),
false},
+ {compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")},
&compute.ArithmeticOptions{true}),
+ compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")},
&compute.ArithmeticOptions{true}), true},
+ {compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")},
&compute.ArithmeticOptions{true}),
+ compute.NewCall("add",
[]compute.Expression{compute.NewLiteral(3), compute.NewLiteral("a")},
&compute.ArithmeticOptions{false}), false},
+ {compute.Cast(compute.NewFieldRef("a"),
arrow.PrimitiveTypes.Int32), compute.Cast(compute.NewFieldRef("a"),
arrow.PrimitiveTypes.Int32), true},
+ {compute.Cast(compute.NewFieldRef("a"),
arrow.PrimitiveTypes.Int32), compute.Cast(compute.NewFieldRef("a"),
arrow.PrimitiveTypes.Int64), false},
+ {compute.Cast(compute.NewFieldRef("a"),
arrow.PrimitiveTypes.Int32), compute.NewCall("cast",
[]compute.Expression{compute.NewFieldRef("a")},
compute.NewCastOptions(arrow.PrimitiveTypes.Int32, false)), false},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.exp1.String(), func(t *testing.T) {
+ assert.Equal(t, tt.equal, tt.exp1.Equals(tt.exp2))
+ })
+ }
+}
+
+func TestExpressionHashing(t *testing.T) {
+ set := make(map[uint64]compute.Expression)
+
+ e := compute.NewFieldRef("alpha")
+ set[e.Hash()] = e
+
+ e = compute.NewFieldRef("beta")
+ _, ok := set[e.Hash()]
+ assert.False(t, ok)
+ set[e.Hash()] = e
+
+ e = compute.NewFieldRef("beta")
+ ex, ok := set[e.Hash()]
+ assert.True(t, ok)
+ assert.True(t, e.Equals(ex))
+
+ e = compute.NewLiteral(1)
+ set[e.Hash()] = e
+ _, ok = set[compute.NewLiteral(1).Hash()]
+ assert.True(t, ok)
+ _, ok = set[compute.NewLiteral(3).Hash()]
+ assert.False(t, ok)
+ set[compute.NewLiteral(3).Hash()] = compute.NewLiteral(3)
+
+ e = compute.NullLiteral(arrow.PrimitiveTypes.Int32)
+ set[e.Hash()] = e
+ _, ok = set[compute.NullLiteral(arrow.PrimitiveTypes.Int32).Hash()]
+ assert.True(t, ok)
+ e = compute.NullLiteral(arrow.PrimitiveTypes.Float32)
+ _, ok = set[e.Hash()]
+ assert.False(t, ok)
+ set[e.Hash()] = e
+
+ e = compute.NewCall("add", []compute.Expression{}, nil)
+ set[e.Hash()] = e
+ _, ok = set[compute.NewCall("add", nil, nil).Hash()]
+ assert.True(t, ok)
+ e = compute.NewCall("widgetify", nil, nil)
+ _, ok = set[e.Hash()]
+ assert.False(t, ok)
+ set[e.Hash()] = e
+
+ assert.Len(t, set, 8)
+}
+
+func TestIsScalarExpression(t *testing.T) {
+ assert.True(t, compute.NewLiteral(true).IsScalarExpr())
+ arr := array.MakeFromData(array.NewData(arrow.PrimitiveTypes.Int8, 0,
[]*memory.Buffer{nil, nil}, nil, 0, 0))
+ defer arr.Release()
+
+ assert.False(t, compute.NewLiteral(arr).IsScalarExpr())
+ assert.True(t, compute.NewFieldRef("a").IsScalarExpr())
+}
+
+func TestExpressionIsSatisfiable(t *testing.T) {
+ assert.True(t, compute.NewLiteral(true).IsSatisfiable())
+ assert.False(t, compute.NewLiteral(false).IsSatisfiable())
+
+ null := scalar.MakeNullScalar(arrow.FixedWidthTypes.Boolean)
+ assert.False(t, compute.NewLiteral(null).IsSatisfiable())
+ assert.True(t, compute.NewFieldRef("a").IsSatisfiable())
+ assert.True(t, compute.Equal(compute.NewFieldRef("a"),
compute.NewLiteral(1)).IsSatisfiable())
+ // no constant folding here
+ assert.True(t, compute.Equal(compute.NewLiteral(0),
compute.NewLiteral(1)).IsSatisfiable())
+
+ // when a top level conjunction contains an Expression which is certain
to
+ // evaluate to null, it can only evaluate to null or false
+ neverTrue := compute.And(compute.NewLiteral(null),
compute.NewFieldRef("a"))
+ // this may appear in satisfiable filters if coalesced (for example,
wrapped in fill_na)
+ assert.True(t, compute.NewCall("is_null",
[]compute.Expression{neverTrue}, nil).IsSatisfiable())
+}
+
+func TestExpressionSerializationRoundTrip(t *testing.T) {
+ bldr := array.NewInt32Builder(memory.DefaultAllocator)
+ defer bldr.Release()
+
+ bldr.AppendValues([]int32{1, 2, 3}, nil)
+ lookupArr := bldr.NewArray()
+ defer lookupArr.Release()
+
+ intvalueset := compute.NewDatum(lookupArr)
+ defer intvalueset.Release()
+
+ bldr2 := array.NewFloat64Builder(memory.DefaultAllocator)
+ defer bldr2.Release()
+
+ bldr2.AppendValues([]float64{0.5, 1.0, 2.0}, nil)
+ lookupArr = bldr2.NewArray()
+ defer lookupArr.Release()
+
+ fltvalueset := compute.NewDatum(lookupArr)
+ defer fltvalueset.Release()
+
+ tests := []struct {
+ name string
+ expr compute.Expression
+ }{
+ {"null literal",
compute.NewLiteral(scalar.MakeNullScalar(arrow.Null))},
+ {"null int32 literal",
compute.NewLiteral(scalar.MakeNullScalar(arrow.PrimitiveTypes.Int32))},
+ {"null struct literal",
compute.NewLiteral(scalar.MakeNullScalar(arrow.StructOf(
+ arrow.Field{Name: "i", Type:
arrow.PrimitiveTypes.Int32, Nullable: true},
+ arrow.Field{Name: "s", Type: arrow.BinaryTypes.String,
Nullable: true},
+ )))},
+ {"literal true", compute.NewLiteral(true)},
+ {"literal false", compute.NewLiteral(false)},
+ {"literal int", compute.NewLiteral(1)},
+ {"literal float", compute.NewLiteral(1.125)},
+ {"stringy strings", compute.NewLiteral("stringy strings")},
+ {"field ref", compute.NewFieldRef("field")},
+ {"greater", compute.Greater(compute.NewFieldRef("a"),
compute.NewLiteral(0.25))},
+ {"or", compute.Or(
+ compute.Equal(compute.NewFieldRef("a"),
compute.NewLiteral(1)),
+ compute.NotEqual(compute.NewFieldRef("b"),
compute.NewLiteral("hello")),
+ compute.Equal(compute.NewFieldRef("b"),
compute.NewLiteral("foo bar")))},
+ {"not", compute.Not(compute.NewFieldRef("alpha"))},
+ {"is_in", compute.NewCall("is_in",
[]compute.Expression{compute.NewLiteral(1)},
&compute.SetLookupOptions{ValueSet: intvalueset})},
+ {"is_in cast", compute.NewCall("is_in", []compute.Expression{
+ compute.NewCall("cast",
[]compute.Expression{compute.NewFieldRef("version")},
compute.NewCastOptions(arrow.PrimitiveTypes.Float64, true))},
+ &compute.SetLookupOptions{ValueSet: fltvalueset})},
+ {"is valid", compute.IsValid(compute.NewFieldRef("validity"))},
+ {"lots and", compute.And(
+ compute.And(
+ compute.GreaterEqual(compute.NewFieldRef("x"),
compute.NewLiteral(-1.5)),
+ compute.Less(compute.NewFieldRef("x"),
compute.NewLiteral(0.0))),
+
compute.And(compute.GreaterEqual(compute.NewFieldRef("y"),
compute.NewLiteral(0.0)),
+ compute.Less(compute.NewFieldRef("y"),
compute.NewLiteral(1.5))),
+ compute.And(compute.Greater(compute.NewFieldRef("z"),
compute.NewLiteral(1.5)),
+ compute.LessEqual(compute.NewFieldRef("z"),
compute.NewLiteral(3.0))))},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mem :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+ serialized, err := compute.SerializeExpr(tt.expr, mem)
+ assert.NoError(t, err)
+ defer serialized.Release()
+ roundTripped, err := compute.DeserializeExpr(mem,
serialized)
+ assert.NoError(t, err)
+ defer roundTripped.Release()
+ assert.Truef(t, tt.expr.Equals(roundTripped), "started
with: %s, got: %s", tt.expr, roundTripped)
+ })
+ }
+}
diff --git a/go/arrow/compute/fieldref.go b/go/arrow/compute/fieldref.go
new file mode 100644
index 0000000..88b01bc
--- /dev/null
+++ b/go/arrow/compute/fieldref.go
@@ -0,0 +1,603 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+ "errors"
+ "fmt"
+ "hash/maphash"
+ "math/bits"
+ "reflect"
+ "strconv"
+ "strings"
+ "unicode"
+ "unsafe"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+)
+
+var (
+ ErrEmpty = errors.New("cannot traverse empty field path")
+ ErrNoChildren = errors.New("trying to get child of type with no
children")
+ ErrIndexRange = errors.New("index out of range")
+ ErrMultipleMatches = errors.New("multiple matches")
+ ErrNoMatch = errors.New("no match")
+ ErrInvalid = errors.New("field ref invalid")
+)
+
+func getFields(typ arrow.DataType) []arrow.Field {
+ if nested, ok := typ.(arrow.NestedType); ok {
+ return nested.Fields()
+ }
+ return nil
+}
+
+type listvals interface {
+ ListValues() array.Interface
+}
+
+func getChildren(arr array.Interface) (ret []array.Interface) {
+ switch arr := arr.(type) {
+ case *array.Struct:
+ ret = make([]array.Interface, arr.NumField())
+ for i := 0; i < arr.NumField(); i++ {
+ ret[i] = arr.Field(i)
+ }
+ case listvals:
+ ret = []array.Interface{arr.ListValues()}
+ }
+ return
+}
+
+// FieldPath represents a path to a nested field using indices of child fields.
+// For example, given the indices {5, 9, 3} the field could be retrieved with:
+//
schema.Field(5).Type().(*arrow.StructType).Field(9).Type().(*arrow.StructType).Field(3)
+//
+// Attempting to retrieve a child field using a FieldPath which is not valid
for a given
+// schema will get an error such as an out of range index, or an empty path.
+//
+// FieldPaths provide for drilling down to potentially nested children for
convenience
+// of accepting a slice of fields, a schema or a datatype (which should
contain child fields).
+//
+// A fieldpath can also be used to retrieve a child array.Interface or column
from a record batch.
+type FieldPath []int
+
+func (f FieldPath) String() string {
+ if len(f) == 0 {
+ return "FieldPath(empty)"
+ }
+
+ var b strings.Builder
+ b.WriteString("FieldPath(")
+ for _, i := range f {
+ fmt.Fprint(&b, i)
+ b.WriteByte(' ')
+ }
+ ret := b.String()
+ return ret[:len(ret)-1] + ")"
+}
+
+// Get retrieves the corresponding nested child field by drilling through the
schema's
+// fields as per the field path.
+func (f FieldPath) Get(s *arrow.Schema) (*arrow.Field, error) {
+ return f.GetFieldFromSlice(s.Fields())
+}
+
+// GetFieldFromSlice treats the slice as the top layer of fields, so the first
value
+// in the field path will index into the slice, and then drill down from there.
+func (f FieldPath) GetFieldFromSlice(fields []arrow.Field) (*arrow.Field,
error) {
+ if len(f) == 0 {
+ return nil, ErrEmpty
+ }
+
+ var (
+ depth = 0
+ out *arrow.Field
+ )
+ for _, idx := range f {
+ if len(fields) == 0 {
+ return nil, fmt.Errorf("%w: %s", ErrNoChildren,
out.Type)
+ }
+
+ if idx < 0 || idx >= len(fields) {
+ return nil, fmt.Errorf("%w: indices=%s", ErrIndexRange,
f[:depth+1])
+ }
+
+ out = &fields[idx]
+ fields = getFields(out.Type)
+ depth++
+ }
+
+ return out, nil
+}
+
+func (f FieldPath) getArray(arrs []array.Interface) (array.Interface, error) {
+ if len(f) == 0 {
+ return nil, ErrEmpty
+ }
+
+ var (
+ depth = 0
+ out array.Interface
+ )
+ for _, idx := range f {
+ if len(arrs) == 0 {
+ return nil, fmt.Errorf("%w: %s", ErrNoChildren,
out.DataType())
+ }
+
+ if idx < 0 || idx >= len(arrs) {
+ return nil, fmt.Errorf("%w. indices=%s", ErrIndexRange,
f[:depth+1])
+ }
+
+ out = arrs[idx]
+ arrs = getChildren(out)
+ depth++
+ }
+ return out, nil
+}
+
+// GetFieldFromType returns the nested field from a datatype by drilling into
it's
+// child fields.
+func (f FieldPath) GetFieldFromType(typ arrow.DataType) (*arrow.Field, error) {
+ return f.GetFieldFromSlice(getFields(typ))
+}
+
+// GetField is equivalent to GetFieldFromType(field.Type)
+func (f FieldPath) GetField(field arrow.Field) (*arrow.Field, error) {
+ return f.GetFieldFromType(field.Type)
+}
+
+// GetColumn will return the correct child array by traversing the fieldpath
+// going to the nested arrays of the columns in the record batch.
+func (f FieldPath) GetColumn(batch array.Record) (array.Interface, error) {
+ return f.getArray(batch.Columns())
+}
+
+func (f FieldPath) hash(h *maphash.Hash) {
+ raw := (*reflect.SliceHeader)(unsafe.Pointer(&f)).Data
+
+ var b []byte
+ s := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+ s.Data = raw
+ if bits.UintSize == 32 {
+ s.Len = arrow.Int32Traits.BytesRequired(len(f))
+ } else {
+ s.Len = arrow.Int64Traits.BytesRequired(len(f))
+ }
+ s.Cap = s.Len
+ h.Write(b)
+}
+
+func (f FieldPath) findAll(fields []arrow.Field) []FieldPath {
+ _, err := f.GetFieldFromSlice(fields)
+ if err == nil {
+ return []FieldPath{f}
+ }
+ return nil
+}
+
+// a nameref represents a FieldRef by name of the field
+type nameRef string
+
+func (n nameRef) String() string {
+ return "Name(" + string(n) + ")"
+}
+
+func (ref nameRef) findAll(fields []arrow.Field) []FieldPath {
+ out := []FieldPath{}
+ for i, f := range fields {
+ if f.Name == string(ref) {
+ out = append(out, FieldPath{i})
+ }
+ }
+ return out
+}
+
+func (ref nameRef) hash(h *maphash.Hash) { h.WriteString(string(ref)) }
+
+type matches struct {
+ prefixes []FieldPath
+ refs []*arrow.Field
+}
+
+func (m *matches) add(prefix, suffix FieldPath, fields []arrow.Field) {
+ f, err := suffix.GetFieldFromSlice(fields)
+ if err != nil {
+ panic(err)
+ }
+
+ m.refs = append(m.refs, f)
+ m.prefixes = append(m.prefixes, append(prefix, suffix...))
+}
+
+// refList represents a list of references to use to determine which nested
+// field is being referenced. allowing combinations of field indices and names
+type refList []FieldRef
+
+func (r refList) String() string {
+ var b strings.Builder
+ b.WriteString("Nested(")
+ for _, f := range r {
+ fmt.Fprint(&b, f)
+ b.WriteByte(' ')
+ }
+ ret := b.String()
+ return ret[:len(ret)-1] + ")"
+}
+
+func (ref refList) hash(h *maphash.Hash) {
+ for _, r := range ref {
+ r.hash(h)
+ }
+}
+
+func (ref refList) findAll(fields []arrow.Field) []FieldPath {
+ if len(ref) == 0 {
+ return nil
+ }
+
+ m := matches{}
+ for _, list := range ref[0].FindAll(fields) {
+ m.add(FieldPath{}, list, fields)
+ }
+
+ for _, r := range ref[1:] {
+ next := matches{}
+ for i, f := range m.refs {
+ for _, match := range r.FindAllField(*f) {
+ next.add(m.prefixes[i], match,
getFields(f.Type))
+ }
+ }
+ m = next
+ }
+ return m.prefixes
+}
+
+type refImpl interface {
+ fmt.Stringer
+ findAll(fields []arrow.Field) []FieldPath
+ hash(h *maphash.Hash)
+}
+
+// FieldRef is a descriptor of a (potentially nested) field within a schema.
+//
+// Unlike FieldPath (which is exclusively indices of child fields), FieldRef
+// may reference a field by name. It can be constructed from either
+// a field index, field name, or field path.
+//
+// Nested fields can be referenced as well, given the schema:
+//
+// arrow.NewSchema([]arrow.Field{
+// {Name: "a", Type: arrow.StructOf(arrow.Field{Name: "n",
Type: arrow.Null})},
+// {Name: "b", Type: arrow.PrimitiveTypes.Int32},
+// })
+//
+// the following all indicate the nested field named "n":
+//
+// FieldRefPath(FieldPath{0, 0})
+// FieldRefList("a", 0)
+// FieldRefList("a", "n")
+// FieldRefList(0, "n")
+// NewFieldRefFromDotPath(".a[0]")
+//
+// FieldPaths matching a FieldRef are retrieved with the FindAll* functions
+// Multiple matches are possible because field names may be duplicated within
+// a schema. For example:
+//
+// aIsAmbiguous := arrow.NewSchema([]arrow.Field{
+// {Name: "a", Type: arrow.PrimitiveTypes.Int32},
+// {Name: "a", Type: arrow.PrimitiveTypes.Float32},
+// })
+// matches := FieldRefName("a").FindAll(aIsAmbiguous)
+// assert.Len(matches, 2)
+//
assert.True(matches[0].Get(aIsAmbiguous).Equals(aIsAmbiguous.Field(0))
+//
assert.True(matches[1].Get(aIsAmbiguous).Equals(aIsAmbiguous.Field(1))
+type FieldRef struct {
+ impl refImpl
+}
+
+// FieldRefPath constructs a FieldRef from a given FieldPath
+func FieldRefPath(p FieldPath) FieldRef {
+ return FieldRef{impl: p}
+}
+
+// FieldRefIndex is a convenience function to construct a FieldPath reference
+// of a single index
+func FieldRefIndex(i int) FieldRef {
+ return FieldRef{impl: FieldPath{i}}
+}
+
+// FieldRefName constructs a FieldRef by name
+func FieldRefName(n string) FieldRef {
+ return FieldRef{impl: nameRef(n)}
+}
+
+// FieldRefList takes an arbitrary number of arguments which can be either
+// strings or ints. This will panic if anything other than a string or int
+// is passed in.
+func FieldRefList(elems ...interface{}) FieldRef {
+ list := make(refList, len(elems))
+ for i, e := range elems {
+ switch e := e.(type) {
+ case string:
+ list[i] = FieldRefName(e)
+ case int:
+ list[i] = FieldRefIndex(e)
+ }
+ }
+ return FieldRef{impl: list}
+}
+
+// NewFieldRefFromDotPath parses a dot path into a field ref.
+//
+// dot_path = '.' name
+// | '[' digit+ ']'
+// | dot_path+
+//
+// Examples
+//
+// ".alpha" => FieldRefName("alpha")
+// "[2]" => FieldRefIndex(2)
+// ".beta[3]" => FieldRefList("beta", 3)
+// "[5].gamma.delta[7]" => FieldRefList(5, "gamma", "delta", 7)
+// ".hello world" => FieldRefName("hello world")
+// `.\[y\]\\tho\.\` => FieldRef(`[y]\tho.\`)
+//
+// Note: when parsing a name, a '\' preceding any other character will be
+// dropped from the resulting name. therefore if a name must contain the
characters
+// '.', '\', '[' or ']' then they must be escaped with a preceding '\'.
+func NewFieldRefFromDotPath(dotpath string) (out FieldRef, err error) {
+ if len(dotpath) == 0 {
+ return out, fmt.Errorf("%w dotpath was empty", ErrInvalid)
+ }
+
+ parseName := func() string {
+ var name string
+ for {
+ idx := strings.IndexAny(dotpath, `\[.`)
+ if idx == -1 {
+ name += dotpath
+ dotpath = ""
+ break
+ }
+
+ if dotpath[idx] != '\\' {
+ // subscript for a new field ref
+ name += dotpath[:idx]
+ dotpath = dotpath[idx:]
+ break
+ }
+
+ if len(dotpath) == idx+1 {
+ // dotpath ends with a backslash; consume it all
+ name += dotpath
+ dotpath = ""
+ break
+ }
+
+ // append all characters before backslash, then the
character which follows it
+ name += dotpath[:idx] + string(dotpath[idx+1])
+ dotpath = dotpath[idx+2:]
+ }
+ return name
+ }
+
+ children := make([]FieldRef, 0)
+
+ for len(dotpath) > 0 {
+ subscript := dotpath[0]
+ dotpath = dotpath[1:]
+ switch subscript {
+ case '.':
+ // next element is a name
+ children = append(children,
FieldRef{nameRef(parseName())})
+ case '[':
+ subend := strings.IndexFunc(dotpath, func(r rune) bool
{ return !unicode.IsDigit(r) })
+ if subend == -1 || dotpath[subend] != ']' {
+ return out, fmt.Errorf("%w: dot path '%s'
contained an unterminated index", ErrInvalid, dotpath)
+ }
+ idx, _ := strconv.Atoi(dotpath[:subend])
+ children = append(children, FieldRef{FieldPath{idx}})
+ dotpath = dotpath[subend+1:]
+ default:
+ return out, fmt.Errorf("%w: dot path must begin with
'[' or '.' got '%s'", ErrInvalid, dotpath)
+ }
+ }
+
+ out.flatten(children)
+ return
+}
+
+func (f FieldRef) hash(h *maphash.Hash) { f.impl.hash(h) }
+
+// Hash produces a hash of this field reference and takes in a seed so that
+// it can maintain consistency across multiple places / processes /etc.
+func (f FieldRef) Hash(seed maphash.Seed) uint64 {
+ h := maphash.Hash{}
+ h.SetSeed(seed)
+ f.hash(&h)
+ return h.Sum64()
+}
+
+// IsName returns true if this fieldref is a name reference
+func (f *FieldRef) IsName() bool {
+ _, ok := f.impl.(nameRef)
+ return ok
+}
+
+// IsFieldPath returns true if this FieldRef uses a fieldpath
+func (f *FieldRef) IsFieldPath() bool {
+ _, ok := f.impl.(FieldPath)
+ return ok
+}
+
+// IsNested returns true if this FieldRef expects to represent
+// a nested field.
+func (f *FieldRef) IsNested() bool {
+ switch impl := f.impl.(type) {
+ case nameRef:
+ return false
+ case FieldPath:
+ return len(impl) > 1
+ default:
+ return true
+ }
+}
+
+// Name returns the name of the field this references if it is
+// a Name reference, otherwise the empty string
+func (f *FieldRef) Name() string {
+ n, _ := f.impl.(nameRef)
+ return string(n)
+}
+
+// FieldPath returns the fieldpath that this FieldRef uses, otherwise
+// an empty FieldPath if it's not a FieldPath reference
+func (f *FieldRef) FieldPath() FieldPath {
+ p, _ := f.impl.(FieldPath)
+ return p
+}
+
+func (f *FieldRef) Equals(other FieldRef) bool {
+ return reflect.DeepEqual(f.impl, other.impl)
+}
+
+func (f *FieldRef) flatten(children []FieldRef) {
+ out := make([]FieldRef, 0, len(children))
+
+ var populate func(refImpl)
+ populate = func(refs refImpl) {
+ switch r := refs.(type) {
+ case nameRef:
+ out = append(out, FieldRef{r})
+ case FieldPath:
+ out = append(out, FieldRef{r})
+ case refList:
+ for _, c := range r {
+ populate(c.impl)
+ }
+ }
+ }
+
+ populate(refList(children))
+
+ if len(out) == 1 {
+ f.impl = out[0].impl
+ } else {
+ f.impl = refList(out)
+ }
+}
+
+// FindAll returns all the fieldpaths which this FieldRef matches in the given
+// slice of fields.
+func (f FieldRef) FindAll(fields []arrow.Field) []FieldPath {
+ return f.impl.findAll(fields)
+}
+
+// FindAllField returns all the fieldpaths that this FieldRef matches against
+// the type of the given field.
+func (f FieldRef) FindAllField(field arrow.Field) []FieldPath {
+ return f.impl.findAll(getFields(field.Type))
+}
+
+// FindOneOrNone is a convenience helper that will either return 1 fieldpath,
+// or an empty fieldpath, and will return an error if there are multiple
matches.
+func (f FieldRef) FindOneOrNone(schema *arrow.Schema) (FieldPath, error) {
+ matches := f.FindAll(schema.Fields())
+ if len(matches) > 1 {
+ return nil, fmt.Errorf("%w for %s in %s", ErrMultipleMatches,
f, schema)
+ }
+ if len(matches) == 0 {
+ return nil, nil
+ }
+ return matches[0], nil
+}
+
+// FindOneOrNoneRecord is like FindOneOrNone but for the schema of a record,
+// returning an error only if there are multiple matches.
+func (f FieldRef) FindOneOrNoneRecord(root array.Record) (FieldPath, error) {
+ return f.FindOneOrNone(root.Schema())
+}
+
+// FindOne returns an error if the field isn't matched or if there are
multiple matches
+// otherwise it returns the path to the single valid match.
+func (f FieldRef) FindOne(schema *arrow.Schema) (FieldPath, error) {
+ matches := f.FindAll(schema.Fields())
+ if len(matches) == 0 {
+ return nil, fmt.Errorf("%w for %s in %s", ErrNoMatch, f, schema)
+ }
+ if len(matches) > 1 {
+ return nil, fmt.Errorf("%w for %s in %s", ErrMultipleMatches,
f, schema)
+ }
+ return matches[0], nil
+}
+
+// GetAllColumns gets all the matching column arrays from the given record that
+// this FieldRef references.
+func (f FieldRef) GetAllColumns(root array.Record) ([]array.Interface, error) {
+ out := make([]array.Interface, 0)
+ for _, m := range f.FindAll(root.Schema().Fields()) {
+ n, err := m.GetColumn(root)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, n)
+ }
+ return out, nil
+}
+
+// GetOneField will return a pointer to a field or an error if it is not found
+// or if there are multiple matches.
+func (f FieldRef) GetOneField(schema *arrow.Schema) (*arrow.Field, error) {
+ match, err := f.FindOne(schema)
+ if err != nil {
+ return nil, err
+ }
+
+ return match.GetFieldFromSlice(schema.Fields())
+}
+
+// GetOneOrNone will return a field or a nil if the field is found or not, and
+// only errors if there are multiple matches.
+func (f FieldRef) GetOneOrNone(schema *arrow.Schema) (*arrow.Field, error) {
+ match, err := f.FindOneOrNone(schema)
+ if err != nil {
+ return nil, err
+ }
+ if len(match) == 0 {
+ return nil, nil
+ }
+ return match.GetFieldFromSlice(schema.Fields())
+}
+
+// GetOneColumnOrNone returns either a nil or the referenced array if it can be
+// found, erroring only if there is an ambiguous multiple matches.
+func (f FieldRef) GetOneColumnOrNone(root array.Record) (array.Interface,
error) {
+ match, err := f.FindOneOrNoneRecord(root)
+ if err != nil {
+ return nil, err
+ }
+ if len(match) == 0 {
+ return nil, nil
+ }
+ return match.GetColumn(root)
+}
+
+func (f FieldRef) String() string {
+ return "FieldRef." + f.impl.String()
+}
diff --git a/go/arrow/compute/fieldref_test.go
b/go/arrow/compute/fieldref_test.go
new file mode 100644
index 0000000..244f7ac
--- /dev/null
+++ b/go/arrow/compute/fieldref_test.go
@@ -0,0 +1,316 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute_test
+
+import (
+ "testing"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+ "github.com/apache/arrow/go/v7/arrow/compute"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestFieldPathBasics(t *testing.T) {
+ f0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f1 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f2 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f3 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+
+ s := arrow.NewSchema([]arrow.Field{f0, f1, f2, f3}, nil)
+
+ for i := range s.Fields() {
+ f, err := compute.FieldPath{i}.Get(s)
+ assert.NoError(t, err)
+ assert.Equal(t, s.Field(i), *f)
+ }
+
+ f, err := compute.FieldPath{}.Get(s)
+ assert.Nil(t, f)
+ assert.ErrorIs(t, err, compute.ErrEmpty)
+
+ f, err = compute.FieldPath{len(s.Fields()) * 2}.Get(s)
+ assert.Nil(t, f)
+ assert.ErrorIs(t, err, compute.ErrIndexRange)
+}
+
+func TestFieldRefBasics(t *testing.T) {
+ f0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f1 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f2 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f3 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+
+ s := arrow.NewSchema([]arrow.Field{f0, f1, f2, f3}, nil)
+
+ // lookup by index returns Indices{index}
+ for i := range s.Fields() {
+ assert.ElementsMatch(t, []compute.FieldPath{{i}},
compute.FieldRefIndex(i).FindAll(s.Fields()))
+ }
+
+ // out of range index results in failure to match
+ assert.Empty(t,
compute.FieldRefIndex(len(s.Fields())*2).FindAll(s.Fields()))
+
+ // lookup by name returns the indices of both matching fields
+ assert.Equal(t, []compute.FieldPath{{0}, {2}},
compute.FieldRefName("alpha").FindAll(s.Fields()))
+ assert.Equal(t, []compute.FieldPath{{1}, {3}},
compute.FieldRefName("beta").FindAll(s.Fields()))
+}
+
+func TestFieldRefDotPath(t *testing.T) {
+ ref, err := compute.NewFieldRefFromDotPath(`.alpha`)
+ assert.True(t, ref.IsName())
+ assert.Equal(t, "alpha", ref.Name())
+ assert.False(t, ref.IsFieldPath())
+ assert.False(t, ref.IsNested())
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefName("alpha"), ref)
+ assert.True(t, ref.Equals(compute.FieldRefName("alpha")))
+
+ ref, err = compute.NewFieldRefFromDotPath(`..`)
+ assert.Empty(t, ref.Name())
+ assert.False(t, ref.IsName())
+ assert.False(t, ref.IsFieldPath())
+ assert.Nil(t, ref.FieldPath())
+ assert.True(t, ref.IsNested())
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefList("", ""), ref)
+
+ ref, err = compute.NewFieldRefFromDotPath(`[2]`)
+ assert.False(t, ref.IsName())
+ assert.True(t, ref.IsFieldPath())
+ assert.Equal(t, compute.FieldPath{2}, ref.FieldPath())
+ assert.False(t, ref.IsNested())
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefIndex(2), ref)
+
+ ref, err = compute.NewFieldRefFromDotPath(`.beta[3]`)
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefList("beta", 3), ref)
+
+ ref, err = compute.NewFieldRefFromDotPath(`[5].gamma.delta[7]`)
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefList(5, "gamma", "delta", 7), ref)
+
+ ref, err = compute.NewFieldRefFromDotPath(`.hello world`)
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefName("hello world"), ref)
+
+ ref, err = compute.NewFieldRefFromDotPath(`.\[y\]\\tho\.\`)
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldRefName(`[y]\tho.\`), ref)
+
+ _, err = compute.NewFieldRefFromDotPath(``)
+ assert.ErrorIs(t, err, compute.ErrInvalid)
+
+ _, err = compute.NewFieldRefFromDotPath(`alpha`)
+ assert.ErrorIs(t, err, compute.ErrInvalid)
+
+ _, err = compute.NewFieldRefFromDotPath(`[134234`)
+ assert.ErrorIs(t, err, compute.ErrInvalid)
+
+ _, err = compute.NewFieldRefFromDotPath(`[1stuf]`)
+ assert.ErrorIs(t, err, compute.ErrInvalid)
+}
+
+func TestFieldPathNested(t *testing.T) {
+ f0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f1_0 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f1 := arrow.Field{Name: "beta", Type: arrow.StructOf(f1_0)}
+ f2_0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f2_1_0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f2_1_1 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f2_1 := arrow.Field{Name: "gamma", Type: arrow.StructOf(f2_1_0, f2_1_1)}
+ f2 := arrow.Field{Name: "beta", Type: arrow.StructOf(f2_0, f2_1)}
+ s := arrow.NewSchema([]arrow.Field{f0, f1, f2}, nil)
+
+ f, err := compute.FieldPath{0}.Get(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f0, *f)
+
+ f, err = compute.FieldPath{0, 0}.Get(s)
+ assert.ErrorIs(t, err, compute.ErrNoChildren)
+ assert.Nil(t, f)
+
+ f, err = compute.FieldPath{1, 0}.Get(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f1_0, *f)
+
+ f, err = compute.FieldPath{2, 0}.Get(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f2_0, *f)
+
+ f, err = compute.FieldPath{2, 1, 0}.Get(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f2_1_0, *f)
+
+ f, err = compute.FieldPath{1, 0}.GetField(s.Field(2))
+ assert.NoError(t, err)
+ assert.Equal(t, f2_1_0, *f)
+
+ f, err = compute.FieldPath{2, 1, 1}.Get(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f2_1_1, *f)
+}
+
+func TestFindFuncs(t *testing.T) {
+ f0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f1_0 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f1 := arrow.Field{Name: "alpha", Type: arrow.StructOf(f1_0)}
+ f2_0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f2_1_0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f2_1_1 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f2_1 := arrow.Field{Name: "gamma", Type: arrow.StructOf(f2_1_0, f2_1_1)}
+ f2 := arrow.Field{Name: "beta", Type: arrow.StructOf(f2_0, f2_1)}
+ s := arrow.NewSchema([]arrow.Field{f0, f1, f2}, nil)
+
+ assert.Equal(t, []compute.FieldPath{{1}},
compute.FieldRefName("gamma").FindAllField(f2))
+ fp, err := compute.FieldRefName("alpha").FindOneOrNone(s)
+ assert.ErrorIs(t, err, compute.ErrMultipleMatches)
+ assert.Len(t, fp, 0)
+ fp, err = compute.FieldRefName("alpha").FindOne(s)
+ assert.ErrorIs(t, err, compute.ErrMultipleMatches)
+ assert.Len(t, fp, 0)
+
+ fp, err = compute.FieldRefName("beta").FindOneOrNone(s)
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldPath{2}, fp)
+ fp, err = compute.FieldRefName("beta").FindOne(s)
+ assert.NoError(t, err)
+ assert.Equal(t, compute.FieldPath{2}, fp)
+
+ fp, err = compute.FieldRefName("gamma").FindOneOrNone(s)
+ assert.NoError(t, err)
+ assert.Len(t, fp, 0)
+
+ fp, err = compute.FieldRefName("gamma").FindOne(s)
+ assert.ErrorIs(t, err, compute.ErrNoMatch)
+ assert.Nil(t, fp)
+}
+
+func TestGetFieldFuncs(t *testing.T) {
+ f0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f1_0 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f1 := arrow.Field{Name: "alpha", Type: arrow.StructOf(f1_0)}
+ f2_0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f2_1_0 := arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32}
+ f2_1_1 := arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32}
+ f2_1 := arrow.Field{Name: "gamma", Type: arrow.StructOf(f2_1_0, f2_1_1)}
+ f2 := arrow.Field{Name: "beta", Type: arrow.StructOf(f2_0, f2_1)}
+ s := arrow.NewSchema([]arrow.Field{f0, f1, f2}, nil)
+
+ ref, err := compute.NewFieldRefFromDotPath(`[2].alpha`)
+ assert.NoError(t, err)
+
+ f, err := ref.GetOneField(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f2_0, *f)
+ f, err = ref.GetOneOrNone(s)
+ assert.NoError(t, err)
+ assert.Equal(t, f2_0, *f)
+
+ ref = compute.FieldRefList("beta", "gamma", 2)
+ f, err = ref.GetOneField(s)
+ assert.ErrorIs(t, err, compute.ErrNoMatch)
+ assert.Nil(t, f)
+ f, err = ref.GetOneOrNone(s)
+ assert.NoError(t, err)
+ assert.Nil(t, f)
+
+ f, err = compute.FieldRefName("alpha").GetOneOrNone(s)
+ assert.ErrorIs(t, err, compute.ErrMultipleMatches)
+ assert.Nil(t, f)
+}
+
+func TestFieldRefRecord(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ alphaBldr := array.NewInt32Builder(mem)
+ defer alphaBldr.Release()
+
+ betaBldr := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer betaBldr.Release()
+
+ gammaBldr := array.NewStructBuilder(mem, arrow.StructOf(
+ arrow.Field{Name: "alpha", Type: arrow.PrimitiveTypes.Int32,
Nullable: true},
+ arrow.Field{Name: "beta", Type: arrow.PrimitiveTypes.Int32,
Nullable: true}))
+ defer gammaBldr.Release()
+
+ alphaBldr.AppendValues([]int32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, nil)
+ betaBldr.AppendValues([]int32{0, 3, 7, 8, 8, 10, 13, 14, 17, 20, 22},
[]bool{true, true, true, false, true, true, true, true, true, true})
+ for i := 0; i < 22; i++ {
+ betaBldr.ValueBuilder().(*array.Int32Builder).Append(int32(i *
2))
+ }
+
+ gammaBldr.AppendValues([]bool{true, true, true, true, true, true, true,
true, true, true})
+
gammaBldr.FieldBuilder(0).(*array.Int32Builder).AppendValues([]int32{10, 20,
30, 40, 50, 60, 70, 80, 90, 100}, nil)
+
gammaBldr.FieldBuilder(1).(*array.Int32Builder).AppendValues([]int32{-10, -20,
-30, -40, -50, -60, -70, -80, -90, -100}, nil)
+
+ alpha := alphaBldr.NewInt32Array()
+ defer alpha.Release()
+ beta := betaBldr.NewListArray()
+ defer beta.Release()
+ gamma := gammaBldr.NewStructArray()
+ defer gamma.Release()
+
+ rec := array.NewRecord(arrow.NewSchema([]arrow.Field{
+ {Name: "alpha", Type: alpha.DataType(), Nullable: true},
+ {Name: "alpha", Type: beta.DataType(), Nullable: true},
+ {Name: "alpha", Type: gamma.DataType(), Nullable: true},
+ }, nil), []array.Interface{alpha, beta, gamma}, 10)
+ defer rec.Release()
+
+ arr, err := compute.FieldPath{2, 0}.GetColumn(rec)
+ assert.NoError(t, err)
+ assert.Same(t, gamma.Field(0), arr)
+
+ arr, err = compute.FieldPath{}.GetColumn(rec)
+ assert.ErrorIs(t, err, compute.ErrEmpty)
+ assert.Nil(t, arr)
+
+ arr, err = compute.FieldPath{1, 0}.GetColumn(rec)
+ assert.NoError(t, err)
+ assert.Same(t, beta.ListValues(), arr)
+
+ arr, err = compute.FieldPath{1, 0, 0}.GetColumn(rec)
+ assert.ErrorIs(t, err, compute.ErrNoChildren)
+ assert.Nil(t, arr)
+
+ arr, err = compute.FieldPath{2, 2}.GetColumn(rec)
+ assert.ErrorIs(t, err, compute.ErrIndexRange)
+ assert.Nil(t, arr)
+
+ arrs, err := compute.FieldRefName("alpha").GetAllColumns(rec)
+ assert.NoError(t, err)
+ assert.Equal(t, []array.Interface{alpha, beta, gamma}, arrs)
+
+ arrs, err = compute.FieldRefName("delta").GetAllColumns(rec)
+ assert.NoError(t, err)
+ assert.Len(t, arrs, 0)
+
+ arr, err = compute.FieldRefName("delta").GetOneColumnOrNone(rec)
+ assert.NoError(t, err)
+ assert.Nil(t, arr)
+
+ arr, err = compute.FieldRefName("alpha").GetOneColumnOrNone(rec)
+ assert.ErrorIs(t, err, compute.ErrMultipleMatches)
+ assert.Nil(t, arr)
+
+ arr, err = compute.FieldRefList("alpha", "beta").GetOneColumnOrNone(rec)
+ assert.NoError(t, err)
+ assert.Same(t, gamma.Field(1), arr)
+}
diff --git a/go/arrow/compute/hash_util.go b/go/arrow/compute/hash_util.go
new file mode 100644
index 0000000..d0ecca5
--- /dev/null
+++ b/go/arrow/compute/hash_util.go
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+// ADAPTED FROM HASH UTILITIES FOR BOOST
+
+func hashCombine(seed, value uint64) uint64 {
+ seed ^= value + 0x9e3779b9 + (seed << 6) + (seed >> 2)
+ return seed
+}
diff --git a/go/arrow/compute/no_exec.go b/go/arrow/compute/no_exec.go
new file mode 100644
index 0000000..ebde1dc
--- /dev/null
+++ b/go/arrow/compute/no_exec.go
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// this file is used to provide dummy implementations for exec
+// functions that are called elsewhere in the compute package by
+// the expression handlers so that the logic can stay where it should
+// belong.
+
+package compute
+
+import (
+ "context"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+)
+
+// dummy function which always returns false when not loading the C++ lib
+func isFuncScalar(funcName string) bool {
+ return false
+}
+
+type boundRef uintptr
+
+func (boundRef) release() {}
+
+// when compiled without the c++ library (the build tags control whether it
looks for it)
+// then we do not have pure go implementation of the expression binding
currently.
+func bindExprSchema(context.Context, memory.Allocator, Expression,
*arrow.Schema) (boundRef, ValueDescr, int, Expression, error) {
+ panic("arrow/compute: bind expression not implemented")
+}
diff --git a/go/arrow/compute/utils.go b/go/arrow/compute/utils.go
new file mode 100644
index 0000000..8976fc8
--- /dev/null
+++ b/go/arrow/compute/utils.go
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+ "io"
+ "math"
+
+ "github.com/apache/arrow/go/v7/arrow/bitutil"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "golang.org/x/xerrors"
+)
+
+type bufferWriteSeeker struct {
+ buf *memory.Buffer
+ pos int
+ mem memory.Allocator
+}
+
+func (b *bufferWriteSeeker) Reserve(nbytes int) {
+ if b.buf == nil {
+ b.buf = memory.NewResizableBuffer(b.mem)
+ }
+ newCap := int(math.Max(float64(b.buf.Cap()), 256))
+ for newCap < b.pos+nbytes {
+ newCap = bitutil.NextPowerOf2(newCap)
+ }
+ b.buf.Reserve(newCap)
+}
+
+func (b *bufferWriteSeeker) Write(p []byte) (n int, err error) {
+ if len(p) == 0 {
+ return 0, nil
+ }
+
+ if b.buf == nil {
+ b.Reserve(len(p))
+ } else if b.pos+len(p) >= b.buf.Cap() {
+ b.Reserve(len(p))
+ }
+
+ return b.UnsafeWrite(p)
+}
+
+func (b *bufferWriteSeeker) UnsafeWrite(p []byte) (n int, err error) {
+ n = copy(b.buf.Buf()[b.pos:], p)
+ b.pos += len(p)
+ if b.pos > b.buf.Len() {
+ b.buf.ResizeNoShrink(b.pos)
+ }
+ return
+}
+
+func (b *bufferWriteSeeker) Seek(offset int64, whence int) (int64, error) {
+ newpos, offs := 0, int(offset)
+ switch whence {
+ case io.SeekStart:
+ newpos = offs
+ case io.SeekCurrent:
+ newpos = b.pos + offs
+ case io.SeekEnd:
+ newpos = b.buf.Len() + offs
+ }
+ if newpos < 0 {
+ return 0, xerrors.New("negative result pos")
+ }
+ b.pos = newpos
+ return int64(newpos), nil
+}
diff --git a/go/arrow/compute/valueshape_string.go
b/go/arrow/compute/valueshape_string.go
new file mode 100644
index 0000000..1381d2e
--- /dev/null
+++ b/go/arrow/compute/valueshape_string.go
@@ -0,0 +1,25 @@
+// Code generated by "stringer -type=ValueShape -linecomment"; DO NOT EDIT.
+
+package compute
+
+import "strconv"
+
+func _() {
+ // An "invalid array index" compiler error signifies that the constant
values have changed.
+ // Re-run the stringer command to generate them again.
+ var x [1]struct{}
+ _ = x[ShapeAny-0]
+ _ = x[ShapeArray-1]
+ _ = x[ShapeScalar-2]
+}
+
+const _ValueShape_name = "anyarrayscalar"
+
+var _ValueShape_index = [...]uint8{0, 3, 8, 14}
+
+func (i ValueShape) String() string {
+ if i < 0 || i >= ValueShape(len(_ValueShape_index)-1) {
+ return "ValueShape(" + strconv.FormatInt(int64(i), 10) + ")"
+ }
+ return _ValueShape_name[_ValueShape_index[i]:_ValueShape_index[i+1]]
+}
diff --git a/go/arrow/datatype_extension.go b/go/arrow/datatype_extension.go
index 90b9a15..495a23f 100644
--- a/go/arrow/datatype_extension.go
+++ b/go/arrow/datatype_extension.go
@@ -157,6 +157,13 @@ func (e *ExtensionBase) StorageType() DataType { return
e.Storage }
func (e *ExtensionBase) Fingerprint() string { return typeFingerprint(e) +
e.Storage.Fingerprint() }
+func (e *ExtensionBase) Fields() []Field {
+ if nested, ok := e.Storage.(NestedType); ok {
+ return nested.Fields()
+ }
+ return nil
+}
+
// this no-op exists to ensure that this type must be embedded in any
user-defined extension type.
func (ExtensionBase) mustEmbedExtensionBase() {}
diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go
index 3d11884..0d4f834 100644
--- a/go/arrow/datatype_fixedwidth.go
+++ b/go/arrow/datatype_fixedwidth.go
@@ -220,17 +220,17 @@ func (t Time64) ToTime(unit TimeUnit) time.Time {
}
const (
- Nanosecond TimeUnit = iota
- Microsecond
+ Second TimeUnit = iota
Millisecond
- Second
+ Microsecond
+ Nanosecond
)
func (u TimeUnit) Multiplier() time.Duration {
- return [...]time.Duration{time.Nanosecond, time.Microsecond,
time.Millisecond, time.Second}[uint(u)&3]
+ return [...]time.Duration{time.Second, time.Millisecond,
time.Microsecond, time.Nanosecond}[uint(u)&3]
}
-func (u TimeUnit) String() string { return [...]string{"ns", "us", "ms",
"s"}[uint(u)&3] }
+func (u TimeUnit) String() string { return [...]string{"s", "ms", "us",
"ns"}[uint(u)&3] }
// TimestampType is encoded as a 64-bit signed integer since the UNIX epoch
(2017-01-01T00:00:00Z).
// The zero-value is a nanosecond and time zone neutral. Time zone neutral can
be
diff --git a/go/arrow/datatype_nested.go b/go/arrow/datatype_nested.go
index 98ac5f9..ee4a1be 100644
--- a/go/arrow/datatype_nested.go
+++ b/go/arrow/datatype_nested.go
@@ -21,6 +21,11 @@ import (
"strings"
)
+type NestedType interface {
+ DataType
+ Fields() []Field
+}
+
// ListType describes a nested type in which each array slot contains
// a variable-size sequence of values, all having the same relative type.
type ListType struct {
@@ -83,6 +88,8 @@ func (t *ListType) ElemField() Field {
return t.elem
}
+func (t *ListType) Fields() []Field { return []Field{t.ElemField()} }
+
// FixedSizeListType describes a nested type in which each array slot contains
// a fixed-size sequence of values, all having the same relative type.
type FixedSizeListType struct {
@@ -155,6 +162,8 @@ func (t *FixedSizeListType) Fingerprint() string {
return ""
}
+func (t *FixedSizeListType) Fields() []Field { return []Field{t.ElemField()} }
+
// StructType describes a nested type parameterized by an ordered sequence
// of relative types, called its fields.
type StructType struct {
@@ -302,6 +311,8 @@ func (t *MapType) Fingerprint() string {
return fingerprint + "{" + keyFingerprint + itemFingerprint + "}"
}
+func (t *MapType) Fields() []Field { return t.ValueType().Fields() }
+
type Field struct {
Name string // Field name
Type DataType // The field's data type
@@ -360,6 +371,7 @@ func (f Field) String() string {
var (
_ DataType = (*ListType)(nil)
+ _ DataType = (*FixedSizeListType)(nil)
_ DataType = (*StructType)(nil)
_ DataType = (*MapType)(nil)
)
diff --git a/go/arrow/scalar/nested.go b/go/arrow/scalar/nested.go
index 483dbfe..e09ef09 100644
--- a/go/arrow/scalar/nested.go
+++ b/go/arrow/scalar/nested.go
@@ -115,6 +115,10 @@ func NewListScalar(val array.Interface) *List {
return &List{scalar{arrow.ListOf(val.DataType()), true},
array.MakeFromData(val.Data())}
}
+func NewListScalarData(val *array.Data) *List {
+ return &List{scalar{arrow.ListOf(val.DataType()), true},
array.MakeFromData(val)}
+}
+
func makeMapType(typ *arrow.StructType) *arrow.MapType {
debug.Assert(len(typ.Fields()) == 2, "must pass struct with only 2
fields for MapScalar")
return arrow.MapOf(typ.Field(0).Type, typ.Field(1).Type)
@@ -165,6 +169,14 @@ type Struct struct {
Value Vector
}
+func (s *Struct) Release() {
+ for _, v := range s.Value {
+ if v, ok := v.(Releasable); ok {
+ v.Release()
+ }
+ }
+}
+
func (s *Struct) Field(name string) (Scalar, error) {
idx, ok := s.Type.(*arrow.StructType).FieldIdx(name)
if !ok {
diff --git a/go/arrow/scalar/parse.go b/go/arrow/scalar/parse.go
index 6a54628..a217dea 100644
--- a/go/arrow/scalar/parse.go
+++ b/go/arrow/scalar/parse.go
@@ -17,7 +17,10 @@
package scalar
import (
+ "errors"
+ "fmt"
"math/bits"
+ "reflect"
"strconv"
"time"
@@ -28,6 +31,336 @@ import (
"golang.org/x/xerrors"
)
+type TypeToScalar interface {
+ ToScalar() (Scalar, error)
+}
+
+type TypeFromScalar interface {
+ FromStructScalar(*Struct) error
+}
+
+type hasTypename interface {
+ TypeName() string
+}
+
+var (
+ hasTypenameType = reflect.TypeOf((*hasTypename)(nil)).Elem()
+ dataTypeType = reflect.TypeOf((*arrow.DataType)(nil)).Elem()
+)
+
+func FromScalar(sc *Struct, val interface{}) error {
+ if sc == nil || len(sc.Value) == 0 {
+ return nil
+ }
+
+ if v, ok := val.(TypeFromScalar); ok {
+ return v.FromStructScalar(sc)
+ }
+
+ v := reflect.ValueOf(val)
+ if v.Kind() != reflect.Ptr {
+ return errors.New("fromscalar must be given a pointer to an
object to populate")
+ }
+ value := reflect.Indirect(v)
+
+ for i := 0; i < value.Type().NumField(); i++ {
+ fld := value.Type().Field(i)
+ tag := fld.Tag.Get("compute")
+ if tag == "-" || fld.Name == "_type_name" {
+ continue
+ }
+
+ fldVal, err := sc.Field(tag)
+ if err != nil {
+ return err
+ }
+ if err := setFromScalar(fldVal, value.Field(i)); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func setFromScalar(s Scalar, v reflect.Value) error {
+ if v.Type() == dataTypeType {
+ v.Set(reflect.ValueOf(s.DataType()))
+ return nil
+ }
+
+ switch s := s.(type) {
+ case BinaryScalar:
+ value := s.value().(*memory.Buffer)
+ switch v.Kind() {
+ case reflect.String:
+ if value == nil {
+ v.SetString("")
+ } else {
+ v.SetString(string(value.Bytes()))
+ }
+ default:
+ if value == nil {
+ v.SetBytes(nil)
+ } else {
+ v.SetBytes(value.Bytes())
+ }
+ }
+ case ListScalar:
+ return fromListScalar(s, v)
+ case *Struct:
+ return FromScalar(s, v.Interface())
+ default:
+ if v.Type() == reflect.TypeOf(arrow.TimeUnit(0)) {
+
v.Set(reflect.ValueOf(arrow.TimeUnit(s.value().(uint32))))
+ } else {
+ v.Set(reflect.ValueOf(s.value()))
+ }
+ }
+ return nil
+}
+
+func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) {
+ switch v := val.(type) {
+ case arrow.DataType:
+ return MakeScalar(v), nil
+ case TypeToScalar:
+ return v.ToScalar()
+ }
+
+ v := reflect.Indirect(reflect.ValueOf(val))
+ switch v.Kind() {
+ case reflect.Struct:
+ scalars := make([]Scalar, 0, v.Type().NumField())
+ fields := make([]string, 0, v.Type().NumField())
+ for i := 0; i < v.Type().NumField(); i++ {
+ fld := v.Type().Field(i)
+ tag := fld.Tag.Get("compute")
+ if tag == "-" {
+ continue
+ }
+
+ fldVal := v.Field(i)
+ s, err := ToScalar(fldVal.Interface(), mem)
+ if err != nil {
+ return nil, err
+ }
+ scalars = append(scalars, s)
+ fields = append(fields, tag)
+ }
+
+ if v.Type().Implements(hasTypenameType) {
+ t := val.(hasTypename)
+ scalars = append(scalars,
NewBinaryScalar(memory.NewBufferBytes([]byte(t.TypeName())),
arrow.BinaryTypes.Binary))
+ fields = append(fields, "_type_name")
+ }
+
+ return NewStructScalarWithNames(scalars, fields)
+ case reflect.Slice:
+ return createListScalar(v, mem)
+ default:
+ return MakeScalar(val), nil
+ }
+}
+
+func createListScalar(sliceval reflect.Value, mem memory.Allocator) (Scalar,
error) {
+ if sliceval.Kind() != reflect.Slice {
+ return nil, xerrors.Errorf("createListScalar only works for
slices, not %s", sliceval.Kind())
+ }
+
+ var arr array.Interface
+
+ switch sliceval.Type().Elem().Kind() {
+ case reflect.String:
+ bldr := array.NewStringBuilder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]string), nil)
+ arr = bldr.NewArray()
+ case reflect.Bool:
+ bldr := array.NewBooleanBuilder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]bool), nil)
+ arr = bldr.NewArray()
+ case reflect.Int8:
+ bldr := array.NewInt8Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]int8), nil)
+ arr = bldr.NewArray()
+ case reflect.Uint8:
+ bldr := array.NewUint8Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]uint8), nil)
+ arr = bldr.NewArray()
+ case reflect.Int16:
+ bldr := array.NewInt16Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]int16), nil)
+ arr = bldr.NewArray()
+ case reflect.Uint16:
+ bldr := array.NewUint16Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]uint16), nil)
+ arr = bldr.NewArray()
+ case reflect.Int32:
+ bldr := array.NewInt32Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]int32), nil)
+ arr = bldr.NewArray()
+ case reflect.Uint32:
+ bldr := array.NewUint32Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]uint32), nil)
+ arr = bldr.NewArray()
+ case reflect.Int64:
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]int64), nil)
+ arr = bldr.NewArray()
+ case reflect.Uint64:
+ bldr := array.NewUint64Builder(mem)
+ defer bldr.Release()
+ bldr.AppendValues(sliceval.Interface().([]uint64), nil)
+ arr = bldr.NewArray()
+ case reflect.Int:
+ if bits.UintSize == 32 {
+ bldr := array.NewInt32Builder(mem)
+ defer bldr.Release()
+ for _, v := range sliceval.Interface().([]int) {
+ bldr.Append(int32(v))
+ }
+ arr = bldr.NewArray()
+ break
+ }
+ bldr := array.NewInt64Builder(mem)
+ defer bldr.Release()
+ for _, v := range sliceval.Interface().([]int) {
+ bldr.Append(int64(v))
+ }
+ arr = bldr.NewArray()
+ case reflect.Uint:
+ if bits.UintSize == 32 {
+ bldr := array.NewUint32Builder(mem)
+ defer bldr.Release()
+ for _, v := range sliceval.Interface().([]uint) {
+ bldr.Append(uint32(v))
+ }
+ arr = bldr.NewArray()
+ break
+ }
+ bldr := array.NewUint64Builder(mem)
+ defer bldr.Release()
+ for _, v := range sliceval.Interface().([]uint) {
+ bldr.Append(uint64(v))
+ }
+ arr = bldr.NewArray()
+ case reflect.Ptr:
+ meta, ok := sliceval.Interface().([]*arrow.Metadata)
+ if !ok {
+ break
+ }
+
+ bldr := array.NewMapBuilder(mem, arrow.BinaryTypes.Binary,
arrow.BinaryTypes.Binary, false)
+ defer bldr.Release()
+
+ kbldr := bldr.KeyBuilder().(*array.BinaryBuilder)
+ ibldr := bldr.ItemBuilder().(*array.BinaryBuilder)
+ for _, md := range meta {
+ bldr.Append(true)
+ if md != nil {
+ kbldr.AppendStringValues(md.Keys(), nil)
+ ibldr.AppendStringValues(md.Values(), nil)
+ }
+ }
+
+ arr := bldr.NewMapArray()
+ defer arr.Release()
+
+ return NewListScalar(arr), nil
+ }
+
+ if arr == nil {
+ return nil, xerrors.Errorf("createListScalar not implemented
for %s", sliceval.Type())
+ }
+
+ defer arr.Release()
+ return MakeScalarParam(arr, arrow.ListOf(arr.DataType()))
+}
+
+func fromListScalar(s ListScalar, v reflect.Value) error {
+ if v.Kind() != reflect.Slice {
+ return fmt.Errorf("could not populate field from list scalar,
incompatible types: %s is not a slice", v.Type().String())
+ }
+
+ arr := s.GetList()
+ v.Set(reflect.MakeSlice(v.Type(), arr.Len(), arr.Len()))
+ switch arr := arr.(type) {
+ case *array.Boolean:
+ for i := 0; i < arr.Len(); i++ {
+ v.Index(i).SetBool(arr.Value(i))
+ }
+ case *array.Int8:
+ reflect.Copy(v, reflect.ValueOf(arr.Int8Values()))
+ case *array.Uint8:
+ reflect.Copy(v, reflect.ValueOf(arr.Uint8Values()))
+ case *array.Int16:
+ reflect.Copy(v, reflect.ValueOf(arr.Int16Values()))
+ case *array.Uint16:
+ reflect.Copy(v, reflect.ValueOf(arr.Uint16Values()))
+ case *array.Int32:
+ reflect.Copy(v, reflect.ValueOf(arr.Int32Values()))
+ case *array.Uint32:
+ reflect.Copy(v, reflect.ValueOf(arr.Uint32Values()))
+ case *array.Int64:
+ reflect.Copy(v, reflect.ValueOf(arr.Int64Values()))
+ case *array.Uint64:
+ reflect.Copy(v, reflect.ValueOf(arr.Uint64Values()))
+ case *array.Float32:
+ reflect.Copy(v, reflect.ValueOf(arr.Float32Values()))
+ case *array.Float64:
+ reflect.Copy(v, reflect.ValueOf(arr.Float64Values()))
+ case *array.Binary:
+ for i := 0; i < arr.Len(); i++ {
+ v.Index(i).SetString(arr.ValueString(i))
+ }
+ case *array.String:
+ for i := 0; i < arr.Len(); i++ {
+ v.Index(i).SetString(arr.Value(i))
+ }
+ case *array.Map:
+ // only implementing slice of metadata for now
+ if v.Type().Elem() !=
reflect.PtrTo(reflect.TypeOf(arrow.Metadata{})) {
+ return fmt.Errorf("unimplemented fromListScalar type %s
to %s", arr.DataType(), v.Type().String())
+ }
+
+ var (
+ offsets = arr.Offsets()
+ keys = arr.Keys().(*array.Binary)
+ values = arr.Items().(*array.Binary)
+ metaKeys []string
+ metaValues []string
+ )
+
+ for i, o := range offsets[:len(offsets)-1] {
+ start := o
+ end := offsets[i+1]
+
+ metaKeys = make([]string, end-start)
+ metaValues = make([]string, end-start)
+ for j := start; j < end; j++ {
+ metaKeys = append(metaKeys,
keys.ValueString(int(j)))
+ metaValues = append(metaValues,
values.ValueString(int(j)))
+ }
+
+ m := arrow.NewMetadata(metaKeys, metaValues)
+ v.Index(i).Set(reflect.ValueOf(&m))
+ }
+
+ default:
+ return fmt.Errorf("unimplemented fromListScalar type: %s",
arr.DataType())
+ }
+
+ return nil
+}
+
// MakeScalarParam is for converting a value to a scalar when it requires a
// parameterized data type such as a time type that needs units, or a fixed
// size list which needs it's size.
@@ -154,6 +487,11 @@ func MakeScalar(val interface{}) Scalar {
return NewMonthDayNanoIntervalScalar(v)
case arrow.DataType:
return MakeNullScalar(v)
+ default:
+ testval := reflect.ValueOf(v)
+ if testval.Type().ConvertibleTo(reflect.TypeOf(uint32(0))) {
+ return
NewUint32Scalar(uint32(testval.Convert(reflect.TypeOf(uint32(0))).Uint()))
+ }
}
panic(xerrors.Errorf("makescalar not implemented for type value %#v",
val))
diff --git a/go/arrow/scalar/scalar.go b/go/arrow/scalar/scalar.go
index 145e5e1..6a35406 100644
--- a/go/arrow/scalar/scalar.go
+++ b/go/arrow/scalar/scalar.go
@@ -466,6 +466,10 @@ func init() {
// GetScalar creates a scalar object from the value at a given index in the
// passed in array, returns an error if unable to do so.
func GetScalar(arr array.Interface, idx int) (Scalar, error) {
+ if arr.IsNull(idx) {
+ return MakeNullScalar(arr.DataType()), nil
+ }
+
switch arr := arr.(type) {
case *array.Binary:
buf := memory.NewBufferBytes(arr.Value(idx))
@@ -561,12 +565,35 @@ func GetScalar(arr array.Interface, idx int) (Scalar,
error) {
// MakeArrayOfNull creates an array of size length which is all null of the
given data type.
func MakeArrayOfNull(dt arrow.DataType, length int, mem memory.Allocator)
array.Interface {
- nullBuf := memory.NewResizableBuffer(mem)
- nullBuf.Resize(int(bitutil.BytesForBits(int64(length))))
- defer nullBuf.Release()
- memory.Set(nullBuf.Bytes(), 0xFF)
+ var (
+ buffers = []*memory.Buffer{nil}
+ children []*array.Data
+ )
+
+ buffers[0] = memory.NewResizableBuffer(mem)
+ buffers[0].Resize(int(bitutil.BytesForBits(int64(length))))
+ defer buffers[0].Release()
+
+ switch t := dt.(type) {
+ case arrow.NestedType:
+ fieldList := t.Fields()
+ children = make([]*array.Data, len(fieldList))
+ for i, f := range fieldList {
+ arr := MakeArrayOfNull(f.Type, length, mem)
+ defer arr.Release()
+ children[i] = arr.Data()
+ }
+ case arrow.FixedWidthDataType:
+ buffers = append(buffers, memory.NewResizableBuffer(mem))
+
buffers[1].Resize(int(bitutil.BytesForBits(int64(t.BitWidth()))) * length)
+ defer buffers[1].Release()
+ case arrow.BinaryDataType:
+ buffers = append(buffers, memory.NewResizableBuffer(mem), nil)
+ buffers[1].Resize(arrow.Int32Traits.BytesRequired(length + 1))
+ defer buffers[1].Release()
+ }
- data := array.NewData(dt, length, []*memory.Buffer{nullBuf, nil}, nil,
length, 0)
+ data := array.NewData(dt, length, buffers, children, length, 0)
defer data.Release()
return array.MakeFromData(data)
}
@@ -617,7 +644,6 @@ func MakeArrayFromScalar(sc Scalar, length int, mem
memory.Allocator) (array.Int
c = 0xFF
}
memory.Set(data.Bytes(), c)
- defer data.Release()
return array.NewBoolean(length, data, nil, 0), nil
case BinaryScalar:
if s.DataType().ID() == arrow.FIXED_SIZE_BINARY {
diff --git a/go/arrow/scalar/scalar_test.go b/go/arrow/scalar/scalar_test.go
index 882c305..d983401 100644
--- a/go/arrow/scalar/scalar_test.go
+++ b/go/arrow/scalar/scalar_test.go
@@ -907,3 +907,74 @@ func TestMakeArrayFromScalar(t *testing.T) {
}
}
}
+
+type OptionListTest struct {
+ FieldNames []string `compute:"field_names"`
+ FieldNulls []bool `compute:"field_null"`
+ FieldMeta []*arrow.Metadata `compute:"field_metadata"`
+ Val8 []int8 `compute:"val8"`
+ ValU8 []uint8 `compute:"u8"`
+ Val16 []int16 `compute:"val16"`
+ ValU16 []uint16 `compute:"u16"`
+ Val32 []int32 `compute:"val32"`
+ ValU32 []uint32 `compute:"u32"`
+ Val64 []int64 `compute:"val64"`
+ ValU64 []uint64 `compute:"u64"`
+ ValInt []int `compute:"valint"`
+ ValUint []uint `compute:"valuint"`
+}
+
+type OptionValTest struct {
+ ToType arrow.DataType `compute:"type"`
+ Allow bool `compute:"allow"`
+}
+
+func (OptionValTest) TypeName() string { return "OptionValTest" }
+
+func TestToScalar(t *testing.T) {
+ ot := &OptionValTest{ToType: arrow.BinaryTypes.String, Allow: true}
+ sc, err := scalar.ToScalar(ot, memory.DefaultAllocator)
+ assert.NoError(t, err)
+ assert.Equal(t, `{type:utf8 = null, allow:bool = true,
_type_name:binary = OptionValTest}`, sc.String())
+
+ meta := arrow.MetadataFrom(map[string]string{
+ "option": "val",
+ "captain": "planet",
+ "souper": "bowl",
+ })
+
+ olt := OptionListTest{
+ FieldNames: []string{"foo", "bar", "baz"},
+ FieldNulls: []bool{true, false},
+ FieldMeta: []*arrow.Metadata{&meta, nil, &meta},
+ Val8: []int8{1, 2, 3, 4},
+ ValU8: []uint8{5, 6},
+ Val16: []int16{7, 8, 9, 10},
+ ValU16: []uint16{},
+ Val32: nil,
+ ValU32: []uint32{25, 26, 27, 28},
+ Val64: []int64{-1, -2, -3, -4, -5},
+ ValU64: []uint64{1, 2, 3},
+ ValInt: []int{10, 11, 12, 13},
+ ValUint: []uint{14, 15, 16},
+ }
+ sc, err = scalar.ToScalar(olt, memory.DefaultAllocator)
+ assert.NoError(t, err)
+
+ expected := `{field_names:list<item: utf8, nullable> = ["foo" "bar"
"baz"], ` +
+ `field_null:list<item: bool, nullable> = [true false], ` +
+ `field_metadata:list<item: map<binary, binary>, nullable> = ` +
+ `[{["captain" "option" "souper"] ["planet" "val" "bowl"]} {[]
[]} {["captain" "option" "souper"] ["planet" "val" "bowl"]}], ` +
+ `val8:list<item: int8, nullable> = [1 2 3 4], ` +
+ `u8:list<item: uint8, nullable> = [5 6], ` +
+ `val16:list<item: int16, nullable> = [7 8 9 10], ` +
+ `u16:list<item: uint16, nullable> = [], ` +
+ `val32:list<item: int32, nullable> = [], ` +
+ `u32:list<item: uint32, nullable> = [25 26 27 28], ` +
+ `val64:list<item: int64, nullable> = [-1 -2 -3 -4 -5], ` +
+ `u64:list<item: uint64, nullable> = [1 2 3], ` +
+ `valint:list<item: int64, nullable> = [10 11 12 13], ` +
+ `valuint:list<item: uint64, nullable> = [14 15 16]}`
+
+ assert.Equal(t, expected, sc.String())
+}
diff --git a/go/go.mod b/go/go.mod
index 6bd4237..4aab7c7 100644
--- a/go/go.mod
+++ b/go/go.mod
@@ -21,12 +21,12 @@ go 1.15
require (
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c
github.com/andybalholm/brotli v1.0.3
- github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40
github.com/apache/thrift v0.15.0
github.com/goccy/go-json v0.7.10
github.com/golang/protobuf v1.5.2
github.com/golang/snappy v0.0.4
github.com/google/flatbuffers v2.0.0+incompatible
+ github.com/google/go-cmp v0.5.6 // indirect
github.com/klauspost/asmfmt v1.3.1
github.com/klauspost/compress v1.13.6
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8
@@ -35,10 +35,12 @@ require (
github.com/stretchr/testify v1.7.0
github.com/zeebo/xxh3 v0.13.0
golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a
+ golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
golang.org/x/tools v0.1.4
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
gonum.org/v1/gonum v0.9.3
+ google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79 //
indirect
google.golang.org/grpc v1.41.0
google.golang.org/protobuf v1.27.1
)
diff --git a/go/go.sum b/go/go.sum
index 6c1e79c..cfc84b9 100644
--- a/go/go.sum
+++ b/go/go.sum
@@ -20,8 +20,6 @@ github.com/alecthomas/units
v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/andybalholm/brotli v1.0.3
h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
github.com/andybalholm/brotli v1.0.3/go.mod
h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod
h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40
h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ=
-github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod
h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs=
github.com/apache/thrift v0.12.0/go.mod
h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod
h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.15.0
h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
@@ -46,7 +44,6 @@ github.com/clbanning/x2j
v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4
github.com/client9/misspell v0.3.4/go.mod
h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod
h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod
h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod
h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod
h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod
h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod
h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -71,7 +68,6 @@ github.com/envoyproxy/go-control-plane
v0.9.1-0.20191026205805-5f8ba28d4473/go.m
github.com/envoyproxy/go-control-plane v0.9.4/go.mod
h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane
v0.9.9-0.20201210154907-fd9021fe5dad/go.mod
h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane
v0.9.9-0.20210217033140-668b12f5399d/go.mod
h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane
v0.9.9-0.20210512163311-63b5d3c536b0/go.mod
h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane
v0.9.10-0.20210907150352-cf90f659a021/go.mod
h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod
h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod
h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -125,7 +121,6 @@ github.com/golang/protobuf v1.5.0/go.mod
h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/golang/protobuf v1.5.2
h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod
h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod
h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.3/go.mod
h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod
h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod
h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -190,7 +185,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod
h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
github.com/kisielk/gotool v1.0.0/go.mod
h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/asmfmt v1.3.1
h1:7xZi1N7s9gTLbqiM8KUv8TLyysavbTRGBT5/ly0bRtw=
github.com/klauspost/asmfmt v1.3.1/go.mod
h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
-github.com/klauspost/compress v1.13.1/go.mod
h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6
h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod
h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod
h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -256,7 +250,6 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod
h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod
h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible
h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod
h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4/v4 v4.1.8/go.mod
h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.9
h1:xkrjwpOP5xg1k4Nn4GX4a4YFGhscyQL/3EddJ1Xxqm8=
github.com/pierrec/lz4/v4 v4.1.9/go.mod
h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.0/go.mod
h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -446,7 +439,6 @@ golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod
h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod
h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod
h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod
h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod
h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -521,7 +513,6 @@ google.golang.org/grpc v1.27.0/go.mod
h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.33.1/go.mod
h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod
h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.38.0/go.mod
h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.39.0/go.mod
h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E=
google.golang.org/grpc v1.41.0/go.mod
h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod
h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
