This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new fb1040a0 feat(arrow/array): add ValueAsAny for native Go values (#1170)
fb1040a0 is described below
commit fb1040a01a754596623ddb724d442905b4bfbb7b
Author: Digvijay <[email protected]>
AuthorDate: Fri Aug 14 11:13:27 2026 -0500
feat(arrow/array): add ValueAsAny for native Go values (#1170)
### Rationale for this change
Downstream code often needs type-agnostic access to Arrow values as
native Go types. `GetOneForMarshal` is JSON-oriented (e.g. `int8` →
`float64`, timestamps/decimals as strings), which forces boilerplate
type switches for generic ingestion.
Fixes #462
### What changes are included in this PR?
- Add `array.ValueAsAny(arr, i)` plus an optional `ValueAsAnyer`
interface, without adding a method to `arrow.Array`
- Implement it across array types and common extensions so nulls return
`nil` and values stay native (`int8`, `arrow.Timestamp`, decimal nums,
`[]any` for lists, ordered `[name, value]` pairs for structs)
- Union slots keep their type id when the selected child is null
(`[]any{typeID, nil}`)
- Variant decode errors are returned as the `any` value rather than
collapsed to `nil`
- Add unit tests covering primitives, temporal/decimal, nested,
dictionary, union, duplicate struct fields, and variant decode errors,
including differences vs `GetOneForMarshal`
### Are these changes tested?
- `go test ./arrow/ ./arrow/array/ ./arrow/extensions/`
### Are there any user-facing changes?
Yes — new `array.ValueAsAny` helper for native Go value access.
---------
Signed-off-by: Digvijay <[email protected]>
Co-authored-by: Digvijay Waghela
<[email protected]>
---
arrow/array/binary.go | 21 ++
arrow/array/boolean.go | 7 +
arrow/array/decimal.go | 7 +
arrow/array/dictionary.go | 7 +
arrow/array/encoded.go | 7 +
arrow/array/extension.go | 4 +
arrow/array/fixed_size_list.go | 4 +
arrow/array/fixedsize_binary.go | 7 +
arrow/array/float16.go | 7 +
arrow/array/interval.go | 21 ++
arrow/array/list.go | 16 ++
arrow/array/null.go | 4 +
arrow/array/numeric_generic.go | 48 +++++
arrow/array/string.go | 21 ++
arrow/array/struct.go | 12 ++
arrow/array/timestamp.go | 7 +
arrow/array/union.go | 15 ++
arrow/array/value_as_any.go | 60 ++++++
arrow/array/value_as_any_test.go | 333 ++++++++++++++++++++++++++++++
arrow/extensions/bool8.go | 7 +
arrow/extensions/json.go | 7 +
arrow/extensions/timestamp_with_offset.go | 7 +
arrow/extensions/uuid.go | 7 +
arrow/extensions/variant.go | 11 +
arrow/extensions/variant_test.go | 31 +++
25 files changed, 678 insertions(+)
diff --git a/arrow/array/binary.go b/arrow/array/binary.go
index d23e2a0b..fe986583 100644
--- a/arrow/array/binary.go
+++ b/arrow/array/binary.go
@@ -159,6 +159,13 @@ func (a *Binary) GetOneForMarshal(i int) interface{} {
return a.Value(i)
}
+func (a *Binary) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *Binary) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
@@ -353,6 +360,13 @@ func (a *LargeBinary) GetOneForMarshal(i int) interface{} {
return a.Value(i)
}
+func (a *LargeBinary) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *LargeBinary) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
@@ -540,6 +554,13 @@ func (a *BinaryView) GetOneForMarshal(i int) interface{} {
return a.Value(i)
}
+func (a *BinaryView) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *BinaryView) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
diff --git a/arrow/array/boolean.go b/arrow/array/boolean.go
index d579fa0c..80957f19 100644
--- a/arrow/array/boolean.go
+++ b/arrow/array/boolean.go
@@ -97,6 +97,13 @@ func (a *Boolean) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *Boolean) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *Boolean) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
diff --git a/arrow/array/decimal.go b/arrow/array/decimal.go
index 704b1d93..aac075a2 100644
--- a/arrow/array/decimal.go
+++ b/arrow/array/decimal.go
@@ -99,6 +99,13 @@ func (a *baseDecimal[T]) GetOneForMarshal(i int) any {
return n.ToBigFloat(scale).Text('g', int(typ.GetPrecision()))
}
+func (a *baseDecimal[T]) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *baseDecimal[T]) MarshalJSON() ([]byte, error) {
vals := make([]any, a.Len())
for i := 0; i < a.Len(); i++ {
diff --git a/arrow/array/dictionary.go b/arrow/array/dictionary.go
index d151b209..14c234d2 100644
--- a/arrow/array/dictionary.go
+++ b/arrow/array/dictionary.go
@@ -295,6 +295,13 @@ func (d *Dictionary) GetOneForMarshal(i int) interface{} {
return d.Dictionary().GetOneForMarshal(vidx)
}
+func (d *Dictionary) ValueAsAny(i int) any {
+ if d.IsNull(i) {
+ return nil
+ }
+ return ValueAsAny(d.Dictionary(), d.GetValueIndex(i))
+}
+
func (d *Dictionary) MarshalJSON() ([]byte, error) {
vals := make([]any, d.Len())
for i := range d.Len() {
diff --git a/arrow/array/encoded.go b/arrow/array/encoded.go
index 05d5bda5..219a75cb 100644
--- a/arrow/array/encoded.go
+++ b/arrow/array/encoded.go
@@ -332,6 +332,13 @@ func (r *RunEndEncoded) GetOneForMarshal(i int)
interface{} {
return r.values.GetOneForMarshal(r.GetPhysicalIndex(i))
}
+func (r *RunEndEncoded) ValueAsAny(i int) any {
+ if r.IsNull(i) {
+ return nil
+ }
+ return ValueAsAny(r.values, r.GetPhysicalIndex(i))
+}
+
func (r *RunEndEncoded) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
diff --git a/arrow/array/extension.go b/arrow/array/extension.go
index 5a4d3c86..c82db8e0 100644
--- a/arrow/array/extension.go
+++ b/arrow/array/extension.go
@@ -124,6 +124,10 @@ func (e *ExtensionArrayBase) GetOneForMarshal(i int)
interface{} {
return e.storage.GetOneForMarshal(i)
}
+func (e *ExtensionArrayBase) ValueAsAny(i int) any {
+ return ValueAsAny(e.storage, i)
+}
+
func (e *ExtensionArrayBase) MarshalJSON() ([]byte, error) {
return json.Marshal(e.storage)
}
diff --git a/arrow/array/fixed_size_list.go b/arrow/array/fixed_size_list.go
index 6a4f75ca..f13797cc 100644
--- a/arrow/array/fixed_size_list.go
+++ b/arrow/array/fixed_size_list.go
@@ -140,6 +140,10 @@ func (a *FixedSizeList) GetOneForMarshal(i int)
interface{} {
return json.RawMessage(v)
}
+func (a *FixedSizeList) ValueAsAny(i int) any {
+ return valueAsAnyFromListLike(a, i)
+}
+
func (a *FixedSizeList) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
diff --git a/arrow/array/fixedsize_binary.go b/arrow/array/fixedsize_binary.go
index 8530ed6b..567bcba0 100644
--- a/arrow/array/fixedsize_binary.go
+++ b/arrow/array/fixedsize_binary.go
@@ -97,6 +97,13 @@ func (a *FixedSizeBinary) GetOneForMarshal(i int)
interface{} {
return a.Value(i)
}
+func (a *FixedSizeBinary) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *FixedSizeBinary) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
diff --git a/arrow/array/float16.go b/arrow/array/float16.go
index 41276803..02802c7d 100644
--- a/arrow/array/float16.go
+++ b/arrow/array/float16.go
@@ -84,6 +84,13 @@ func (a *Float16) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *Float16) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *Float16) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i, v := range a.values {
diff --git a/arrow/array/interval.go b/arrow/array/interval.go
index 2c029c25..5053c3a2 100644
--- a/arrow/array/interval.go
+++ b/arrow/array/interval.go
@@ -101,6 +101,13 @@ func (a *MonthInterval) GetOneForMarshal(i int)
interface{} {
return nil
}
+func (a *MonthInterval) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
// MarshalJSON will create a json array out of a MonthInterval array,
// each value will be an object of the form {"months": #} where
// # is the numeric value of that index
@@ -406,6 +413,13 @@ func (a *DayTimeInterval) GetOneForMarshal(i int)
interface{} {
return nil
}
+func (a *DayTimeInterval) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
// MarshalJSON will marshal this array to JSON as an array of objects,
// consisting of the form {"days": #, "milliseconds": #} for each element.
func (a *DayTimeInterval) MarshalJSON() ([]byte, error) {
@@ -710,6 +724,13 @@ func (a *MonthDayNanoInterval) GetOneForMarshal(i int)
interface{} {
return nil
}
+func (a *MonthDayNanoInterval) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
// MarshalJSON will marshal this array to a JSON array with elements
// marshalled to the form {"months": #, "days": #, "nanoseconds": #}
func (a *MonthDayNanoInterval) MarshalJSON() ([]byte, error) {
diff --git a/arrow/array/list.go b/arrow/array/list.go
index b848192b..dd9d82ef 100644
--- a/arrow/array/list.go
+++ b/arrow/array/list.go
@@ -112,6 +112,10 @@ func (a *List) GetOneForMarshal(i int) interface{} {
return json.RawMessage(v)
}
+func (a *List) ValueAsAny(i int) any {
+ return valueAsAnyFromListLike(a, i)
+}
+
func (a *List) Validate() error { return validateListArray(a, false) }
func (a *List) ValidateFull() error { return validateListArray(a, true) }
@@ -247,6 +251,10 @@ func (a *LargeList) GetOneForMarshal(i int) interface{} {
return json.RawMessage(v)
}
+func (a *LargeList) ValueAsAny(i int) any {
+ return valueAsAnyFromListLike(a, i)
+}
+
func (a *LargeList) Validate() error { return validateLargeListArray(a,
false) }
func (a *LargeList) ValidateFull() error { return validateLargeListArray(a,
true) }
@@ -732,6 +740,10 @@ func (a *ListView) GetOneForMarshal(i int) interface{} {
return json.RawMessage(v)
}
+func (a *ListView) ValueAsAny(i int) any {
+ return valueAsAnyFromListLike(a, i)
+}
+
func (a *ListView) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
@@ -879,6 +891,10 @@ func (a *LargeListView) GetOneForMarshal(i int)
interface{} {
return json.RawMessage(v)
}
+func (a *LargeListView) ValueAsAny(i int) any {
+ return valueAsAnyFromListLike(a, i)
+}
+
func (a *LargeListView) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
diff --git a/arrow/array/null.go b/arrow/array/null.go
index 8f8f5805..3f7db087 100644
--- a/arrow/array/null.go
+++ b/arrow/array/null.go
@@ -84,6 +84,10 @@ func (a *Null) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *Null) ValueAsAny(i int) any {
+ return nil
+}
+
func (a *Null) MarshalJSON() ([]byte, error) {
return json.Marshal(make([]interface{}, a.Len()))
}
diff --git a/arrow/array/numeric_generic.go b/arrow/array/numeric_generic.go
index 24edd549..2ff88427 100644
--- a/arrow/array/numeric_generic.go
+++ b/arrow/array/numeric_generic.go
@@ -91,6 +91,14 @@ func (a *numericArray[T]) GetOneForMarshal(i int) any {
return a.values[i]
}
+func (a *numericArray[T]) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
func (a *numericArray[T]) MarshalJSON() ([]byte, error) {
vals := make([]any, a.Len())
for i := range a.Len() {
@@ -115,6 +123,14 @@ func (a *oneByteArrs[T]) GetOneForMarshal(i int) any {
return float64(a.values[i]) // prevent uint8/int8 from being seen as
binary data
}
+func (a *oneByteArrs[T]) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
func (a *oneByteArrs[T]) MarshalJSON() ([]byte, error) {
vals := make([]any, a.Len())
for i := range a.Len() {
@@ -157,6 +173,14 @@ func (a *floatArray[T]) GetOneForMarshal(i int) any {
}
}
+func (a *floatArray[T]) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.Value(i)
+}
+
func (a *floatArray[T]) MarshalJSON() ([]byte, error) {
vals := make([]any, a.Len())
for i := range a.values {
@@ -210,6 +234,14 @@ func (d *dateArray[T]) GetOneForMarshal(i int) interface{}
{
return d.values[i].FormattedString()
}
+func (d *dateArray[T]) ValueAsAny(i int) any {
+ if d.IsNull(i) {
+ return nil
+ }
+
+ return d.values[i]
+}
+
type timeType interface {
TimeUnit() arrow.TimeUnit
}
@@ -246,6 +278,14 @@ func (a *timeArray[T]) GetOneForMarshal(i int) interface{}
{
return
a.values[i].ToTime(a.DataType().(timeType).TimeUnit()).Format("15:04:05.999999999")
}
+func (a *timeArray[T]) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
type Duration struct {
numericArray[arrow.Duration]
}
@@ -281,6 +321,14 @@ func (a *Duration) GetOneForMarshal(i int) any {
return fmt.Sprintf("%d%s", a.values[i],
a.DataType().(timeType).TimeUnit())
}
+func (a *Duration) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
type Int64 struct {
numericArray[int64]
}
diff --git a/arrow/array/string.go b/arrow/array/string.go
index 519b2cae..afd78278 100644
--- a/arrow/array/string.go
+++ b/arrow/array/string.go
@@ -161,6 +161,13 @@ func (a *String) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *String) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *String) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
@@ -369,6 +376,13 @@ func (a *LargeString) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *LargeString) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *LargeString) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
@@ -549,6 +563,13 @@ func (a *StringView) GetOneForMarshal(i int) interface{} {
return a.Value(i)
}
+func (a *StringView) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *StringView) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
diff --git a/arrow/array/struct.go b/arrow/array/struct.go
index 7a3622bb..49bcc570 100644
--- a/arrow/array/struct.go
+++ b/arrow/array/struct.go
@@ -280,6 +280,18 @@ func (a *Struct) GetOneForMarshal(i int) interface{} {
return tmp
}
+func (a *Struct) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ fieldList := a.data.dtype.(*arrow.StructType).Fields()
+ out := make([]any, len(a.fields))
+ for j, d := range a.fields {
+ out[j] = []any{fieldList[j].Name, ValueAsAny(d, i)}
+ }
+ return out
+}
+
func (a *Struct) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
diff --git a/arrow/array/timestamp.go b/arrow/array/timestamp.go
index cf17b31f..523c3a01 100644
--- a/arrow/array/timestamp.go
+++ b/arrow/array/timestamp.go
@@ -116,6 +116,13 @@ func (a *Timestamp) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *Timestamp) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
func (a *Timestamp) MarshalJSON() ([]byte, error) {
vals := make([]interface{}, a.Len())
for i := range a.values {
diff --git a/arrow/array/union.go b/arrow/array/union.go
index ed894ab1..afa4e84c 100644
--- a/arrow/array/union.go
+++ b/arrow/array/union.go
@@ -333,6 +333,13 @@ func (a *SparseUnion) GetOneForMarshal(i int) interface{} {
return []interface{}{typeID, data.GetOneForMarshal(i)}
}
+func (a *SparseUnion) ValueAsAny(i int) any {
+ typeID := a.RawTypeCodes()[i]
+ childID := a.ChildID(i)
+ data := a.Field(childID)
+ return []any{typeID, ValueAsAny(data, i)}
+}
+
func (a *SparseUnion) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
@@ -627,6 +634,14 @@ func (a *DenseUnion) GetOneForMarshal(i int) interface{} {
return []interface{}{typeID, data.GetOneForMarshal(offset)}
}
+func (a *DenseUnion) ValueAsAny(i int) any {
+ typeID := a.RawTypeCodes()[i]
+ childID := a.ChildID(i)
+ data := a.Field(childID)
+ offset := int(a.RawValueOffsets()[i])
+ return []any{typeID, ValueAsAny(data, offset)}
+}
+
func (a *DenseUnion) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
diff --git a/arrow/array/value_as_any.go b/arrow/array/value_as_any.go
new file mode 100644
index 00000000..2a5f46fe
--- /dev/null
+++ b/arrow/array/value_as_any.go
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "fmt"
+
+ "github.com/apache/arrow-go/v18/arrow"
+)
+
+// ValueAsAnyer is an optional interface for arrays that can return a native
+// Go value for a slot. Built-in array types implement it. It is not part of
+// arrow.Array so that adding this capability is not a source-incompatible
+// change for downstream Array implementations.
+type ValueAsAnyer interface {
+ ValueAsAny(i int) any
+}
+
+// ValueAsAny returns the native Go value at index i, or nil if the slot is
null.
+// Unlike GetOneForMarshal, values are not converted for JSON encoding
+// (for example int8 stays int8, timestamps stay arrow.Timestamp, lists are
+// []any of native values, and structs are []any of [name, value] pairs so
+// field order and duplicate names are preserved).
+//
+// If arr does not implement ValueAsAnyer, ValueAsAny panics.
+func ValueAsAny(arr arrow.Array, i int) any {
+ if v, ok := arr.(ValueAsAnyer); ok {
+ return v.ValueAsAny(i)
+ }
+ panic(fmt.Sprintf("arrow/array: %T does not implement ValueAsAny", arr))
+}
+
+// valueAsAnyFromListLike builds a []any of native child values for one list
slot.
+func valueAsAnyFromListLike(a ListLike, i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ start, end := a.ValueOffsets(i)
+ vals := a.ListValues()
+ out := make([]any, end-start)
+ for j := start; j < end; j++ {
+ out[j-start] = ValueAsAny(vals, int(j))
+ }
+ return out
+}
diff --git a/arrow/array/value_as_any_test.go b/arrow/array/value_as_any_test.go
new file mode 100644
index 00000000..ae173896
--- /dev/null
+++ b/arrow/array/value_as_any_test.go
@@ -0,0 +1,333 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "math"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/decimal128"
+ "github.com/apache/arrow-go/v18/arrow/float16"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestValueAsAnyPrimitives(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ t.Run("int8 native vs marshal", func(t *testing.T) {
+ b := array.NewInt8Builder(mem)
+ defer b.Release()
+ b.AppendValues([]int8{1, -2}, []bool{true, true})
+ b.AppendNull()
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, int8(1), array.ValueAsAny(arr, 0))
+ assert.Equal(t, int8(-2), array.ValueAsAny(arr, 1))
+ assert.Nil(t, array.ValueAsAny(arr, 2))
+
+ // GetOneForMarshal widens int8 to float64 for JSON safety.
+ assert.Equal(t, float64(1), arr.GetOneForMarshal(0))
+ assert.NotEqual(t, arr.GetOneForMarshal(0),
array.ValueAsAny(arr, 0))
+ })
+
+ t.Run("uint8 native vs marshal", func(t *testing.T) {
+ b := array.NewUint8Builder(mem)
+ defer b.Release()
+ b.Append(255)
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, uint8(255), array.ValueAsAny(arr, 0))
+ assert.Equal(t, float64(255), arr.GetOneForMarshal(0))
+ })
+
+ t.Run("int64", func(t *testing.T) {
+ b := array.NewInt64Builder(mem)
+ defer b.Release()
+ b.AppendValues([]int64{42, 0}, []bool{true, false})
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, int64(42), array.ValueAsAny(arr, 0))
+ assert.Nil(t, array.ValueAsAny(arr, 1))
+ })
+
+ t.Run("boolean", func(t *testing.T) {
+ b := array.NewBooleanBuilder(mem)
+ defer b.Release()
+ b.AppendValues([]bool{true, false}, []bool{true, false})
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, true, array.ValueAsAny(arr, 0))
+ assert.Nil(t, array.ValueAsAny(arr, 1))
+ })
+
+ t.Run("float64 keeps NaN", func(t *testing.T) {
+ b := array.NewFloat64Builder(mem)
+ defer b.Release()
+ b.Append(math.NaN())
+ b.Append(math.Inf(1))
+ arr := b.NewArray()
+ defer arr.Release()
+
+ got := array.ValueAsAny(arr, 0).(float64)
+ assert.True(t, math.IsNaN(got))
+ assert.Equal(t, math.Inf(1), array.ValueAsAny(arr, 1))
+ assert.Equal(t, "NaN", arr.GetOneForMarshal(0))
+ assert.Equal(t, "+Inf", arr.GetOneForMarshal(1))
+ })
+
+ t.Run("string and binary", func(t *testing.T) {
+ sb := array.NewStringBuilder(mem)
+ defer sb.Release()
+ sb.Append("hello")
+ sb.AppendNull()
+ sarr := sb.NewArray()
+ defer sarr.Release()
+ assert.Equal(t, "hello", array.ValueAsAny(sarr, 0))
+ assert.Nil(t, array.ValueAsAny(sarr, 1))
+
+ bb := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
+ defer bb.Release()
+ bb.Append([]byte{0x01, 0x02})
+ barr := bb.NewArray()
+ defer barr.Release()
+ assert.Equal(t, []byte{0x01, 0x02}, array.ValueAsAny(barr, 0))
+ })
+
+ t.Run("float16", func(t *testing.T) {
+ b := array.NewFloat16Builder(mem)
+ defer b.Release()
+ b.Append(float16.New(1.5))
+ arr := b.NewArray()
+ defer arr.Release()
+
+ got, ok := array.ValueAsAny(arr, 0).(float16.Num)
+ require.True(t, ok)
+ assert.Equal(t, float32(1.5), got.Float32())
+ assert.Equal(t, float32(1.5), arr.GetOneForMarshal(0))
+ })
+}
+
+func TestValueAsAnyTemporalAndDecimal(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ t.Run("timestamp", func(t *testing.T) {
+ dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: "UTC"}
+ b := array.NewTimestampBuilder(mem, dt)
+ defer b.Release()
+ b.Append(arrow.Timestamp(1_700_000_000))
+ b.AppendNull()
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, arrow.Timestamp(1_700_000_000),
array.ValueAsAny(arr, 0))
+ assert.Nil(t, array.ValueAsAny(arr, 1))
+ _, isString := arr.GetOneForMarshal(0).(string)
+ assert.True(t, isString)
+ })
+
+ t.Run("date32", func(t *testing.T) {
+ b := array.NewDate32Builder(mem)
+ defer b.Release()
+ b.Append(arrow.Date32(10))
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, arrow.Date32(10), array.ValueAsAny(arr, 0))
+ _, isString := arr.GetOneForMarshal(0).(string)
+ assert.True(t, isString)
+ })
+
+ t.Run("duration", func(t *testing.T) {
+ b := array.NewDurationBuilder(mem, &arrow.DurationType{Unit:
arrow.Millisecond})
+ defer b.Release()
+ b.Append(arrow.Duration(250))
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, arrow.Duration(250), array.ValueAsAny(arr, 0))
+ assert.Equal(t, "250ms", arr.GetOneForMarshal(0))
+ })
+
+ t.Run("decimal128", func(t *testing.T) {
+ dtype := &arrow.Decimal128Type{Precision: 10, Scale: 2}
+ b := array.NewDecimal128Builder(mem, dtype)
+ defer b.Release()
+ n, err := decimal128.FromString("12.34", dtype.Precision,
dtype.Scale)
+ require.NoError(t, err)
+ b.Append(n)
+ b.AppendNull()
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, n, array.ValueAsAny(arr, 0))
+ assert.Nil(t, array.ValueAsAny(arr, 1))
+ _, isString := arr.GetOneForMarshal(0).(string)
+ assert.True(t, isString)
+ })
+}
+
+func TestValueAsAnyNested(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ t.Run("list", func(t *testing.T) {
+ b := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int8)
+ defer b.Release()
+ vb := b.ValueBuilder().(*array.Int8Builder)
+
+ b.Append(true)
+ vb.AppendValues([]int8{1, 2}, nil)
+ b.AppendNull()
+ b.Append(true)
+ vb.Append(3)
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, []any{int8(1), int8(2)}, array.ValueAsAny(arr,
0))
+ assert.Nil(t, array.ValueAsAny(arr, 1))
+ assert.Equal(t, []any{int8(3)}, array.ValueAsAny(arr, 2))
+ })
+
+ t.Run("struct", func(t *testing.T) {
+ fields := []arrow.Field{
+ {Name: "n", Type: arrow.PrimitiveTypes.Int8, Nullable:
true},
+ {Name: "s", Type: arrow.BinaryTypes.String, Nullable:
true},
+ }
+ b := array.NewStructBuilder(mem, arrow.StructOf(fields...))
+ defer b.Release()
+ nb := b.FieldBuilder(0).(*array.Int8Builder)
+ sb := b.FieldBuilder(1).(*array.StringBuilder)
+
+ b.Append(true)
+ nb.Append(7)
+ sb.Append("x")
+ b.AppendNull()
+ nb.AppendNull()
+ sb.AppendNull()
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, []any{[]any{"n", int8(7)}, []any{"s", "x"}},
array.ValueAsAny(arr, 0))
+ assert.Nil(t, array.ValueAsAny(arr, 1))
+ })
+
+ t.Run("struct duplicate field names", func(t *testing.T) {
+ fields := []arrow.Field{
+ {Name: "dup", Type: arrow.PrimitiveTypes.Int32,
Nullable: true},
+ {Name: "dup", Type: arrow.PrimitiveTypes.Int64,
Nullable: true},
+ }
+ b := array.NewStructBuilder(mem, arrow.StructOf(fields...))
+ defer b.Release()
+ i32 := b.FieldBuilder(0).(*array.Int32Builder)
+ i64 := b.FieldBuilder(1).(*array.Int64Builder)
+
+ b.Append(true)
+ i32.Append(11)
+ i64.Append(22)
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, []any{[]any{"dup", int32(11)}, []any{"dup",
int64(22)}}, array.ValueAsAny(arr, 0))
+ })
+
+ t.Run("null array", func(t *testing.T) {
+ arr := array.NewNull(3)
+ defer arr.Release()
+ assert.Nil(t, array.ValueAsAny(arr, 0))
+ assert.Nil(t, array.ValueAsAny(arr, 2))
+ })
+
+ t.Run("dictionary", func(t *testing.T) {
+ b := array.NewDictionaryBuilder(mem, &arrow.DictionaryType{
+ IndexType: arrow.PrimitiveTypes.Int8,
+ ValueType: arrow.BinaryTypes.String,
+ })
+ defer b.Release()
+ db := b.(*array.BinaryDictionaryBuilder)
+ require.NoError(t, db.Append([]byte("a")))
+ require.NoError(t, db.Append([]byte("b")))
+ db.AppendNull()
+ require.NoError(t, db.Append([]byte("a")))
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, "a", array.ValueAsAny(arr, 0))
+ assert.Equal(t, "b", array.ValueAsAny(arr, 1))
+ assert.Nil(t, array.ValueAsAny(arr, 2))
+ assert.Equal(t, "a", array.ValueAsAny(arr, 3))
+ })
+}
+
+func TestValueAsAnyUnionKeepsTypeIDWhenChildNull(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ fields := []arrow.Field{
+ {Name: "i8", Type: arrow.PrimitiveTypes.Int8, Nullable: true},
+ {Name: "str", Type: arrow.BinaryTypes.String, Nullable: true},
+ }
+ codes := []arrow.UnionTypeCode{0, 1}
+
+ t.Run("sparse", func(t *testing.T) {
+ b := array.NewSparseUnionBuilder(mem,
arrow.SparseUnionOf(fields, codes))
+ defer b.Release()
+ i8b := b.Child(0).(*array.Int8Builder)
+ strb := b.Child(1).(*array.StringBuilder)
+
+ b.Append(0)
+ i8b.Append(5)
+ strb.AppendEmptyValue()
+
+ b.AppendNull()
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, []any{arrow.UnionTypeCode(0), int8(5)},
array.ValueAsAny(arr, 0))
+ assert.Equal(t, []any{arrow.UnionTypeCode(0), nil},
array.ValueAsAny(arr, 1))
+ })
+
+ t.Run("dense", func(t *testing.T) {
+ b := array.NewDenseUnionBuilder(mem, arrow.DenseUnionOf(fields,
codes))
+ defer b.Release()
+ i8b := b.Child(0).(*array.Int8Builder)
+
+ b.Append(0)
+ i8b.Append(5)
+
+ b.AppendNull()
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, []any{arrow.UnionTypeCode(0), int8(5)},
array.ValueAsAny(arr, 0))
+ assert.Equal(t, []any{arrow.UnionTypeCode(0), nil},
array.ValueAsAny(arr, 1))
+ })
+}
diff --git a/arrow/extensions/bool8.go b/arrow/extensions/bool8.go
index 97038a1b..43171983 100644
--- a/arrow/extensions/bool8.go
+++ b/arrow/extensions/bool8.go
@@ -121,6 +121,13 @@ func (a *Bool8Array) GetOneForMarshal(i int) interface{} {
return a.Value(i)
}
+func (a *Bool8Array) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
// boolToInt8 performs the simple scalar conversion of bool to the canonical
int8
// value for the Bool8Type.
func boolToInt8(v bool) int8 {
diff --git a/arrow/extensions/json.go b/arrow/extensions/json.go
index e74ba7f6..b9b1bf3b 100644
--- a/arrow/extensions/json.go
+++ b/arrow/extensions/json.go
@@ -148,6 +148,13 @@ func (a *JSONArray) GetOneForMarshal(i int) interface{} {
return a.ValueJSON(i)
}
+func (a *JSONArray) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
var (
_ arrow.ExtensionType = (*JSONType)(nil)
_ array.ExtensionArray = (*JSONArray)(nil)
diff --git a/arrow/extensions/timestamp_with_offset.go
b/arrow/extensions/timestamp_with_offset.go
index 5cc17ed8..257707a0 100644
--- a/arrow/extensions/timestamp_with_offset.go
+++ b/arrow/extensions/timestamp_with_offset.go
@@ -395,6 +395,13 @@ func (a *TimestampWithOffsetArray) GetOneForMarshal(i int)
interface{} {
return a.Value(i)
}
+func (a *TimestampWithOffsetArray) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
// noLastOffset is the sentinel value for TimestampWithOffsetBuilder.lastOffset
// indicating that no run-end-encoded run has been started yet. It is
deliberately
// outside the range of valid timezone offsets in minutes (roughly [-720,
840]) so
diff --git a/arrow/extensions/uuid.go b/arrow/extensions/uuid.go
index 9aac0225..e03f2214 100644
--- a/arrow/extensions/uuid.go
+++ b/arrow/extensions/uuid.go
@@ -202,6 +202,13 @@ func (a *UUIDArray) GetOneForMarshal(i int) interface{} {
return nil
}
+func (a *UUIDArray) ValueAsAny(i int) any {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
// UUIDType is a simple extension type that represents a FixedSizeBinary(16)
// to be used for representing UUIDs
type UUIDType struct {
diff --git a/arrow/extensions/variant.go b/arrow/extensions/variant.go
index 379822c4..fee2e046 100644
--- a/arrow/extensions/variant.go
+++ b/arrow/extensions/variant.go
@@ -612,6 +612,17 @@ func (v *VariantArray) GetOneForMarshal(i int) any {
return val.Value()
}
+func (v *VariantArray) ValueAsAny(i int) any {
+ if v.IsNull(i) {
+ return nil
+ }
+ val, err := v.Value(i)
+ if err != nil {
+ return err
+ }
+ return val.Value()
+}
+
type variantReader interface {
IsNull(i int) bool
Value(i int) (variant.Value, error)
diff --git a/arrow/extensions/variant_test.go b/arrow/extensions/variant_test.go
index 3580123d..a39fd513 100644
--- a/arrow/extensions/variant_test.go
+++ b/arrow/extensions/variant_test.go
@@ -1772,3 +1772,34 @@ func TestUnshredVariant(t *testing.T) {
assert.False(t, out.IsShredded())
})
}
+
+func TestVariantValueAsAnyInvalidMetadata(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ vt := extensions.NewDefaultVariantType()
+ bldr := array.NewStructBuilder(mem,
vt.StorageType().(*arrow.StructType))
+ defer bldr.Release()
+ metaBldr := bldr.FieldBuilder(0).(*array.BinaryBuilder)
+ valueBldr := bldr.FieldBuilder(1).(*array.BinaryBuilder)
+
+ bldr.Append(true)
+ metaBldr.Append([]byte{0x01}) // too short to be valid variant metadata
+ vbytes, err := variant.Encode(int8(1))
+ require.NoError(t, err)
+ valueBldr.Append(vbytes)
+
+ storage := bldr.NewArray()
+ defer storage.Release()
+
+ variantArr := array.NewExtensionArrayWithStorage(vt, storage)
+ defer variantArr.Release()
+ varr := variantArr.(*extensions.VariantArray)
+
+ require.False(t, varr.IsNull(0))
+ got := varr.ValueAsAny(0)
+ require.NotNil(t, got)
+ gotErr, ok := got.(error)
+ require.True(t, ok)
+ assert.ErrorIs(t, gotErr, variant.ErrInvalidMetadata)
+}
