This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 05a57dece1 GH-35421: [Go] Ensure interface contract between
`array.X.ValueStr` & `array.XBuilder.AppendValueFromString` (#35457)
05a57dece1 is described below
commit 05a57dece1d093281fb04e4a0f9a1289ad4ed325
Author: Alex Shcherbakov <[email protected]>
AuthorDate: Mon May 8 23:21:20 2023 +0300
GH-35421: [Go] Ensure interface contract between `array.X.ValueStr` &
`array.XBuilder.AppendValueFromString` (#35457)
### Rationale for this change
I noticed that some values produced by `array.X.ValueStr` can't be parsed
back by `array.XBuilder.AppendValueFromString` while debugging
https://github.com/cloudquery/cloudquery/pull/10284.
Additionally, some arrays didn't implement the corresponding functions at
all.
### What changes are included in this PR?
* Ensure interface contract
* Use `array.NullValueStr` constant
* Fix bugs found along the way
* Synced `internal/types/UUID` with CQ implementation to account for
`valid` param
### Are these changes tested?
This code was developed in TDD mode. The changes workflow was:
1. Introduce `XStringRoundTrip` tests to test that the resulting array will
be equal in the Arrow sense (using
[`array.Equal`](https://pkg.go.dev/github.com/apache/arrow/go/v12/arrow/array#Equal))
2. Fix any discrepancies
### Are there any user-facing changes?
Some fixes to the accepted values & parse logic to correspond to the
interface contract.
* Closes: #35421
Authored-by: candiduslynx <[email protected]>
Signed-off-by: Matt Topol <[email protected]>
---
.gitignore | 2 +-
go/arrow/array/array.go | 4 +
go/arrow/array/binary.go | 8 +-
go/arrow/array/binary_test.go | 42 +-
go/arrow/array/binarybuilder.go | 19 +-
go/arrow/array/binarybuilder_test.go | 1 -
go/arrow/array/boolean.go | 2 +-
go/arrow/array/boolean_test.go | 32 +-
go/arrow/array/booleanbuilder_test.go | 4 +-
go/arrow/array/decimal128.go | 6 +-
go/arrow/array/decimal128_test.go | 44 +-
go/arrow/array/decimal256.go | 6 +-
go/arrow/array/decimal256_test.go | 44 +-
go/arrow/array/dictionary.go | 26 +-
go/arrow/array/dictionary_test.go | 68 ++-
go/arrow/array/encoded.go | 62 ++-
go/arrow/array/encoded_test.go | 45 ++
go/arrow/array/fixed_size_list.go | 9 +-
go/arrow/array/fixed_size_list_test.go | 41 +-
go/arrow/array/fixedsize_binary.go | 2 +-
go/arrow/array/fixedsize_binary_test.go | 35 ++
go/arrow/array/float16.go | 9 +-
go/arrow/array/float16_builder_test.go | 38 +-
go/arrow/array/interval.go | 25 +-
go/arrow/array/interval_test.go | 107 +++++
go/arrow/array/list.go | 13 +-
go/arrow/array/list_test.go | 84 ++++
go/arrow/array/map.go | 4 +-
go/arrow/array/map_test.go | 45 ++
go/arrow/array/null.go | 9 +-
go/arrow/array/null_test.go | 29 ++
go/arrow/array/numeric.gen.go | 32 +-
go/arrow/array/numeric.gen.go.tmpl | 2 +-
go/arrow/array/numericbuilder.gen.go | 7 +-
go/arrow/array/numericbuilder.gen.go.tmpl | 11 +-
go/arrow/array/numericbuilder.gen_test.go | 594 ++++++++++++++++++++++++-
go/arrow/array/numericbuilder.gen_test.go.tmpl | 65 ++-
go/arrow/array/string.go | 22 +-
go/arrow/array/string_test.go | 64 +++
go/arrow/array/struct.go | 21 +-
go/arrow/array/struct_test.go | 42 +-
go/arrow/array/union.go | 66 ++-
go/arrow/array/union_test.go | 88 +++-
go/arrow/csv/reader_test.go | 4 +-
go/arrow/example_test.go | 6 +-
go/arrow/math/float64_arm64.go | 5 +-
go/arrow/math/float64_neon_arm64.go | 3 +-
go/arrow/math/float64_noasm.go | 1 +
go/arrow/math/float64_ppc64le.go | 3 +-
go/arrow/math/float64_s390x.go | 1 +
go/arrow/math/int64_arm64.go | 5 +-
go/arrow/math/int64_neon_arm64.go | 3 +-
go/arrow/math/int64_noasm.go | 1 +
go/arrow/math/int64_ppc64le.go | 3 +-
go/arrow/math/int64_s390x.go | 1 +
go/arrow/math/uint64_arm64.go | 5 +-
go/arrow/math/uint64_neon_arm64.go | 3 +-
go/arrow/math/uint64_noasm.go | 1 +
go/arrow/math/uint64_ppc64le.go | 3 +-
go/arrow/math/uint64_s390x.go | 1 +
go/internal/types/extension_types.go | 184 ++++----
go/internal/types/extension_types_test.go | 33 ++
62 files changed, 1899 insertions(+), 246 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7db35b0ca3..5ab2313829 100644
--- a/.gitignore
+++ b/.gitignore
@@ -94,4 +94,4 @@ dev/archery/build
swift/Arrow/.build
# Go dependencies
-go/vendor
\ No newline at end of file
+go/vendor
diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go
index 1cca0d2633..83273211f6 100644
--- a/go/arrow/array/array.go
+++ b/go/arrow/array/array.go
@@ -27,6 +27,10 @@ import (
const (
// UnknownNullCount specifies the NullN should be calculated from the
null bitmap buffer.
UnknownNullCount = -1
+
+ // NullValueStr represents a null value in arrow.Array.ValueStr and in
Builder.AppendValueFromString.
+ // It should be returned from the arrow.Array.ValueStr implementations.
+ // Using it as the value in Builder.AppendValueFromString should be
equivalent to Builder.AppendNull.
NullValueStr = "(null)"
)
diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go
index 8db30bf52a..787059f071 100644
--- a/go/arrow/array/binary.go
+++ b/go/arrow/array/binary.go
@@ -57,7 +57,7 @@ func (a *Binary) Value(i int) []byte {
return a.valueBytes[a.valueOffsets[idx]:a.valueOffsets[idx+1]]
}
-// ValueString returns the string at index i
+// ValueStr returns a copy of the base64-encoded string value or NullValueStr
func (a *Binary) ValueStr(i int) string {
if a.IsNull(i) {
return NullValueStr
@@ -65,7 +65,7 @@ func (a *Binary) ValueStr(i int) string {
return base64.StdEncoding.EncodeToString(a.Value(i))
}
-// ValueStr returns the string at index i without performing additional
allocations.
+// ValueString returns the string at index i without performing additional
allocations.
// The string is only valid for the lifetime of the Binary array.
func (a *Binary) ValueString(i int) string {
b := a.Value(i)
@@ -112,7 +112,7 @@ func (a *Binary) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%q", a.ValueString(i))
}
@@ -251,7 +251,7 @@ func (a *LargeBinary) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(&o, "%q", a.ValueString(i))
}
diff --git a/go/arrow/array/binary_test.go b/go/arrow/array/binary_test.go
index 863013fa7b..b1dd5a5358 100644
--- a/go/arrow/array/binary_test.go
+++ b/go/arrow/array/binary_test.go
@@ -20,11 +20,10 @@ import (
"reflect"
"testing"
- "github.com/stretchr/testify/assert"
-
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/bitutil"
"github.com/apache/arrow/go/v13/arrow/memory"
+ "github.com/stretchr/testify/assert"
)
func TestBinary(t *testing.T) {
@@ -51,7 +50,7 @@ func TestBinary(t *testing.T) {
assert.Equal(t, []byte{}, a.Value(1))
assert.Equal(t, []byte("BBBB"), a.Value(2))
assert.Equal(t, "QUFB", a.ValueStr(0))
- assert.Equal(t, "(null)", a.ValueStr(1))
+ assert.Equal(t, NullValueStr, a.ValueStr(1))
a.Release()
// Test builder reset and NewArray API.
@@ -63,7 +62,7 @@ func TestBinary(t *testing.T) {
assert.Equal(t, []byte{}, a.Value(1))
assert.Equal(t, []byte("BBBB"), a.Value(2))
assert.Equal(t, "QUFB", a.ValueStr(0))
- assert.Equal(t, "(null)", a.ValueStr(1))
+ assert.Equal(t, NullValueStr, a.ValueStr(1))
a.Release()
b.Release()
@@ -97,7 +96,7 @@ func TestLargeBinary(t *testing.T) {
assert.Equal(t, []byte{}, a.Value(1))
assert.Equal(t, []byte("BBBB"), a.Value(2))
assert.Equal(t, "QUFB", a.ValueStr(0))
- assert.Equal(t, "(null)", a.ValueStr(1))
+ assert.Equal(t, NullValueStr, a.ValueStr(1))
a.Release()
// Test builder reset and NewArray API.
@@ -109,7 +108,7 @@ func TestLargeBinary(t *testing.T) {
assert.Equal(t, []byte{}, a.Value(1))
assert.Equal(t, []byte("BBBB"), a.Value(2))
assert.Equal(t, "QUFB", a.ValueStr(0))
- assert.Equal(t, "(null)", a.ValueStr(1))
+ assert.Equal(t, NullValueStr, a.ValueStr(1))
a.Release()
b.Release()
@@ -670,3 +669,34 @@ func TestBinaryInvalidOffsets(t *testing.T) {
NewBinaryData(NewData(arrow.BinaryTypes.Binary, 1, buffers,
nil, 0, 2))
}, "data has offset and value offset is overflowing")
}
+
+func TestBinaryStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ values := []string{"a", "bc", "", "", "hijk", "lm", "", "opq", "", "tu"}
+ valid := []bool{true, true, false, false, true, true, true, true,
false, true}
+
+ b := NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
+ defer b.Release()
+
+ b.AppendStringValues(values, valid)
+
+ arr := b.NewArray().(*Binary)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+
+ b1 := NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*Binary)
+ defer arr1.Release()
+
+ assert.True(t, Equal(arr, arr1))
+}
diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go
index 955f0d0744..f86dae8504 100644
--- a/go/arrow/array/binarybuilder.go
+++ b/go/arrow/array/binarybuilder.go
@@ -294,18 +294,17 @@ func (b *BinaryBuilder) AppendValueFromString(s string)
error {
b.AppendNull()
return nil
}
- switch b.dtype.ID() {
- case arrow.BINARY, arrow.LARGE_BINARY:
- decodedVal, err := base64.StdEncoding.DecodeString(s)
- if err != nil {
- return fmt.Errorf("could not decode base64 string: %w",
err)
- }
- b.Append(decodedVal)
- case arrow.STRING, arrow.LARGE_STRING:
+
+ if b.dtype.IsUtf8() {
b.Append([]byte(s))
- default:
- return fmt.Errorf("cannot append string to type %s", b.dtype)
+ return nil
+ }
+
+ decodedVal, err := base64.StdEncoding.DecodeString(s)
+ if err != nil {
+ return fmt.Errorf("could not decode base64 string: %w", err)
}
+ b.Append(decodedVal)
return nil
}
diff --git a/go/arrow/array/binarybuilder_test.go
b/go/arrow/array/binarybuilder_test.go
index b22b4f9a0f..48dce6aca4 100644
--- a/go/arrow/array/binarybuilder_test.go
+++ b/go/arrow/array/binarybuilder_test.go
@@ -38,7 +38,6 @@ func TestBinaryBuilder(t *testing.T) {
ab.AppendNull()
} else {
ab.Append(v)
-
}
}
diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go
index b7bfaf5e77..0ad9c9b067 100644
--- a/go/arrow/array/boolean.go
+++ b/go/arrow/array/boolean.go
@@ -73,7 +73,7 @@ func (a *Boolean) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
diff --git a/go/arrow/array/boolean_test.go b/go/arrow/array/boolean_test.go
index 5cc1cc775c..48b127ae9f 100644
--- a/go/arrow/array/boolean_test.go
+++ b/go/arrow/array/boolean_test.go
@@ -288,5 +288,35 @@ func TestBooleanStringer(t *testing.T) {
}
assert.Equal(t, "true", arr.ValueStr(0))
assert.Equal(t, "false", arr.ValueStr(1))
- assert.Equal(t, "(null)", arr.ValueStr(2))
+ assert.Equal(t, array.NullValueStr, arr.ValueStr(2))
+}
+
+func TestBooleanStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ values := []bool{true, false, true, true, true, true, true, false,
true, false}
+ valid := []bool{true, false, false, true, false, true, true, false,
true, false}
+
+ b := array.NewBooleanBuilder(mem)
+ defer b.Release()
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.Boolean)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewBooleanBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Boolean)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
}
diff --git a/go/arrow/array/booleanbuilder_test.go
b/go/arrow/array/booleanbuilder_test.go
index 1fa75c7812..ef7064163e 100644
--- a/go/arrow/array/booleanbuilder_test.go
+++ b/go/arrow/array/booleanbuilder_test.go
@@ -32,7 +32,7 @@ func TestBooleanBuilder_AppendValues(t *testing.T) {
b := array.NewBooleanBuilder(mem)
exp := tools.Bools(1, 1, 0, 1, 1, 0)
- got := make([]bool, len(exp) + 2)
+ got := make([]bool, len(exp)+2)
b.AppendValues(exp, nil)
assert.NoError(t, b.AppendValueFromString("true"))
@@ -44,7 +44,7 @@ func TestBooleanBuilder_AppendValues(t *testing.T) {
got[i] = a.Value(i)
}
assert.Equal(t, exp, got)
-
+
a.Release()
}
diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go
index 13b5fe5c41..f943e0c3da 100644
--- a/go/arrow/array/decimal128.go
+++ b/go/arrow/array/decimal128.go
@@ -48,12 +48,12 @@ func NewDecimal128Data(data arrow.ArrayData) *Decimal128 {
}
func (a *Decimal128) Value(i int) decimal128.Num { return a.values[i] }
+
func (a *Decimal128) ValueStr(i int) string {
if a.IsNull(i) {
return NullValueStr
- } else {
- return a.GetOneForMarshal(i).(string)
}
+ return a.GetOneForMarshal(i).(string)
}
func (a *Decimal128) Values() []decimal128.Num { return a.values }
@@ -67,7 +67,7 @@ func (a *Decimal128) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
diff --git a/go/arrow/array/decimal128_test.go
b/go/arrow/array/decimal128_test.go
index bca928c991..da2c57e93c 100644
--- a/go/arrow/array/decimal128_test.go
+++ b/go/arrow/array/decimal128_test.go
@@ -171,7 +171,7 @@ func TestDecimal128Slice(t *testing.T) {
if got, want := v.String(), `[(null) {4 -4}]`; got != want {
t.Fatalf("got=%q, want=%q", got, want)
}
- assert.Equal(t, "(null)", v.ValueStr(0))
+ assert.Equal(t, array.NullValueStr, v.ValueStr(0))
assert.Equal(t, "-7.378697629e+18", v.ValueStr(1))
if got, want := v.NullN(), 1; got != want {
@@ -182,3 +182,45 @@ func TestDecimal128Slice(t *testing.T) {
t.Fatalf("invalid offset: got=%d, want=%d", got, want)
}
}
+
+func TestDecimal128StringRoundTrip(t *testing.T) {
+ dt := &arrow.Decimal128Type{Precision: 20, Scale: 5}
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewDecimal128Builder(mem, dt)
+ defer b.Release()
+
+ values := []decimal128.Num{
+ decimal128.New(1, 1),
+ decimal128.New(1, 2),
+ decimal128.New(1, 3),
+ {},
+ decimal128.FromI64(-5),
+ decimal128.FromI64(-6),
+ {},
+ decimal128.FromI64(8),
+ decimal128.FromI64(9),
+ decimal128.FromI64(10),
+ }
+ valid := []bool{true, true, true, false, true, true, false, true, true,
true}
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.Decimal128)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDecimal128Builder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Decimal128)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/decimal256.go b/go/arrow/array/decimal256.go
index 8273cb0825..8cd4476030 100644
--- a/go/arrow/array/decimal256.go
+++ b/go/arrow/array/decimal256.go
@@ -48,12 +48,12 @@ func NewDecimal256Data(data arrow.ArrayData) *Decimal256 {
}
func (a *Decimal256) Value(i int) decimal256.Num { return a.values[i] }
+
func (a *Decimal256) ValueStr(i int) string {
if a.IsNull(i) {
return NullValueStr
- } else {
- return a.GetOneForMarshal(i).(string)
}
+ return a.GetOneForMarshal(i).(string)
}
func (a *Decimal256) Values() []decimal256.Num { return a.values }
@@ -67,7 +67,7 @@ func (a *Decimal256) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
diff --git a/go/arrow/array/decimal256_test.go
b/go/arrow/array/decimal256_test.go
index 3441c9eba2..6fa2858d59 100644
--- a/go/arrow/array/decimal256_test.go
+++ b/go/arrow/array/decimal256_test.go
@@ -172,7 +172,7 @@ func TestDecimal256Slice(t *testing.T) {
if got, want := v.String(), `[(null) {[4 4 4 4]}]`; got != want {
t.Fatalf("got=%q, want=%q", got, want)
}
- assert.Equal(t, "(null)", v.ValueStr(0))
+ assert.Equal(t, array.NullValueStr, v.ValueStr(0))
assert.Equal(t, "2.510840694e+57", v.ValueStr(1))
if got, want := v.NullN(), 1; got != want {
@@ -183,3 +183,45 @@ func TestDecimal256Slice(t *testing.T) {
t.Fatalf("invalid offset: got=%d, want=%d", got, want)
}
}
+
+func TestDecimal256StringRoundTrip(t *testing.T) {
+ dt := &arrow.Decimal256Type{Precision: 70, Scale: 10}
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewDecimal256Builder(mem, dt)
+ defer b.Release()
+
+ values := []decimal256.Num{
+ decimal256.New(1, 1, 1, 1),
+ decimal256.New(2, 2, 2, 2),
+ decimal256.New(3, 3, 3, 3),
+ {},
+ decimal256.FromI64(-5),
+ decimal256.FromI64(-6),
+ {},
+ decimal256.FromI64(8),
+ decimal256.FromI64(9),
+ decimal256.FromI64(10),
+ }
+ valid := []bool{true, true, true, false, true, true, false, true, true,
true}
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.Decimal256)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDecimal256Builder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Decimal256)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/dictionary.go b/go/arrow/array/dictionary.go
index 6cf3fb2e5c..1b0de34b8e 100644
--- a/go/arrow/array/dictionary.go
+++ b/go/arrow/array/dictionary.go
@@ -254,6 +254,9 @@ func (d *Dictionary) CanCompareIndices(other *Dictionary)
bool {
}
func (d *Dictionary) ValueStr(i int) string {
+ if d.IsNull(i) {
+ return NullValueStr
+ }
return d.Dictionary().ValueStr(d.GetValueIndex(i))
}
@@ -713,7 +716,6 @@ func (b *dictionaryBuilder) ResetFull() {
func (b *dictionaryBuilder) Cap() int { return b.idxBuilder.Cap() }
-// UnmarshalJSON is not yet implemented for dictionary builders and will
always error.
func (b *dictionaryBuilder) UnmarshalJSON(data []byte) error {
dec := json.NewDecoder(bytes.NewReader(data))
t, err := dec.Token()
@@ -742,11 +744,29 @@ func (b *dictionaryBuilder) Unmarshal(dec *json.Decoder)
error {
}
func (b *dictionaryBuilder) AppendValueFromString(s string) error {
- return fmt.Errorf("%w: AppendValueFromString to dictionary not yet
implemented", arrow.ErrNotImplemented)
+ bldr := NewBuilder(b.mem, b.dt.ValueType)
+ defer bldr.Release()
+
+ if err := bldr.AppendValueFromString(s); err != nil {
+ return err
+ }
+
+ arr := bldr.NewArray()
+ defer arr.Release()
+ return b.AppendArray(arr)
}
func (b *dictionaryBuilder) UnmarshalOne(dec *json.Decoder) error {
- return errors.New("unmarshal json to dictionary not yet implemented")
+ bldr := NewBuilder(b.mem, b.dt.ValueType)
+ defer bldr.Release()
+
+ if err := bldr.UnmarshalOne(dec); err != nil {
+ return err
+ }
+
+ arr := bldr.NewArray()
+ defer arr.Release()
+ return b.AppendArray(arr)
}
func (b *dictionaryBuilder) NewArray() arrow.Array {
diff --git a/go/arrow/array/dictionary_test.go
b/go/arrow/array/dictionary_test.go
index 47e5fa7528..7758c82c45 100644
--- a/go/arrow/array/dictionary_test.go
+++ b/go/arrow/array/dictionary_test.go
@@ -385,6 +385,39 @@ func (p *PrimitiveDictionaryTestSuite) TestResetFull() {
p.True(array.ArrayEqual(exdict, result.Dictionary()))
}
+func (p *PrimitiveDictionaryTestSuite) TestStringRoundTrip() {
+ dt := &arrow.DictionaryType{IndexType: &arrow.Int8Type{}, ValueType:
p.typ}
+ b := array.NewDictionaryBuilder(p.mem, dt)
+ defer b.Release()
+
+ builder := reflect.ValueOf(b)
+ fn := builder.MethodByName("Append")
+
p.Nil(fn.Call([]reflect.Value{reflect.ValueOf(1).Convert(p.reftyp)})[0].Interface())
+
p.Nil(fn.Call([]reflect.Value{reflect.ValueOf(2).Convert(p.reftyp)})[0].Interface())
+
p.Nil(fn.Call([]reflect.Value{reflect.ValueOf(1).Convert(p.reftyp)})[0].Interface())
+ b.AppendNull()
+
+ p.EqualValues(4, b.Len())
+ p.EqualValues(1, b.NullN())
+
+ arr := b.NewArray().(*array.Dictionary)
+ defer arr.Release()
+ p.True(arrow.TypeEqual(dt, arr.DataType()))
+
+ b1 := array.NewDictionaryBuilder(p.mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ p.NoError(b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Dictionary)
+ defer arr1.Release()
+
+ p.Equal(arr.Len(), arr1.Len())
+ p.True(array.Equal(arr, arr1))
+}
+
func TestBasicStringDictionaryBuilder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
@@ -808,6 +841,39 @@ func TestFixedSizeBinaryDictionaryBuilderDeltaDictionary(t
*testing.T) {
assert.True(t, array.ArrayEqual(fsbArr2, delta2))
}
+func TestFixedSizeBinaryDictionaryStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ dictType := &arrow.DictionaryType{IndexType: &arrow.Int8Type{},
ValueType: &arrow.FixedSizeBinaryType{ByteWidth: 4}}
+ b := array.NewDictionaryBuilder(mem, dictType)
+ defer b.Release()
+
+ builder := b.(*array.FixedSizeBinaryDictionaryBuilder)
+ test := []byte{12, 12, 11, 12}
+ test2 := []byte{12, 12, 11, 11}
+ assert.NoError(t, builder.Append(test))
+ assert.NoError(t, builder.Append(test2))
+ assert.NoError(t, builder.Append(test))
+
+ arr := builder.NewDictionaryArray()
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDictionaryBuilder(mem, dictType)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Dictionary)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestDecimalDictionaryBuilderBasic(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
@@ -1624,7 +1690,7 @@ func TestDictionaryUnifierChunkedArrayNestedDict(t
*testing.T) {
assert.EqualError(t, err, "unimplemented dictionary value type,
list<item: dictionary<values=utf8, indices=uint32, ordered=false>, nullable>")
}
-func TestDictioanryUnifierTableZeroColumns(t *testing.T) {
+func TestDictionaryUnifierTableZeroColumns(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
diff --git a/go/arrow/array/encoded.go b/go/arrow/array/encoded.go
index d9cbc0980a..5e9fa65259 100644
--- a/go/arrow/array/encoded.go
+++ b/go/arrow/array/encoded.go
@@ -21,7 +21,6 @@ import (
"fmt"
"math"
"reflect"
- "strings"
"sync/atomic"
"github.com/apache/arrow/go/v13/arrow"
@@ -193,15 +192,19 @@ func (r *RunEndEncoded) GetPhysicalLength() int {
return encoded.GetPhysicalLength(r.data)
}
+// GetPhysicalIndex can be used to get the run-encoded value instead of costly
LogicalValuesArray
+// in the following way:
+//
+// r.Values().(valuetype).Value(r.GetPhysicalIndex(i))
+func (r *RunEndEncoded) GetPhysicalIndex(i int) int {
+ return encoded.FindPhysicalIndex(r.data, i+r.data.offset)
+}
+
+// ValueStr will return the str representation of the value at the logical
offset i.
func (r *RunEndEncoded) ValueStr(i int) string {
- value := r.values.GetOneForMarshal(i)
- if byts, ok := value.(json.RawMessage); ok {
- value = string(byts)
- }
- return fmt.Sprintf("{%d -> %v}",
- r.ends.GetOneForMarshal(i),
- value)
+ return r.values.ValueStr(r.GetPhysicalIndex(i))
}
+
func (r *RunEndEncoded) String() string {
var buf bytes.Buffer
buf.WriteByte('[')
@@ -214,9 +217,7 @@ func (r *RunEndEncoded) String() string {
if byts, ok := value.(json.RawMessage); ok {
value = string(byts)
}
- fmt.Fprintf(&buf, "{%d -> %v}",
- r.ends.GetOneForMarshal(i),
- value)
+ fmt.Fprintf(&buf, "{%d -> %v}", r.ends.GetOneForMarshal(i),
value)
}
buf.WriteByte(']')
@@ -224,8 +225,7 @@ func (r *RunEndEncoded) String() string {
}
func (r *RunEndEncoded) GetOneForMarshal(i int) interface{} {
- physIndex := encoded.FindPhysicalIndex(r.data, i+r.data.offset)
- return r.values.GetOneForMarshal(physIndex)
+ return r.values.GetOneForMarshal(r.GetPhysicalIndex(i))
}
func (r *RunEndEncoded) MarshalJSON() ([]byte, error) {
@@ -280,7 +280,10 @@ type RunEndEncodedBuilder struct {
values Builder
maxRunEnd uint64
+ // currently, mixing AppendValueFromString & UnmarshalOne is unsupported
lastUnmarshalled interface{}
+ unmarshalled bool // tracks if Unmarshal was called (in case
lastUnmarshalled is nil)
+ lastStr *string
}
func NewRunEndEncodedBuilder(mem memory.Allocator, runEnds, encoded
arrow.DataType) *RunEndEncodedBuilder {
@@ -331,6 +334,8 @@ func (b *RunEndEncodedBuilder) addLength(n uint64) {
func (b *RunEndEncodedBuilder) finishRun() {
b.lastUnmarshalled = nil
+ b.lastStr = nil
+ b.unmarshalled = false
if b.length == 0 {
return
}
@@ -407,12 +412,36 @@ func (b *RunEndEncodedBuilder) newData() (data *Data) {
return
}
+// AppendValueFromString can't be used in conjunction with UnmarshalOne
func (b *RunEndEncodedBuilder) AppendValueFromString(s string) error {
- dec := json.NewDecoder(strings.NewReader(s))
- return b.UnmarshalOne(dec)
+ // we don't support mixing AppendValueFromString & UnmarshalOne
+ if b.unmarshalled {
+ return fmt.Errorf("%w: mixing AppendValueFromString &
UnmarshalOne not yet implemented", arrow.ErrNotImplemented)
+ }
+
+ if s == NullValueStr {
+ b.AppendNull()
+ return nil
+ }
+
+ if b.lastStr != nil && s == *b.lastStr {
+ b.ContinueRun(1)
+ return nil
+ }
+
+ b.Append(1)
+ lastStr := s
+ b.lastStr = &lastStr
+ return b.ValueBuilder().AppendValueFromString(s)
}
+// UnmarshalOne can't be used in conjunction with AppendValueFromString
func (b *RunEndEncodedBuilder) UnmarshalOne(dec *json.Decoder) error {
+ // we don't support mixing AppendValueFromString & UnmarshalOne
+ if b.lastStr != nil {
+ return fmt.Errorf("%w: mixing AppendValueFromString &
UnmarshalOne not yet implemented", arrow.ErrNotImplemented)
+ }
+
var value interface{}
if err := dec.Decode(&value); err != nil {
return err
@@ -437,9 +466,11 @@ func (b *RunEndEncodedBuilder) UnmarshalOne(dec
*json.Decoder) error {
b.Append(1)
b.lastUnmarshalled = value
+ b.unmarshalled = true
return
b.ValueBuilder().UnmarshalOne(json.NewDecoder(bytes.NewReader(data)))
}
+// Unmarshal can't be used in conjunction with AppendValueFromString (as it
calls UnmarshalOne)
func (b *RunEndEncodedBuilder) Unmarshal(dec *json.Decoder) error {
b.finishRun()
for dec.More() {
@@ -450,6 +481,7 @@ func (b *RunEndEncodedBuilder) Unmarshal(dec *json.Decoder)
error {
return nil
}
+// UnmarshalJSON can't be used in conjunction with AppendValueFromString (as
it calls UnmarshalOne)
func (b *RunEndEncodedBuilder) UnmarshalJSON(data []byte) error {
dec := json.NewDecoder(bytes.NewReader(data))
t, err := dec.Token()
diff --git a/go/arrow/array/encoded_test.go b/go/arrow/array/encoded_test.go
index c24b71cc5f..c8be6d193a 100644
--- a/go/arrow/array/encoded_test.go
+++ b/go/arrow/array/encoded_test.go
@@ -253,6 +253,51 @@ func TestRunEndEncodedBuilder(t *testing.T) {
assert.Equal(t, "Hello", strValues.ValueStr(0))
}
+func TestRunEndEncodedStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
arrow.BinaryTypes.String)
+ defer b.Release()
+
+ valBldr := b.ValueBuilder().(*array.StringBuilder)
+
+ b.Append(100)
+ valBldr.Append("Hello")
+ b.Append(100)
+ valBldr.Append("beautiful")
+ b.Append(50)
+ valBldr.Append("world")
+ b.ContinueRun(50)
+ b.Append(100)
+ valBldr.Append("of")
+ b.Append(100)
+ valBldr.Append("RLE")
+ b.AppendNull()
+
+ arr := b.NewArray().(*array.RunEndEncoded)
+ defer arr.Release()
+ logical := arr.LogicalValuesArray()
+ defer logical.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewRunEndEncodedBuilder(mem, arrow.PrimitiveTypes.Int16,
arrow.BinaryTypes.String)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.RunEndEncoded)
+ defer arr1.Release()
+ logical1 := arr1.LogicalValuesArray()
+ defer logical1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+ assert.True(t, array.Equal(logical, logical1))
+}
+
func TestREEBuilderOverflow(t *testing.T) {
for _, typ := range []arrow.DataType{arrow.PrimitiveTypes.Int16,
arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int64} {
t.Run("run_ends="+typ.String(), func(t *testing.T) {
diff --git a/go/arrow/array/fixed_size_list.go
b/go/arrow/array/fixed_size_list.go
index 1afbf10332..cadcfeb603 100644
--- a/go/arrow/array/fixed_size_list.go
+++ b/go/arrow/array/fixed_size_list.go
@@ -49,7 +49,7 @@ func NewFixedSizeListData(data arrow.ArrayData)
*FixedSizeList {
func (a *FixedSizeList) ListValues() arrow.Array { return a.values }
func (a *FixedSizeList) ValueStr(i int) string {
- if !a.IsValid(i) {
+ if a.IsNull(i) {
return NullValueStr
}
return string(a.GetOneForMarshal(i).(json.RawMessage))
@@ -62,7 +62,7 @@ func (a *FixedSizeList) String() string {
o.WriteString(" ")
}
if !a.IsValid(i) {
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
continue
}
sub := a.newListValue(i)
@@ -208,6 +208,7 @@ func (b *FixedSizeListBuilder) Append(v bool) {
func (b *FixedSizeListBuilder) AppendNull() {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(false)
+ // require to append this due to value indexes
for i := int32(0); i < b.n; i++ {
b.values.AppendNull()
}
@@ -294,6 +295,10 @@ func (b *FixedSizeListBuilder) newData() (data *Data) {
}
func (b *FixedSizeListBuilder) AppendValueFromString(s string) error {
+ if s == NullValueStr {
+ b.AppendNull()
+ return nil
+ }
dec := json.NewDecoder(strings.NewReader(s))
return b.UnmarshalOne(dec)
}
diff --git a/go/arrow/array/fixed_size_list_test.go
b/go/arrow/array/fixed_size_list_test.go
index fef8054701..10e1f78f02 100644
--- a/go/arrow/array/fixed_size_list_test.go
+++ b/go/arrow/array/fixed_size_list_test.go
@@ -173,7 +173,7 @@ func TestFixedSizeListArrayStringer(t *testing.T) {
t.Fatalf("got=%q, want=%q", got, want)
}
assert.Equal(t, "[0,1,2]", arr.ValueStr(0))
- assert.Equal(t, "(null)", arr.ValueStr(1))
+ assert.Equal(t, array.NullValueStr, arr.ValueStr(1))
}
func TestFixedSizeListArraySlice(t *testing.T) {
@@ -216,3 +216,42 @@ func TestFixedSizeListArraySlice(t *testing.T) {
t.Fatalf("got=%q, want=%q", got, want)
}
}
+
+func TestFixedSizeListStringRoundTrip(t *testing.T) {
+ // 1. create array
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ const N = 3
+ var (
+ values = [][N]int32{{0, 1, 2}, {3, 4, 5}, {6, 7, 8}, {9, -9,
-8}}
+ valid = []bool{true, false, true, true}
+ )
+
+ b := array.NewFixedSizeListBuilder(pool, N, arrow.PrimitiveTypes.Int32)
+ defer b.Release()
+
+ vb := b.ValueBuilder().(*array.Int32Builder)
+ vb.Reserve(len(values))
+
+ for i, v := range values {
+ b.Append(valid[i])
+ vb.AppendValues(v[:], nil)
+ }
+
+ arr := b.NewArray().(*array.FixedSizeList)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewFixedSizeListBuilder(pool, N, arrow.PrimitiveTypes.Int32)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.FixedSizeList)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/fixedsize_binary.go
b/go/arrow/array/fixedsize_binary.go
index 64d7a8a057..c7f0b6f479 100644
--- a/go/arrow/array/fixedsize_binary.go
+++ b/go/arrow/array/fixedsize_binary.go
@@ -68,7 +68,7 @@ func (a *FixedSizeBinary) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
diff --git a/go/arrow/array/fixedsize_binary_test.go
b/go/arrow/array/fixedsize_binary_test.go
index b9db71fe2b..c320605e11 100644
--- a/go/arrow/array/fixedsize_binary_test.go
+++ b/go/arrow/array/fixedsize_binary_test.go
@@ -152,3 +152,38 @@ func TestFixedSizeBinary_MarshalUnmarshalJSON(t
*testing.T) {
t.Fatalf("got=%q, want=%q", gotString, wantString)
}
}
+
+func TestFixedSizeBinaryStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dt := &arrow.FixedSizeBinaryType{ByteWidth: 7}
+ b := array.NewFixedSizeBinaryBuilder(mem, dt)
+
+ values := [][]byte{
+ []byte("7654321"),
+ nil,
+ []byte("AZERTYU"),
+ }
+ valid := []bool{true, false, true}
+ b.AppendValues(values, valid)
+ // encoded abcdefg base64
+ assert.NoError(t, b.AppendValueFromString("YWJjZGVmZw=="))
+
+ arr := b.NewArray().(*array.FixedSizeBinary)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewFixedSizeBinaryBuilder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.FixedSizeBinary)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/float16.go b/go/arrow/array/float16.go
index 98b2e79743..9ccecd3dcd 100644
--- a/go/arrow/array/float16.go
+++ b/go/arrow/array/float16.go
@@ -39,7 +39,12 @@ func NewFloat16Data(data arrow.ArrayData) *Float16 {
}
func (a *Float16) Value(i int) float16.Num { return a.values[i] }
-func (a *Float16) ValueStr(i int) string { return a.Value(i).String()}
+func (a *Float16) ValueStr(i int) string {
+ if a.IsNull(i) {
+ return NullValueStr
+ }
+ return a.Value(i).String()
+}
func (a *Float16) Values() []float16.Num { return a.values }
@@ -52,7 +57,7 @@ func (a *Float16) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", a.values[i].Float32())
}
diff --git a/go/arrow/array/float16_builder_test.go
b/go/arrow/array/float16_builder_test.go
index 2d3867ebd4..a1ad3f7444 100644
--- a/go/arrow/array/float16_builder_test.go
+++ b/go/arrow/array/float16_builder_test.go
@@ -50,7 +50,7 @@ func TestNewFloat16Builder(t *testing.T) {
ab.Append(float16.New(9))
ab.Append(float16.New(10))
assert.NoError(t, ab.AppendValueFromString("11.0"))
-
+
// check state of builder before NewFloat16Array
assert.Equal(t, 11, ab.Len(), "unexpected Len()")
assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
@@ -118,3 +118,39 @@ func TestFloat16Builder_Empty(t *testing.T) {
assert.Equal(t, want, a.Values())
a.Release()
}
+
+func TestFloat16StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewFloat16Builder(mem)
+ defer b.Release()
+
+ b.Append(float16.New(1))
+ b.Append(float16.New(2))
+ b.Append(float16.New(3))
+ b.AppendNull()
+ b.Append(float16.New(5))
+ b.Append(float16.New(6))
+ b.AppendNull()
+ b.Append(float16.New(8))
+ b.Append(float16.New(9))
+ b.Append(float16.New(10))
+
+ arr := b.NewArray().(*array.Float16)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewFloat16Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Float16)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go
index 48f7bf9dda..c105e04f8c 100644
--- a/go/arrow/array/interval.go
+++ b/go/arrow/array/interval.go
@@ -56,7 +56,7 @@ func NewMonthIntervalData(data arrow.ArrayData)
*MonthInterval {
return a
}
-func (a *MonthInterval) Value(i int) arrow.MonthInterval { return
a.values[i] }
+func (a *MonthInterval) Value(i int) arrow.MonthInterval { return a.values[i] }
func (a *MonthInterval) ValueStr(i int) string {
if a.IsNull(i) {
return NullValueStr
@@ -74,7 +74,7 @@ func (a *MonthInterval) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -287,7 +287,7 @@ func (b *MonthIntervalBuilder) AppendValueFromString(s
string) error {
b.Append(arrow.MonthInterval(v))
return nil
}
-
+
func (b *MonthIntervalBuilder) UnmarshalOne(dec *json.Decoder) error {
var v *arrow.MonthInterval
if err := dec.Decode(&v); err != nil {
@@ -341,13 +341,18 @@ func NewDayTimeIntervalData(data arrow.ArrayData)
*DayTimeInterval {
return a
}
-func (a *DayTimeInterval) Value(i int) arrow.DayTimeInterval {
return a.values[i] }
+func (a *DayTimeInterval) Value(i int) arrow.DayTimeInterval { return
a.values[i] }
func (a *DayTimeInterval) ValueStr(i int) string {
if a.IsNull(i) {
return NullValueStr
}
- return fmt.Sprintf("%q", a.Value(i))
+ data, err := json.Marshal(a.GetOneForMarshal(i))
+ if err != nil {
+ panic(err)
+ }
+ return string(data)
}
+
func (a *DayTimeInterval) DayTimeIntervalValues() []arrow.DayTimeInterval {
return a.values }
func (a *DayTimeInterval) String() string {
@@ -359,7 +364,7 @@ func (a *DayTimeInterval) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -628,7 +633,11 @@ func (a *MonthDayNanoInterval) ValueStr(i int) string {
if a.IsNull(i) {
return NullValueStr
}
- return fmt.Sprintf("%q", a.Value(i))
+ data, err := json.Marshal(a.GetOneForMarshal(i))
+ if err != nil {
+ panic(err)
+ }
+ return string(data)
}
func (a *MonthDayNanoInterval) MonthDayNanoIntervalValues()
[]arrow.MonthDayNanoInterval {
@@ -644,7 +653,7 @@ func (a *MonthDayNanoInterval) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
diff --git a/go/arrow/array/interval_test.go b/go/arrow/array/interval_test.go
index c425082d27..cc5b5d8c75 100644
--- a/go/arrow/array/interval_test.go
+++ b/go/arrow/array/interval_test.go
@@ -151,6 +151,38 @@ func TestMonthIntervalBuilder_Empty(t *testing.T) {
arr.Release()
}
+func TestMonthIntervalStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ var (
+ values = []arrow.MonthInterval{1, 2, 3, 4}
+ valid = []bool{true, true, false, true}
+ )
+
+ b := array.NewMonthIntervalBuilder(mem)
+ defer b.Release()
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.MonthInterval)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewMonthIntervalBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.MonthInterval)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestDayTimeArray(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -280,6 +312,43 @@ func TestDayTimeIntervalBuilder_Empty(t *testing.T) {
arr.Release()
}
+func TestDayTimeIntervalStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ var (
+ values = []arrow.DayTimeInterval{
+ {Days: 1, Milliseconds: 1},
+ {Days: 2, Milliseconds: 2},
+ {Days: 3, Milliseconds: 3},
+ {Days: 4, Milliseconds: 4},
+ }
+ valid = []bool{true, true, false, true}
+ )
+
+ b := array.NewDayTimeIntervalBuilder(mem)
+ defer b.Release()
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.DayTimeInterval)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDayTimeIntervalBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.DayTimeInterval)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestMonthDayNanoArray(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -415,3 +484,41 @@ func TestMonthDayNanoIntervalBuilder_Empty(t *testing.T) {
assert.Equal(t, want, dtValues(arr))
arr.Release()
}
+
+func TestMonthDayNanoIntervalStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ var (
+ values = []arrow.MonthDayNanoInterval{
+ {Months: 1, Days: 1, Nanoseconds: 1000}, {Months: 2,
Days: 2, Nanoseconds: 2000},
+ {Months: 3, Days: 3, Nanoseconds: 3000}, {Months: 4,
Days: 4, Nanoseconds: 4000},
+ {Months: 0, Days: 0, Nanoseconds: 0}, {Months: -1,
Days: -2, Nanoseconds: -300},
+ {Months: math.MaxInt32, Days: math.MinInt32,
Nanoseconds: math.MaxInt64},
+ {Months: math.MinInt32, Days: math.MaxInt32,
Nanoseconds: math.MinInt64},
+ }
+ valid = []bool{true, true, false, true, true, true, false, true}
+ )
+
+ b := array.NewMonthDayNanoIntervalBuilder(mem)
+ defer b.Release()
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.MonthDayNanoInterval)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewMonthDayNanoIntervalBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.MonthDayNanoInterval)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go
index 9cf1fc86b0..d61d980aac 100644
--- a/go/arrow/array/list.go
+++ b/go/arrow/array/list.go
@@ -69,7 +69,7 @@ func (a *List) String() string {
o.WriteString(" ")
}
if !a.IsValid(i) {
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
continue
}
sub := a.newListValue(i)
@@ -191,6 +191,7 @@ func (a *LargeList) ValueStr(i int) string {
}
return string(a.GetOneForMarshal(i).(json.RawMessage))
}
+
func (a *LargeList) String() string {
o := new(strings.Builder)
o.WriteString("[")
@@ -199,7 +200,7 @@ func (a *LargeList) String() string {
o.WriteString(" ")
}
if !a.IsValid(i) {
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
continue
}
sub := a.newListValue(i)
@@ -547,8 +548,12 @@ func (b *baseListBuilder) newData() (data *Data) {
}
func (b *baseListBuilder) AppendValueFromString(s string) error {
- dec := json.NewDecoder(strings.NewReader(s))
- return b.UnmarshalOne(dec)
+ if s == NullValueStr {
+ b.AppendNull()
+ return nil
+ }
+
+ return b.UnmarshalOne(json.NewDecoder(strings.NewReader(s)))
}
func (b *baseListBuilder) UnmarshalOne(dec *json.Decoder) error {
diff --git a/go/arrow/array/list_test.go b/go/arrow/array/list_test.go
index 977c7f3e85..138548532f 100644
--- a/go/arrow/array/list_test.go
+++ b/go/arrow/array/list_test.go
@@ -301,3 +301,87 @@ func TestListArraySlice(t *testing.T) {
})
}
}
+
+func TestListStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer b.Release()
+ vb := b.ValueBuilder().(*array.Int32Builder)
+
+ var values = [][]int32{
+ {0, 1, 2, 3, 4, 5, 6},
+ {1, 2, 3, 4, 5, 6, 7},
+ {2, 3, 4, 5, 6, 7, 8},
+ {3, 4, 5, 6, 7, 8, 9},
+ }
+ for _, value := range values {
+ b.AppendNull()
+ b.Append(true)
+ for _, el := range value {
+ vb.Append(el)
+ vb.AppendNull()
+ }
+ b.Append(false)
+ }
+
+ arr := b.NewArray().(*array.List)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.List)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
+func TestLargeListStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewLargeListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer b.Release()
+ vb := b.ValueBuilder().(*array.Int32Builder)
+
+ var values = [][]int32{
+ {0, 1, 2, 3, 4, 5, 6},
+ {1, 2, 3, 4, 5, 6, 7},
+ {2, 3, 4, 5, 6, 7, 8},
+ {3, 4, 5, 6, 7, 8, 9},
+ }
+ for _, value := range values {
+ b.AppendNull()
+ b.Append(true)
+ for _, el := range value {
+ vb.Append(el)
+ vb.AppendNull()
+ }
+ b.Append(false)
+ }
+
+ arr := b.NewArray().(*array.LargeList)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewLargeListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.LargeList)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/map.go b/go/arrow/array/map.go
index 5c3c54713b..815465c1e5 100644
--- a/go/arrow/array/map.go
+++ b/go/arrow/array/map.go
@@ -128,7 +128,7 @@ type MapBuilder struct {
// building using keys in sorted order for each value. The KeysSorted value
will just be
// used when creating the DataType for the map.
//
-// Example
+// # Example
//
// Simple example provided of converting a []map[string]int32 to an array.Map
// by using a MapBuilder:
@@ -303,7 +303,7 @@ func (b *MapBuilder) ValueBuilder() Builder {
}
func (b *MapBuilder) AppendValueFromString(s string) error {
- return arrow.ErrNotImplemented
+ return b.listBuilder.AppendValueFromString(s)
}
func (b *MapBuilder) UnmarshalOne(dec *json.Decoder) error {
diff --git a/go/arrow/array/map_test.go b/go/arrow/array/map_test.go
index b876d01810..c9b87b045a 100644
--- a/go/arrow/array/map_test.go
+++ b/go/arrow/array/map_test.go
@@ -17,6 +17,7 @@
package array_test
import (
+ "strconv"
"testing"
"github.com/apache/arrow/go/v13/arrow"
@@ -172,3 +173,47 @@ func TestMapArrayBuildIntToInt(t *testing.T) {
assert.Equal(t, "[{[0 1 2 3 4 5] [1 1 2 3 5 8]} (null) {[0 1 2 3 4 5]
[(null) (null) 0 1 (null) 2]} {[] []}]", arr.String())
}
+
+func TestMapStringRoundTrip(t *testing.T) {
+ // 1. create array
+ dt := arrow.MapOf(arrow.BinaryTypes.String, arrow.PrimitiveTypes.Int32)
+
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewMapBuilderWithType(mem, dt)
+ defer b.Release()
+
+ kb := b.KeyBuilder().(*array.StringBuilder)
+ ib := b.ItemBuilder().(*array.Int32Builder)
+
+ for n := 0; n < 10; n++ {
+ b.AppendNull()
+ b.Append(true)
+
+ for r := 'a'; r <= 'z'; r++ {
+ kb.Append(string(r) + strconv.Itoa(n))
+ if (n+int(r))%2 == 0 {
+ ib.AppendNull()
+ } else {
+ ib.Append(int32(n + int(r)))
+ }
+ }
+ }
+
+ arr := b.NewArray().(*array.Map)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewMapBuilderWithType(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Map)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go
index d65ba67d94..80cef00777 100644
--- a/go/arrow/array/null.go
+++ b/go/arrow/array/null.go
@@ -58,9 +58,9 @@ func NewNullData(data arrow.ArrayData) *Null {
return a
}
-func (a *Null) ValueStr(i int) string {
- return NullValueStr
-}
+func (a *Null) ValueStr(int) string { return NullValueStr }
+
+func (a *Null) Value(int) interface{} { return nil }
func (a *Null) String() string {
o := new(strings.Builder)
@@ -69,7 +69,7 @@ func (a *Null) String() string {
if i > 0 {
o.WriteString(" ")
}
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
}
o.WriteString("]")
return o.String()
@@ -125,6 +125,7 @@ func (b *NullBuilder) AppendValueFromString(s string) error
{
}
return fmt.Errorf("cannot convert %q to null", s)
}
+
func (b *NullBuilder) AppendEmptyValue() { b.AppendNull() }
func (*NullBuilder) Reserve(size int) {}
diff --git a/go/arrow/array/null_test.go b/go/arrow/array/null_test.go
index c4620ef0b5..ea622dff31 100644
--- a/go/arrow/array/null_test.go
+++ b/go/arrow/array/null_test.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/arrow/go/v13/arrow"
"github.com/apache/arrow/go/v13/arrow/array"
"github.com/apache/arrow/go/v13/arrow/memory"
+ "github.com/stretchr/testify/assert"
)
func TestNullArray(t *testing.T) {
@@ -75,3 +76,31 @@ func TestNullArray(t *testing.T) {
}
}
+
+func TestNullStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewNullBuilder(mem)
+ defer b.Release()
+
+ b.AppendNull()
+ b.AppendNull()
+
+ arr := b.NewArray().(*array.Null)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewNullBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Null)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go
index 1fc26d3c14..1849d6531b 100644
--- a/go/arrow/array/numeric.gen.go
+++ b/go/arrow/array/numeric.gen.go
@@ -62,7 +62,7 @@ func (a *Int64) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -157,7 +157,7 @@ func (a *Uint64) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -252,7 +252,7 @@ func (a *Float64) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -347,7 +347,7 @@ func (a *Int32) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -442,7 +442,7 @@ func (a *Uint32) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -537,7 +537,7 @@ func (a *Float32) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -632,7 +632,7 @@ func (a *Int16) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -727,7 +727,7 @@ func (a *Uint16) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -822,7 +822,7 @@ func (a *Int8) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -917,7 +917,7 @@ func (a *Uint8) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -1012,7 +1012,7 @@ func (a *Timestamp) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -1102,7 +1102,7 @@ func (a *Time32) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -1192,7 +1192,7 @@ func (a *Time64) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -1282,7 +1282,7 @@ func (a *Date32) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -1372,7 +1372,7 @@ func (a *Date64) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
@@ -1462,7 +1462,7 @@ func (a *Duration) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
diff --git a/go/arrow/array/numeric.gen.go.tmpl
b/go/arrow/array/numeric.gen.go.tmpl
index 4b71199289..adece058b8 100644
--- a/go/arrow/array/numeric.gen.go.tmpl
+++ b/go/arrow/array/numeric.gen.go.tmpl
@@ -62,7 +62,7 @@ func (a *{{.Name}}) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%v", v)
}
diff --git a/go/arrow/array/numericbuilder.gen.go
b/go/arrow/array/numericbuilder.gen.go
index a9d00197cc..17fee6688a 100644
--- a/go/arrow/array/numericbuilder.gen.go
+++ b/go/arrow/array/numericbuilder.gen.go
@@ -3534,7 +3534,12 @@ func (b *DurationBuilder) AppendValueFromString(s
string) error {
b.AppendNull()
return nil
}
- return fmt.Errorf("%w: AppendValueFromString not implemented for
Duration", arrow.ErrNotImplemented)
+ dur, err := time.ParseDuration(s)
+ if err != nil {
+ return err
+ }
+
+ b.Append(arrow.Duration(dur / b.dtype.Unit.Multiplier()))
return nil
}
diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl
b/go/arrow/array/numericbuilder.gen.go.tmpl
index 5e615ddc85..d18829b2f0 100644
--- a/go/arrow/array/numericbuilder.gen.go.tmpl
+++ b/go/arrow/array/numericbuilder.gen.go.tmpl
@@ -224,8 +224,13 @@ func (b *{{.Name}}Builder) AppendValueFromString(s string)
error {
return err
}
b.Append(v)
- {{else if (eq .Name "Duration") -}}
- return fmt.Errorf("%w: AppendValueFromString not implemented for
Duration", arrow.ErrNotImplemented)
+ {{else if (eq .Name "Duration") -}}
+ dur, err := time.ParseDuration(s)
+ if err != nil {
+ return err
+ }
+
+ b.Append(arrow.Duration(dur / b.dtype.Unit.Multiplier()))
{{else if or (eq .Name "Int8") (eq .Name "Int16") (eq .Name "Int32") (eq
.Name "Int64") -}}
v, err := strconv.ParseInt(s, 10, {{.Size}} * 8)
if err != nil {
@@ -247,8 +252,6 @@ func (b *{{.Name}}Builder) AppendValueFromString(s string)
error {
return err
}
b.Append({{.name}}(v))
- {{else}}
- return fmt.Errorf("%w: AppendValueFromString not implemented for
{{.Name}}", ErrNotImplemented)
{{end -}}
return nil
}
diff --git a/go/arrow/array/numericbuilder.gen_test.go
b/go/arrow/array/numericbuilder.gen_test.go
index 3a63cdd6ec..90c5a7f452 100644
--- a/go/arrow/array/numericbuilder.gen_test.go
+++ b/go/arrow/array/numericbuilder.gen_test.go
@@ -27,6 +27,42 @@ import (
"github.com/stretchr/testify/assert"
)
+func TestInt64StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewInt64Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Int64)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewInt64Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Int64)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewInt64Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -47,10 +83,9 @@ func TestNewInt64Builder(t *testing.T) {
ab.Append(8)
ab.Append(9)
ab.Append(10)
- assert.NoError(t, ab.AppendValueFromString("11"))
// check state of builder before NewInt64Array
- assert.Equal(t, 11, ab.Len(), "unexpected Len()")
+ assert.Equal(t, 10, ab.Len(), "unexpected Len()")
assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
a := ab.NewInt64Array()
@@ -62,9 +97,9 @@ func TestNewInt64Builder(t *testing.T) {
// check state of array
assert.Equal(t, 2, a.NullN(), "unexpected null count")
- assert.Equal(t, []int64{1, 2, 3, 0, 5, 6, 0, 8, 9, 10, 11},
a.Int64Values(), "unexpected Int64Values")
+ assert.Equal(t, []int64{1, 2, 3, 0, 5, 6, 0, 8, 9, 10},
a.Int64Values(), "unexpected Int64Values")
assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due
to minBuilderCapacity
- assert.Len(t, a.Int64Values(), 11, "unexpected length of Int64Values")
+ assert.Len(t, a.Int64Values(), 10, "unexpected length of Int64Values")
a.Release()
@@ -194,6 +229,42 @@ func TestInt64Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestUint64StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewUint64Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Uint64)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewUint64Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Uint64)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewUint64Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -360,6 +431,42 @@ func TestUint64Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestFloat64StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewFloat64Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Float64)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewFloat64Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Float64)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewFloat64Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -526,6 +633,42 @@ func TestFloat64Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestInt32StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewInt32Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Int32)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewInt32Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Int32)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewInt32Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -692,6 +835,42 @@ func TestInt32Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestUint32StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewUint32Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Uint32)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewUint32Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Uint32)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewUint32Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -858,6 +1037,42 @@ func TestUint32Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestFloat32StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewFloat32Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Float32)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewFloat32Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Float32)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewFloat32Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -1024,6 +1239,42 @@ func TestFloat32Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestInt16StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewInt16Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Int16)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewInt16Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Int16)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewInt16Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -1190,6 +1441,42 @@ func TestInt16Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestUint16StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewUint16Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Uint16)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewUint16Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Uint16)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewUint16Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -1356,6 +1643,42 @@ func TestUint16Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestInt8StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewInt8Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Int8)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewInt8Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Int8)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewInt8Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -1522,6 +1845,42 @@ func TestInt8Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestUint8StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewUint8Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Uint8)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewUint8Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Uint8)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewUint8Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -1688,6 +2047,43 @@ func TestUint8Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestTimestampStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dt := &arrow.TimestampType{Unit: arrow.Second}
+ b := array.NewTimestampBuilder(mem, dt)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Timestamp)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewTimestampBuilder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Timestamp)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewTimestampBuilder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -1858,6 +2254,43 @@ func TestTimestampBuilder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestTime32StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dt := &arrow.Time32Type{Unit: arrow.Second}
+ b := array.NewTime32Builder(mem, dt)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Time32)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewTime32Builder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Time32)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewTime32Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -2028,6 +2461,43 @@ func TestTime32Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestTime64StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dt := &arrow.Time64Type{Unit: arrow.Microsecond}
+ b := array.NewTime64Builder(mem, dt)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Time64)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewTime64Builder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Time64)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewTime64Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -2198,6 +2668,42 @@ func TestTime64Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestDate32StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewDate32Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Date32)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDate32Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Date32)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewDate32Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -2364,6 +2870,49 @@ func TestDate32Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestDate64StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := array.NewDate64Builder(mem)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Date64)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDate64Builder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Date64)
+ defer arr1.Release()
+
+ assert.Exactly(t, arr.Len(), arr1.Len())
+ for i := 0; i < arr.Len(); i++ {
+ assert.Exactly(t, arr.IsValid(i), arr1.IsValid(i))
+ assert.Exactly(t, arr.ValueStr(i), arr1.ValueStr(i))
+ if arr.IsValid(i) {
+ assert.Exactly(t, arr.Value(i).ToTime(),
arr1.Value(i).ToTime())
+ }
+ }
+}
+
func TestNewDate64Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -2530,6 +3079,43 @@ func TestDate64Builder_Resize(t *testing.T) {
assert.Equal(t, 5, ab.Len())
}
+func TestDurationStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dt := &arrow.DurationType{Unit: arrow.Second}
+ b := array.NewDurationBuilder(mem, dt)
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.Duration)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDurationBuilder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Duration)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestNewDurationBuilder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
diff --git a/go/arrow/array/numericbuilder.gen_test.go.tmpl
b/go/arrow/array/numericbuilder.gen_test.go.tmpl
index 63245ef382..6b01684ac6 100644
--- a/go/arrow/array/numericbuilder.gen_test.go.tmpl
+++ b/go/arrow/array/numericbuilder.gen_test.go.tmpl
@@ -26,6 +26,66 @@ import (
)
{{range .In}}
+func Test{{.Name}}StringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+{{if .Opt.Parametric -}}
+{{ if or (eq .Name "Time64") -}}
+ dt := &arrow.{{.Name}}Type{Unit: arrow.Microsecond}
+{{else -}}
+ dt := &arrow.{{.Name}}Type{Unit: arrow.Second}
+{{end -}}
+ b := array.New{{.Name}}Builder(mem, dt)
+{{else -}}
+ b := array.New{{.Name}}Builder(mem)
+{{end -}}
+ defer b.Release()
+
+ b.Append(1)
+ b.Append(2)
+ b.Append(3)
+ b.AppendNull()
+ b.Append(5)
+ b.Append(6)
+ b.AppendNull()
+ b.Append(8)
+ b.Append(9)
+ b.Append(10)
+
+ arr := b.NewArray().(*array.{{.Name}})
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+{{if .Opt.Parametric -}}
+ b1 := array.New{{.Name}}Builder(mem, dt)
+{{else -}}
+ b1 := array.New{{.Name}}Builder(mem)
+{{end -}}
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.{{.Name}})
+ defer arr1.Release()
+
+{{ if or (eq .Name "Date64") -}}
+ assert.Exactly(t, arr.Len(), arr1.Len())
+ for i := 0; i < arr.Len(); i++ {
+ assert.Exactly(t, arr.IsValid(i), arr1.IsValid(i))
+ assert.Exactly(t, arr.ValueStr(i), arr1.ValueStr(i))
+ if arr.IsValid(i) {
+ assert.Exactly(t, arr.Value(i).ToTime(),
arr1.Value(i).ToTime())
+ }
+ }
+{{else -}}
+ assert.True(t, array.Equal(arr, arr1))
+{{end -}}
+}
+
func TestNew{{.Name}}Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -51,10 +111,9 @@ func TestNew{{.Name}}Builder(t *testing.T) {
ab.Append(8)
ab.Append(9)
ab.Append(10)
- ab.AppendValueFromString(11)
// check state of builder before New{{.Name}}Array
- assert.Equal(t, 11, ab.Len(), "unexpected Len()")
+ assert.Equal(t, 10, ab.Len(), "unexpected Len()")
assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
a := ab.New{{.Name}}Array()
@@ -66,7 +125,7 @@ func TestNew{{.Name}}Builder(t *testing.T) {
// check state of array
assert.Equal(t, 2, a.NullN(), "unexpected null count")
- assert.Equal(t, []{{or .QualifiedType .Type}}{1, 2, 3, 0, 5, 6, 0, 8,
9, 10, 11}, a.{{.Name}}Values(), "unexpected {{.Name}}Values")
+ assert.Equal(t, []{{or .QualifiedType .Type}}{1, 2, 3, 0, 5, 6, 0, 8,
9, 10}, a.{{.Name}}Values(), "unexpected {{.Name}}Values")
assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due
to minBuilderCapacity
assert.Len(t, a.{{.Name}}Values(), 10, "unexpected length of
{{.Name}}Values")
diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go
index 48fe032541..dacf66572e 100644
--- a/go/arrow/array/string.go
+++ b/go/arrow/array/string.go
@@ -53,12 +53,12 @@ func (a *String) Value(i int) string {
i = i + a.array.data.offset
return a.values[a.offsets[i]:a.offsets[i+1]]
}
+
func (a *String) ValueStr(i int) string {
if a.IsNull(i) {
- return "(null)"
- } else {
- return a.Value(i)
+ return NullValueStr
}
+ return a.Value(i)
}
// ValueOffset returns the offset of the value at index i.
@@ -100,7 +100,7 @@ func (a *String) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
@@ -195,7 +195,13 @@ func (a *LargeString) Value(i int) string {
i = i + a.array.data.offset
return a.values[a.offsets[i]:a.offsets[i+1]]
}
-func (a *LargeString) ValueStr(i int) string { return a.Value(i) }
+
+func (a *LargeString) ValueStr(i int) string {
+ if a.IsNull(i) {
+ return NullValueStr
+ }
+ return a.Value(i)
+}
// ValueOffset returns the offset of the value at index i.
func (a *LargeString) ValueOffset(i int) int64 {
@@ -236,7 +242,7 @@ func (a *LargeString) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(NullValueStr)
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
@@ -315,7 +321,9 @@ func NewStringBuilder(mem memory.Allocator) *StringBuilder {
return b
}
-func (b *StringBuilder) Type() arrow.DataType { return
arrow.BinaryTypes.String }
+func (b *StringBuilder) Type() arrow.DataType {
+ return arrow.BinaryTypes.String
+}
// Append appends a string to the builder.
func (b *StringBuilder) Append(v string) {
diff --git a/go/arrow/array/string_test.go b/go/arrow/array/string_test.go
index bfbe035807..9f27ceaa86 100644
--- a/go/arrow/array/string_test.go
+++ b/go/arrow/array/string_test.go
@@ -281,6 +281,38 @@ func TestStringInvalidOffsets(t *testing.T) {
}, "data has offset and value offset is overflowing")
}
+func TestStringStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ var (
+ values = []string{"hello", "世界", "", "bye"}
+ valid = []bool{true, true, false, true}
+ )
+
+ b := array.NewStringBuilder(mem)
+ defer b.Release()
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.String)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewStringBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.String)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestLargeStringArray(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
@@ -530,3 +562,35 @@ func TestLargeStringInvalidOffsets(t *testing.T) {
array.NewLargeStringData(array.NewData(arrow.BinaryTypes.LargeString, 1,
buffers, nil, 0, 2))
}, "data has offset and value offset is overflowing")
}
+
+func TestLargeStringStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ var (
+ values = []string{"hello", "世界", "", "bye"}
+ valid = []bool{true, true, false, true}
+ )
+
+ b := array.NewLargeStringBuilder(mem)
+ defer b.Release()
+
+ b.AppendValues(values, valid)
+
+ arr := b.NewArray().(*array.LargeString)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewLargeStringBuilder(mem)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.LargeString)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go
index a37b67f4c8..cb4fb5ec5e 100644
--- a/go/arrow/array/struct.go
+++ b/go/arrow/array/struct.go
@@ -83,12 +83,15 @@ func (a *Struct) Field(i int) arrow.Array { return
a.fields[i] }
// ValueStr returns the string representation (as json) of the value at index
i.
func (a *Struct) ValueStr(i int) string {
- var buf bytes.Buffer
- enc := json.NewEncoder(&buf)
- if err := enc.Encode(a.GetOneForMarshal(i)); err != nil {
+ if a.IsNull(i) {
+ return NullValueStr
+ }
+
+ data, err := json.Marshal(a.GetOneForMarshal(i))
+ if err != nil {
panic(err)
}
- return buf.String()
+ return string(data)
}
func (a *Struct) String() string {
@@ -118,7 +121,8 @@ func (a *Struct) String() string {
// newStructFieldWithParentValidityMask returns the Interface at fieldIndex
// with a nullBitmapBytes adjusted according on the parent struct
nullBitmapBytes.
// From the docs:
-// "When reading the struct array the parent validity bitmap takes priority."
+//
+// "When reading the struct array the parent validity bitmap takes
priority."
func (a *Struct) newStructFieldWithParentValidityMask(fieldIndex int)
arrow.Array {
field := a.Field(fieldIndex)
nullBitmapBytes := field.NullBitmapBytes()
@@ -362,8 +366,13 @@ func (b *StructBuilder) newData() (data *Data) {
}
func (b *StructBuilder) AppendValueFromString(s string) error {
+ if s == NullValueStr {
+ b.AppendNull()
+ return nil
+ }
+
if !strings.HasPrefix(s, "{") && !strings.HasSuffix(s, "}") {
- return fmt.Errorf("%w: invalid string for struct should be be
of form: {*}", arrow.ErrInvalid,)
+ return fmt.Errorf("%w: invalid string for struct should be be
of form: {*}", arrow.ErrInvalid)
}
dec := json.NewDecoder(strings.NewReader(s))
return b.UnmarshalOne(dec)
diff --git a/go/arrow/array/struct_test.go b/go/arrow/array/struct_test.go
index 9c12373170..30f5673f40 100644
--- a/go/arrow/array/struct_test.go
+++ b/go/arrow/array/struct_test.go
@@ -134,6 +134,46 @@ func TestStructArray(t *testing.T) {
}
}
+func TestStructStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dt := arrow.StructOf(
+ arrow.Field{Name: "nullable_bool", Type:
new(arrow.BooleanType), Nullable: true},
+ arrow.Field{Name: "non_nullable_bool", Type:
new(arrow.BooleanType)},
+ )
+
+ builder := array.NewStructBuilder(memory.DefaultAllocator, dt)
+ nullableBld := builder.FieldBuilder(0).(*array.BooleanBuilder)
+ nonNullableBld := builder.FieldBuilder(1).(*array.BooleanBuilder)
+
+ builder.Append(true)
+ nullableBld.Append(true)
+ nonNullableBld.Append(true)
+
+ builder.Append(true)
+ nullableBld.AppendNull()
+ nonNullableBld.Append(true)
+
+ builder.AppendNull()
+
+ arr := builder.NewArray().(*array.Struct)
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewStructBuilder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.Struct)
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
+
func TestStructArrayEmpty(t *testing.T) {
pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer pool.AssertSize(t, 0)
@@ -295,7 +335,7 @@ func TestStructArrayStringer(t *testing.T) {
arr := sb.NewArray().(*array.Struct)
defer arr.Release()
- assert.Equal(t, "{\"f1\":1.1,\"f2\":1}\n", arr.ValueStr(4))
+ assert.Equal(t, `{"f1":1.1,"f2":1}`, arr.ValueStr(4))
want := "{[1.1 (null) 1.3 1.4 1.1] [1 2 (null) 4 1]}"
got := arr.String()
if got != want {
diff --git a/go/arrow/array/union.go b/go/arrow/array/union.go
index 3181ccc6ba..c868ca2599 100644
--- a/go/arrow/array/union.go
+++ b/go/arrow/array/union.go
@@ -344,13 +344,23 @@ func (a *SparseUnion) MarshalJSON() ([]byte, error) {
}
func (a *SparseUnion) ValueStr(i int) string {
- var buf bytes.Buffer
- enc := json.NewEncoder(&buf)
- if err := enc.Encode(a.GetOneForMarshal(i)); err != nil {
+ if a.IsNull(i) {
+ return NullValueStr
+ }
+
+ val := a.GetOneForMarshal(i)
+ if val == nil {
+ // child is nil
+ return NullValueStr
+ }
+
+ data, err := json.Marshal(val)
+ if err != nil {
panic(err)
}
- return buf.String()
+ return string(data)
}
+
func (a *SparseUnion) String() string {
var b strings.Builder
b.WriteByte('[')
@@ -584,12 +594,12 @@ func (a *DenseUnion) GetOneForMarshal(i int) interface{} {
childID := a.ChildID(i)
data := a.Field(childID)
- offsets := a.RawValueOffsets()
- if data.IsNull(int(offsets[i])) {
+ offset := int(a.RawValueOffsets()[i])
+ if data.IsNull(offset) {
return nil
}
- return []interface{}{typeID, data.GetOneForMarshal(int(offsets[i]))}
+ return []interface{}{typeID, data.GetOneForMarshal(offset)}
}
func (a *DenseUnion) MarshalJSON() ([]byte, error) {
@@ -610,12 +620,21 @@ func (a *DenseUnion) MarshalJSON() ([]byte, error) {
}
func (a *DenseUnion) ValueStr(i int) string {
- var buf bytes.Buffer
- enc := json.NewEncoder(&buf)
- if err := enc.Encode(a.GetOneForMarshal(i)); err != nil {
+ if a.IsNull(i) {
+ return NullValueStr
+ }
+
+ val := a.GetOneForMarshal(i)
+ if val == nil {
+ // child in nil
+ return NullValueStr
+ }
+
+ data, err := json.Marshal(val)
+ if err != nil {
panic(err)
}
- return buf.String()
+ return string(data)
}
func (a *DenseUnion) String() string {
@@ -730,8 +749,8 @@ func newUnionBuilder(mem memory.Allocator, children
[]Builder, typ arrow.UnionTy
mode: typ.Mode(),
codes: typ.TypeCodes(),
children: children,
- typeIDtoChildID: make([]int, typ.MaxTypeCode()+1),
- typeIDtoBuilder: make([]Builder, typ.MaxTypeCode()+1),
+ typeIDtoChildID: make([]int, int(typ.MaxTypeCode())+1), //
convert to int as int8(127) +1 panics
+ typeIDtoBuilder: make([]Builder, int(typ.MaxTypeCode())+1), //
convert to int as int8(127) +1 panics
childFields: make([]arrow.Field, len(children)),
typesBuilder: newInt8BufferBuilder(mem),
}
@@ -1005,6 +1024,10 @@ func (b *SparseUnionBuilder) Unmarshal(dec
*json.Decoder) error {
}
func (b *SparseUnionBuilder) AppendValueFromString(s string) error {
+ if s == NullValueStr {
+ b.AppendNull()
+ return nil
+ }
dec := json.NewDecoder(strings.NewReader(s))
return b.UnmarshalOne(dec)
}
@@ -1110,10 +1133,15 @@ func NewEmptyDenseUnionBuilder(mem memory.Allocator)
*DenseUnionBuilder {
// children and type codes. Builders will be constructed for each child
// using the fields in typ
func NewDenseUnionBuilder(mem memory.Allocator, typ *arrow.DenseUnionType)
*DenseUnionBuilder {
- children := make([]Builder, len(typ.Fields()))
- for i, f := range typ.Fields() {
- children[i] = NewBuilder(mem, f.Type)
- defer children[i].Release()
+ children := make([]Builder, 0, len(typ.Fields()))
+ defer func() {
+ for _, child := range children {
+ child.Release()
+ }
+ }()
+
+ for _, f := range typ.Fields() {
+ children = append(children, NewBuilder(mem, f.Type))
}
return NewDenseUnionBuilderWithBuilders(mem, typ, children)
}
@@ -1255,6 +1283,10 @@ func (b *DenseUnionBuilder) Unmarshal(dec *json.Decoder)
error {
}
func (d *DenseUnionBuilder) AppendValueFromString(s string) error {
+ if s == NullValueStr {
+ d.AppendNull()
+ return nil
+ }
dec := json.NewDecoder(strings.NewReader(s))
return d.UnmarshalOne(dec)
}
diff --git a/go/arrow/array/union_test.go b/go/arrow/array/union_test.go
index 78129b8f05..5ce1f97ebe 100644
--- a/go/arrow/array/union_test.go
+++ b/go/arrow/array/union_test.go
@@ -373,12 +373,13 @@ func (s *UnionFactorySuite) TestMakeDenseUnions() {
children := make([]arrow.Array, 4)
children[0], _, _ = array.FromJSON(s.mem, arrow.BinaryTypes.String,
strings.NewReader(`["abc", "def", "xyz"]`))
+ defer children[0].Release()
children[1], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Uint8,
strings.NewReader(`[10, 20, 30]`))
+ defer children[1].Release()
children[2], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Float64,
strings.NewReader(`[1.618, 2.718, 3.142]`))
+ defer children[2].Release()
children[3], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Int8,
strings.NewReader(`[-12]`))
- for _, c := range children {
- defer c.Release()
- }
+ defer children[3].Release()
fieldNames := []string{"str", "int1", "real", "int2"}
@@ -458,6 +459,46 @@ func (s *UnionFactorySuite) TestMakeDenseUnions() {
})
}
+func (s *UnionFactorySuite) TestDenseUnionStringRoundTrip() {
+ // typeIDs: {0, 1, 2, 0, 1, 3, 2, 0, 2, 1}
+ offsets := s.offsetsFromSlice(0, 0, 0, 1, 1, 0, 1, 2, 1, 2)
+ defer offsets.Release()
+
+ children := make([]arrow.Array, 4)
+ children[0], _, _ = array.FromJSON(s.mem, arrow.BinaryTypes.String,
strings.NewReader(`["abc", "def", "xyz"]`))
+ defer children[0].Release()
+ children[1], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Uint8,
strings.NewReader(`[10, 20, 30]`))
+ defer children[1].Release()
+ children[2], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Float64,
strings.NewReader(`[1.618, 2.718, 3.142]`))
+ defer children[2].Release()
+ children[3], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Int8,
strings.NewReader(`[-12]`))
+ defer children[3].Release()
+
+ fields := []string{"str", "int1", "real", "int2"}
+
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(s.T(), 0)
+
+ dt := arrow.DenseUnionFromArrays(children, fields, s.codes)
+ arr, err :=
array.NewDenseUnionFromArraysWithFieldCodes(s.logicalTypeIDs, offsets,
children, fields, s.codes)
+ s.NoError(err)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewDenseUnionBuilder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ s.NoError(b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.DenseUnion)
+ defer arr1.Release()
+
+ s.True(array.Equal(arr, arr1))
+}
+
func (s *UnionFactorySuite) TestMakeSparse() {
children := make([]arrow.Array, 4)
children[0], _, _ = array.FromJSON(s.mem, arrow.BinaryTypes.String,
@@ -533,6 +574,47 @@ func (s *UnionFactorySuite) TestMakeSparse() {
})
}
+func (s *UnionFactorySuite) TestSparseUnionStringRoundTrip() {
+ children := make([]arrow.Array, 4)
+ children[0], _, _ = array.FromJSON(s.mem, arrow.BinaryTypes.String,
+ strings.NewReader(`["abc", "", "", "def", "", "", "", "xyz",
"", ""]`))
+ defer children[0].Release()
+ children[1], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Uint8,
+ strings.NewReader(`[0, 10, 0, 0, 20, 0, 0, 0, 0, 30]`))
+ defer children[1].Release()
+ children[2], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Float64,
+ strings.NewReader(`[0.0, 0.0, 1.618, 0.0, 0.0, 0.0, 2.718, 0.0,
3.142, 0.0]`))
+ defer children[2].Release()
+ children[3], _, _ = array.FromJSON(s.mem, arrow.PrimitiveTypes.Int8,
+ strings.NewReader(`[0, 0, 0, 0, 0, -12, 0, 0, 0, 0]`))
+ defer children[3].Release()
+
+ fields := []string{"str", "int1", "real", "int2"}
+
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(s.T(), 0)
+
+ dt := arrow.SparseUnionFromArrays(children, fields, s.codes)
+
+ arr, err :=
array.NewSparseUnionFromArraysWithFieldCodes(s.logicalTypeIDs, children,
fields, s.codes)
+ s.NoError(err)
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ b1 := array.NewSparseUnionBuilder(mem, dt)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ s.NoError(b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray().(*array.SparseUnion)
+ defer arr1.Release()
+
+ s.True(array.Equal(arr, arr1))
+}
+
type UnionBuilderSuite struct {
suite.Suite
diff --git a/go/arrow/csv/reader_test.go b/go/arrow/csv/reader_test.go
index 08e9f260db..4382ef6296 100644
--- a/go/arrow/csv/reader_test.go
+++ b/go/arrow/csv/reader_test.go
@@ -388,8 +388,8 @@ func testCSVReader(t *testing.T, filepath string,
withHeader bool, stringsCanBeN
str1Value := `""`
str2Value := `"null"`
if stringsCanBeNull {
- str1Value = "(null)"
- str2Value = "(null)"
+ str1Value = array.NullValueStr
+ str2Value = array.NullValueStr
}
want := fmt.Sprintf(`rec[0]["bool"]: [true]
diff --git a/go/arrow/example_test.go b/go/arrow/example_test.go
index 276aa510ff..ee23231319 100644
--- a/go/arrow/example_test.go
+++ b/go/arrow/example_test.go
@@ -53,7 +53,7 @@ func Example_minimal() {
for i, v := range ints.Int64Values() {
fmt.Printf("ints[%d] = ", i)
if ints.IsNull(i) {
- fmt.Println("(null)")
+ fmt.Println(array.NullValueStr)
} else {
fmt.Println(v)
}
@@ -97,7 +97,7 @@ func Example_fromMemory() {
for i := 0; i < n; i++ {
fmt.Printf("bools[%d] = ", i)
if bools.IsNull(i) {
- fmt.Println("(null)")
+ fmt.Println(array.NullValueStr)
} else {
fmt.Printf("%t\n", bools.Value(i))
}
@@ -640,7 +640,7 @@ func Example_mapArray() {
if itemArr.IsValid(int(j)) {
fmt.Printf("%v", itemArr.Value(int(j)))
} else {
- fmt.Printf("(null)")
+ fmt.Printf(array.NullValueStr)
}
}
fmt.Printf("}\n")
diff --git a/go/arrow/math/float64_arm64.go b/go/arrow/math/float64_arm64.go
index 759d967474..31ed189d5a 100644
--- a/go/arrow/math/float64_arm64.go
+++ b/go/arrow/math/float64_arm64.go
@@ -1,4 +1,4 @@
-// Code generated by type_s390x.go.tmpl. DO NOT EDIT.
+// Code generated by type_arm64.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,12 +16,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
func initFloat64NEON() {
- Float64.sum = sum_float64_neon
+ Float64.sum = sum_float64_neon
}
func initFloat64Go() {
diff --git a/go/arrow/math/float64_neon_arm64.go
b/go/arrow/math/float64_neon_arm64.go
index e31494ef94..5c09f28624 100755
--- a/go/arrow/math/float64_neon_arm64.go
+++ b/go/arrow/math/float64_neon_arm64.go
@@ -1,4 +1,4 @@
-// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+// Code generated by type_simd_arm64.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/float64_noasm.go b/go/arrow/math/float64_noasm.go
index b480ad34f3..bfa3e6e589 100644
--- a/go/arrow/math/float64_noasm.go
+++ b/go/arrow/math/float64_noasm.go
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build noasm
// +build noasm
package math
diff --git a/go/arrow/math/float64_ppc64le.go b/go/arrow/math/float64_ppc64le.go
index f60be90721..3d962005ce 100644
--- a/go/arrow/math/float64_ppc64le.go
+++ b/go/arrow/math/float64_ppc64le.go
@@ -1,4 +1,4 @@
-// Code generated by type_s390x.go.tmpl. DO NOT EDIT.
+// Code generated by type_ppc64le.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/float64_s390x.go b/go/arrow/math/float64_s390x.go
index f60be90721..cf091f53d1 100644
--- a/go/arrow/math/float64_s390x.go
+++ b/go/arrow/math/float64_s390x.go
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/int64_arm64.go b/go/arrow/math/int64_arm64.go
index 917902be0b..7bd200746e 100644
--- a/go/arrow/math/int64_arm64.go
+++ b/go/arrow/math/int64_arm64.go
@@ -1,4 +1,4 @@
-// Code generated by type_s390x.go.tmpl. DO NOT EDIT.
+// Code generated by type_arm64.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,12 +16,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
func initInt64NEON() {
- Int64.sum = sum_int64_neon
+ Int64.sum = sum_int64_neon
}
func initInt64Go() {
diff --git a/go/arrow/math/int64_neon_arm64.go
b/go/arrow/math/int64_neon_arm64.go
index 6639bf46f5..fab545aa1e 100755
--- a/go/arrow/math/int64_neon_arm64.go
+++ b/go/arrow/math/int64_neon_arm64.go
@@ -1,4 +1,4 @@
-// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+// Code generated by type_simd_arm64.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/int64_noasm.go b/go/arrow/math/int64_noasm.go
index 5a5efc31bb..09e945da75 100644
--- a/go/arrow/math/int64_noasm.go
+++ b/go/arrow/math/int64_noasm.go
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build noasm
// +build noasm
package math
diff --git a/go/arrow/math/int64_ppc64le.go b/go/arrow/math/int64_ppc64le.go
index 1a615a9b27..ff40824cc1 100644
--- a/go/arrow/math/int64_ppc64le.go
+++ b/go/arrow/math/int64_ppc64le.go
@@ -1,4 +1,4 @@
-// Code generated by type_s390x.go.tmpl. DO NOT EDIT.
+// Code generated by type_ppc64le.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/int64_s390x.go b/go/arrow/math/int64_s390x.go
index 1a615a9b27..d32d860ca0 100644
--- a/go/arrow/math/int64_s390x.go
+++ b/go/arrow/math/int64_s390x.go
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/uint64_arm64.go b/go/arrow/math/uint64_arm64.go
index df87289dff..54981c2278 100644
--- a/go/arrow/math/uint64_arm64.go
+++ b/go/arrow/math/uint64_arm64.go
@@ -1,4 +1,4 @@
-// Code generated by type_s390x.go.tmpl. DO NOT EDIT.
+// Code generated by type_arm64.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,12 +16,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
func initUint64NEON() {
- Uint64.sum = sum_uint64_neon
+ Uint64.sum = sum_uint64_neon
}
func initUint64Go() {
diff --git a/go/arrow/math/uint64_neon_arm64.go
b/go/arrow/math/uint64_neon_arm64.go
index 867ef74522..7bfa0aba0b 100755
--- a/go/arrow/math/uint64_neon_arm64.go
+++ b/go/arrow/math/uint64_neon_arm64.go
@@ -1,4 +1,4 @@
-// Code generated by type_simd_amd64.go.tmpl. DO NOT EDIT.
+// Code generated by type_simd_arm64.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/uint64_noasm.go b/go/arrow/math/uint64_noasm.go
index b7174b3d74..c40ecc4678 100644
--- a/go/arrow/math/uint64_noasm.go
+++ b/go/arrow/math/uint64_noasm.go
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build noasm
// +build noasm
package math
diff --git a/go/arrow/math/uint64_ppc64le.go b/go/arrow/math/uint64_ppc64le.go
index 8f7419fd48..b6b1c536af 100644
--- a/go/arrow/math/uint64_ppc64le.go
+++ b/go/arrow/math/uint64_ppc64le.go
@@ -1,4 +1,4 @@
-// Code generated by type_s390x.go.tmpl. DO NOT EDIT.
+// Code generated by type_ppc64le.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/arrow/math/uint64_s390x.go b/go/arrow/math/uint64_s390x.go
index 8f7419fd48..d6e7da7cf3 100644
--- a/go/arrow/math/uint64_s390x.go
+++ b/go/arrow/math/uint64_s390x.go
@@ -16,6 +16,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+//go:build !noasm
// +build !noasm
package math
diff --git a/go/internal/types/extension_types.go
b/go/internal/types/extension_types.go
index 53de27b8f6..ee3b2ddd9f 100644
--- a/go/internal/types/extension_types.go
+++ b/go/internal/types/extension_types.go
@@ -32,26 +32,50 @@ import (
"golang.org/x/xerrors"
)
+var UUID = NewUUIDType()
+
type UUIDBuilder struct {
*array.ExtensionBuilder
}
-func NewUUIDBuilder(bldr *array.ExtensionBuilder) *UUIDBuilder {
- b := &UUIDBuilder{
- ExtensionBuilder: bldr,
- }
- return b
+func NewUUIDBuilder(builder *array.ExtensionBuilder) *UUIDBuilder {
+ return &UUIDBuilder{ExtensionBuilder: builder}
}
func (b *UUIDBuilder) Append(v uuid.UUID) {
b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).Append(v[:])
}
+func (b *UUIDBuilder) UnsafeAppend(v uuid.UUID) {
+
b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).UnsafeAppend(v[:])
+}
+
+func (b *UUIDBuilder) AppendValueFromString(s string) error {
+ if s == array.NullValueStr {
+ b.AppendNull()
+ return nil
+ }
+
+ uid, err := uuid.Parse(s)
+ if err != nil {
+ return err
+ }
+
+ b.Append(uid)
+ return nil
+}
+
func (b *UUIDBuilder) AppendValues(v []uuid.UUID, valid []bool) {
+ if len(v) != len(valid) && len(valid) != 0 {
+ panic("len(v) != len(valid) && len(valid) != 0")
+ }
+
data := make([][]byte, len(v))
- for i, u := range v {
- data[i] = make([]byte, 16)
- copy(data[i][:], u[:])
+ for i := range v {
+ if len(valid) > 0 && !valid[i] {
+ continue
+ }
+ data[i] = v[i][:]
}
b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).AppendValues(data,
valid)
}
@@ -65,11 +89,15 @@ func (b *UUIDBuilder) UnmarshalOne(dec *json.Decoder) error
{
var val uuid.UUID
switch v := t.(type) {
case string:
- data, err := uuid.Parse(v)
+ val, err = uuid.Parse(v)
+ if err != nil {
+ return err
+ }
+ case []byte:
+ val, err = uuid.ParseBytes(v)
if err != nil {
return err
}
- val = data
case nil:
b.AppendNull()
return nil
@@ -82,14 +110,6 @@ func (b *UUIDBuilder) UnmarshalOne(dec *json.Decoder) error
{
}
}
- if len(val) != 16 {
- return &json.UnmarshalTypeError{
- Value: fmt.Sprint(val),
- Type: reflect.TypeOf([]byte{}),
- Offset: dec.InputOffset(),
- Struct: fmt.Sprintf("FixedSizeBinary[%d]", 16),
- }
- }
b.Append(val)
return nil
}
@@ -111,43 +131,18 @@ func (b *UUIDBuilder) UnmarshalJSON(data []byte) error {
}
if delim, ok := t.(json.Delim); !ok || delim != '[' {
- return fmt.Errorf("fixed size binary builder must unpack from
json array, found %s", delim)
+ return fmt.Errorf("uuid builder must unpack from json array,
found %s", delim)
}
return b.Unmarshal(dec)
}
-func (b *UUIDBuilder) AppendValueFromString(s string) error {
- if s == array.NullValueStr {
- b.AppendNull()
- return nil
- }
- u, err := uuid.Parse(s)
- if err != nil {
- return fmt.Errorf("%w: invalid uuid: %v", arrow.ErrInvalid, err)
- }
- b.Append(u)
- return nil
-}
-
// UUIDArray is a simple array which is a FixedSizeBinary(16)
type UUIDArray struct {
array.ExtensionArrayBase
}
-func (a UUIDArray) ValueStr(i int) string {
- if a.IsNull(i) {
- return "(null)"
- }
- arr := a.Storage().(*array.FixedSizeBinary)
- uuidStr, err := uuid.FromBytes(arr.Value(i))
- if err != nil {
- panic(fmt.Errorf("invalid uuid: %w", err))
- }
- return uuidStr.String()
-}
-
-func (a UUIDArray) String() string {
+func (a *UUIDArray) String() string {
arr := a.Storage().(*array.FixedSizeBinary)
o := new(strings.Builder)
o.WriteString("[")
@@ -157,46 +152,47 @@ func (a UUIDArray) String() string {
}
switch {
case a.IsNull(i):
- o.WriteString("(null)")
+ o.WriteString(array.NullValueStr)
default:
- uuidStr, err := uuid.FromBytes(arr.Value(i))
- if err != nil {
- panic(fmt.Errorf("invalid uuid: %w", err))
- }
- fmt.Fprintf(o, "%q", uuidStr)
+ fmt.Fprintf(o, "%q", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
+func (a *UUIDArray) Value(i int) uuid.UUID {
+ if a.IsNull(i) {
+ return uuid.Nil
+ }
+ return
uuid.Must(uuid.FromBytes(a.Storage().(*array.FixedSizeBinary).Value(i)))
+}
+
+func (a *UUIDArray) ValueStr(i int) string {
+ switch {
+ case a.IsNull(i):
+ return array.NullValueStr
+ default:
+ return a.Value(i).String()
+ }
+}
+
func (a *UUIDArray) MarshalJSON() ([]byte, error) {
arr := a.Storage().(*array.FixedSizeBinary)
- vals := make([]interface{}, a.Len())
+ values := make([]interface{}, a.Len())
for i := 0; i < a.Len(); i++ {
if a.IsValid(i) {
- uuidStr, err := uuid.FromBytes(arr.Value(i))
- if err != nil {
- panic(fmt.Errorf("invalid uuid: %w", err))
- }
- vals[i] = uuidStr
- } else {
- vals[i] = nil
+ values[i] =
uuid.Must(uuid.FromBytes(arr.Value(i))).String()
}
}
- return json.Marshal(vals)
+ return json.Marshal(values)
}
func (a *UUIDArray) GetOneForMarshal(i int) interface{} {
- arr := a.Storage().(*array.FixedSizeBinary)
- if a.IsValid(i) {
- uuidObj, err := uuid.FromBytes(arr.Value(i))
- if err != nil {
- panic(fmt.Errorf("invalid uuid: %w", err))
- }
- return uuidObj
+ if a.IsNull(i) {
+ return nil
}
- return nil
+ return a.Value(i)
}
// UUIDType is a simple extension type that represents a FixedSizeBinary(16)
@@ -205,27 +201,39 @@ type UUIDType struct {
arrow.ExtensionBase
}
-// NewUUIDType is a convenience function to create an instance of UuidType
+// NewUUIDType is a convenience function to create an instance of UUIDType
// with the correct storage type
func NewUUIDType() *UUIDType {
- return &UUIDType{
- ExtensionBase: arrow.ExtensionBase{
- Storage: &arrow.FixedSizeBinaryType{ByteWidth: 16}}}
+ return &UUIDType{ExtensionBase: arrow.ExtensionBase{Storage:
&arrow.FixedSizeBinaryType{ByteWidth: 16}}}
}
-// ArrayType returns TypeOf(UuidArray) for constructing uuid arrays
-func (UUIDType) ArrayType() reflect.Type { return reflect.TypeOf(UUIDArray{}) }
+// ArrayType returns TypeOf(UUIDArray{}) for constructing UUID arrays
+func (*UUIDType) ArrayType() reflect.Type {
+ return reflect.TypeOf(UUIDArray{})
+}
+
+func (*UUIDType) ExtensionName() string {
+ return "uuid"
+}
+
+func (e *UUIDType) String() string {
+ return fmt.Sprintf("extension_type<storage=%s>", e.Storage)
+}
-func (UUIDType) ExtensionName() string { return "uuid" }
+func (e *UUIDType) MarshalJSON() ([]byte, error) {
+ return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`,
e.ExtensionName(), e.Serialize())), nil
+}
// Serialize returns "uuid-serialized" for testing proper metadata passing
-func (UUIDType) Serialize() string { return "uuid-serialized" }
+func (*UUIDType) Serialize() string {
+ return "uuid-serialized"
+}
// Deserialize expects storageType to be FixedSizeBinaryType{ByteWidth: 16}
and the data to be
// "uuid-serialized" in order to correctly create a UUIDType for testing
deserialize.
-func (UUIDType) Deserialize(storageType arrow.DataType, data string)
(arrow.ExtensionType, error) {
- if string(data) != "uuid-serialized" {
- return nil, fmt.Errorf("type identifier did not match: '%s'",
string(data))
+func (*UUIDType) Deserialize(storageType arrow.DataType, data string)
(arrow.ExtensionType, error) {
+ if data != "uuid-serialized" {
+ return nil, fmt.Errorf("type identifier did not match: '%s'",
data)
}
if !arrow.TypeEqual(storageType, &arrow.FixedSizeBinaryType{ByteWidth:
16}) {
return nil, fmt.Errorf("invalid storage type for UUIDType: %s",
storageType.Name())
@@ -233,12 +241,12 @@ func (UUIDType) Deserialize(storageType arrow.DataType,
data string) (arrow.Exte
return NewUUIDType(), nil
}
-// UuidTypes are equal if both are named "uuid"
-func (u UUIDType) ExtensionEquals(other arrow.ExtensionType) bool {
- return u.ExtensionName() == other.ExtensionName()
+// ExtensionEquals returns true if both extensions have the same name
+func (e *UUIDType) ExtensionEquals(other arrow.ExtensionType) bool {
+ return e.ExtensionName() == other.ExtensionName()
}
-func (u UUIDType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder {
+func (*UUIDType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder {
return NewUUIDBuilder(bldr)
}
@@ -251,7 +259,7 @@ type Parametric1Array struct {
func (a Parametric1Array) ValueStr(i int) string {
arr := a.Storage().(*array.Int32)
if a.IsNull(i) {
- return "(null)"
+ return array.NullValueStr
}
return fmt.Sprintf("%d", arr.Value(i))
}
@@ -266,7 +274,7 @@ type Parametric2Array struct {
func (a Parametric2Array) ValueStr(i int) string {
arr := a.Storage().(*array.Int32)
if a.IsNull(i) {
- return "(null)"
+ return array.NullValueStr
}
return fmt.Sprintf("%d", arr.Value(i))
}
@@ -384,7 +392,7 @@ type ExtStructArray struct {
func (a ExtStructArray) ValueStr(i int) string {
arr := a.Storage().(*array.Struct)
if a.IsNull(i) {
- return "(null)"
+ return array.NullValueStr
}
b, err := arr.MarshalJSON()
if err != nil {
@@ -441,7 +449,7 @@ type DictExtensionArray struct {
func (a DictExtensionArray) ValueStr(i int) string {
arr := a.Storage().(*array.Dictionary)
if a.IsNull(i) {
- return "(null)"
+ return array.NullValueStr
}
b, err := arr.MarshalJSON()
if err != nil {
@@ -491,7 +499,7 @@ type SmallintArray struct {
func (a SmallintArray) ValueStr(i int) string {
if a.IsNull(i) {
- return "(null)"
+ return array.NullValueStr
}
arr := a.Storage().(*array.Int16)
return fmt.Sprintf("%d", arr.Value(i))
diff --git a/go/internal/types/extension_types_test.go
b/go/internal/types/extension_types_test.go
index 0ece6eb49e..51764b0e8f 100644
--- a/go/internal/types/extension_types_test.go
+++ b/go/internal/types/extension_types_test.go
@@ -66,3 +66,36 @@ func TestExtensionRecordBuilder(t *testing.T) {
require.NoError(t, err)
require.Equal(t, record, record1)
}
+
+func TestUUIDStringRoundTrip(t *testing.T) {
+ // 1. create array
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ extBuilder := array.NewExtensionBuilder(mem, types.NewUUIDType())
+ defer extBuilder.Release()
+ b := types.NewUUIDBuilder(extBuilder)
+ b.Append(uuid.Nil)
+ b.AppendNull()
+ b.Append(uuid.NameSpaceURL)
+ b.AppendNull()
+ b.Append(testUUID)
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ // 2. create array via AppendValueFromString
+ extBuilder1 := array.NewExtensionBuilder(mem, types.NewUUIDType())
+ defer extBuilder1.Release()
+ b1 := types.NewUUIDBuilder(extBuilder1)
+ defer b1.Release()
+
+ for i := 0; i < arr.Len(); i++ {
+ assert.NoError(t, b1.AppendValueFromString(arr.ValueStr(i)))
+ }
+
+ arr1 := b1.NewArray()
+ defer arr1.Release()
+
+ assert.True(t, array.Equal(arr, arr1))
+}
