This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 2900ed6 fix(arrow/array): update timestamp json format (#450)
2900ed6 is described below
commit 2900ed6f8ae27398d825b2fde2f67d77dcf780b4
Author: Matt Topol <[email protected]>
AuthorDate: Sun Jul 27 16:42:20 2025 -0400
fix(arrow/array): update timestamp json format (#450)
### Rationale for this change
resolves #445
### What changes are included in this PR?
Use `time.RFC3339Nano` for the format to output the time value
### Are these changes tested?
Yes
### Are there any user-facing changes?
Changes the format of timestamp strings for `ValueStr` and marshalling
to JSON to match JSON and Go conventions.
---
arrow/array/timestamp.go | 2 +-
arrow/array/timestamp_test.go | 56 +++++++++++++++++++++++++++++++++++------
arrow/avro/testdata/testdata.go | 8 +++---
3 files changed, 53 insertions(+), 13 deletions(-)
diff --git a/arrow/array/timestamp.go b/arrow/array/timestamp.go
index ff9ff4b..3f11c68 100644
--- a/arrow/array/timestamp.go
+++ b/arrow/array/timestamp.go
@@ -93,7 +93,7 @@ func (a *Timestamp) ValueStr(i int) string {
}
toTime, _ := a.DataType().(*arrow.TimestampType).GetToTimeFunc()
- return toTime(a.values[i]).Format("2006-01-02 15:04:05.999999999Z0700")
+ return toTime(a.values[i]).Format(time.RFC3339Nano)
}
func (a *Timestamp) GetOneForMarshal(i int) interface{} {
diff --git a/arrow/array/timestamp_test.go b/arrow/array/timestamp_test.go
index 2dba3cf..f0dd838 100644
--- a/arrow/array/timestamp_test.go
+++ b/arrow/array/timestamp_test.go
@@ -23,7 +23,9 @@ import (
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/arrow-go/v18/internal/json"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestTimestampStringRoundTrip(t *testing.T) {
@@ -248,8 +250,8 @@ func TestTimestampValueStr(t *testing.T) {
arr := b.NewArray()
defer arr.Release()
- assert.Equal(t, "1968-11-30 13:30:45-0700", arr.ValueStr(0))
- assert.Equal(t, "2016-02-29 10:42:23-0700", arr.ValueStr(1))
+ assert.Equal(t, "1968-11-30T13:30:45-07:00", arr.ValueStr(0))
+ assert.Equal(t, "2016-02-29T10:42:23-07:00", arr.ValueStr(1))
}
func TestTimestampEquality(t *testing.T) {
@@ -278,16 +280,16 @@ func TestTimestampEquality(t *testing.T) {
// No timezone, "wall clock" semantics
// These timestamps have no actual timezone, but we still represent as
UTC per Go conventions
- assert.Equal(t, "1968-11-30 20:30:45Z", arrs[0].ValueStr(0))
- assert.Equal(t, "2016-02-29 17:42:23Z", arrs[0].ValueStr(1))
+ assert.Equal(t, "1968-11-30T20:30:45Z", arrs[0].ValueStr(0))
+ assert.Equal(t, "2016-02-29T17:42:23Z", arrs[0].ValueStr(1))
// UTC timezone, "instant" semantics
- assert.Equal(t, "1968-11-30 20:30:45Z", arrs[1].ValueStr(0))
- assert.Equal(t, "2016-02-29 17:42:23Z", arrs[1].ValueStr(1))
+ assert.Equal(t, "1968-11-30T20:30:45Z", arrs[1].ValueStr(0))
+ assert.Equal(t, "2016-02-29T17:42:23Z", arrs[1].ValueStr(1))
// America/Phoenix timezone, "instant" semantics
- assert.Equal(t, "1968-11-30 13:30:45-0700", arrs[2].ValueStr(0))
- assert.Equal(t, "2016-02-29 10:42:23-0700", arrs[2].ValueStr(1))
+ assert.Equal(t, "1968-11-30T13:30:45-07:00", arrs[2].ValueStr(0))
+ assert.Equal(t, "2016-02-29T10:42:23-07:00", arrs[2].ValueStr(1))
// Despite timezone and semantics, the physical values are equivalent
assert.Equal(t, arrs[0].Value(0), arrs[1].Value(0))
@@ -298,3 +300,41 @@ func TestTimestampEquality(t *testing.T) {
assert.Equal(t, arrs[0].Value(1), arrs[2].Value(1))
assert.Equal(t, arrs[1].Value(1), arrs[2].Value(1))
}
+
+func TestTimestampArrayJSONRoundTrip(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ tz, _ := time.LoadLocation("America/Phoenix")
+ dt := &arrow.TimestampType{Unit: arrow.Second, TimeZone: tz.String()}
+ b := array.NewTimestampBuilder(mem, dt)
+ defer b.Release()
+
+ b.Append(-34226955)
+ b.Append(1456767743)
+
+ arr := b.NewArray()
+ defer arr.Release()
+
+ assert.Equal(t, "1968-11-30T13:30:45-07:00", arr.ValueStr(0))
+ assert.Equal(t, "2016-02-29T10:42:23-07:00", arr.ValueStr(1))
+ assert.Equal(t, 2, arr.Len())
+ assert.Equal(t, 0, arr.NullN())
+
+ json_bytes, err := arr.MarshalJSON()
+ require.NoError(t, err)
+
+ expectedJSON :=
`["1968-11-30T13:30:45-07:00","2016-02-29T10:42:23-07:00"]`
+ require.Equal(t, expectedJSON, string(json_bytes))
+
+ var timestamps []time.Time
+ err = json.Unmarshal(json_bytes, ×tamps)
+ require.NoError(t, err)
+ require.Len(t, timestamps, 2)
+
+ expectedTime1 := time.Date(1968, time.November, 30, 13, 30, 45, 0, tz)
+ expectedTime2 := time.Date(2016, time.February, 29, 10, 42, 23, 0, tz)
+
+ assert.Equal(t, expectedTime1.Unix(), timestamps[0].Unix())
+ assert.Equal(t, expectedTime2.Unix(), timestamps[1].Unix())
+}
diff --git a/arrow/avro/testdata/testdata.go b/arrow/avro/testdata/testdata.go
index 926c054..cb9c37e 100644
--- a/arrow/avro/testdata/testdata.go
+++ b/arrow/avro/testdata/testdata.go
@@ -50,16 +50,16 @@ func (b ByteArray) MarshalJSON() ([]byte, error) {
type TimestampMicros int64
func (t TimestampMicros) MarshalJSON() ([]byte, error) {
- ts := time.Unix(0,
int64(t)*int64(time.Microsecond)).UTC().Format("2006-01-02 15:04:05.000000")
+ ts := time.Unix(0,
int64(t)*int64(time.Microsecond)).UTC().Format(time.RFC3339Nano)
// arrow record marshaller trims trailing zero digits from timestamp so
we do the same
- return json.Marshal(fmt.Sprintf("%sZ", strings.TrimRight(ts, "0.")))
+ return json.Marshal(ts)
}
type TimestampMillis int64
func (t TimestampMillis) MarshalJSON() ([]byte, error) {
- ts := time.Unix(0,
int64(t)*int64(time.Millisecond)).UTC().Format("2006-01-02 15:04:05.000")
- return json.Marshal(fmt.Sprintf("%sZ", strings.TrimRight(ts, "0.")))
+ ts := time.Unix(0,
int64(t)*int64(time.Millisecond)).UTC().Format(time.RFC3339Nano)
+ return json.Marshal(ts)
}
type TimeMillis time.Duration
