This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 8c72476d fix(parquet/pqarrow): preserve INT96 timestamps across units
(#1006)
8c72476d is described below
commit 8c72476df63df6d3d29b938ca528e1526985370b
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 20:57:18 2026 +0200
fix(parquet/pqarrow): preserve INT96 timestamps across units (#1006)
### Rationale for this change
INT96 timestamp conversion currently routes values through
`time.Duration` and applies a nanoseconds-per-day modulus directly to
the source value. This produces incorrect results for second,
millisecond, and microsecond timestamps, overflows for timestamps
outside the duration range, and leaves negative timestamps with a
negative time-of-day remainder.
### What changes are included in this PR?
- Convert each Arrow timestamp unit using integer units-per-day and
nanoseconds-per-unit constants.
- Normalize negative remainders onto the preceding Julian day.
- Add focused coverage for every timestamp unit on both sides of the
Unix epoch.
### Are these changes tested?
Yes. The focused pqarrow tests pass.
---
parquet/pqarrow/encode_arrow.go | 24 ++++++++------
parquet/pqarrow/encode_arrow_int96_test.go | 51 ++++++++++++++++++++++++++++++
2 files changed, 66 insertions(+), 9 deletions(-)
diff --git a/parquet/pqarrow/encode_arrow.go b/parquet/pqarrow/encode_arrow.go
index 9ec11ff8..c7f089c2 100644
--- a/parquet/pqarrow/encode_arrow.go
+++ b/parquet/pqarrow/encode_arrow.go
@@ -24,7 +24,6 @@ import (
"errors"
"fmt"
"math"
- "time"
"unsafe"
"github.com/apache/arrow-go/v18/arrow"
@@ -712,20 +711,27 @@ const (
)
func arrowTimestampToImpalaTimestamp(unit arrow.TimeUnit, t int64, out
*parquet.Int96) {
- var d time.Duration
+ var unitsPerDay, nanosPerUnit int64
switch unit {
case arrow.Second:
- d = time.Duration(t) * time.Second
- case arrow.Microsecond:
- d = time.Duration(t) * time.Microsecond
+ unitsPerDay, nanosPerUnit = 24*60*60, 1000*1000*1000
case arrow.Millisecond:
- d = time.Duration(t) * time.Millisecond
+ unitsPerDay, nanosPerUnit = 24*60*60*1000, 1000*1000
+ case arrow.Microsecond:
+ unitsPerDay, nanosPerUnit = 24*60*60*1000*1000, 1000
case arrow.Nanosecond:
- d = time.Duration(t) * time.Nanosecond
+ unitsPerDay, nanosPerUnit = nanoSecondsPerDay, 1
+ }
+
+ days := t / unitsPerDay
+ remainder := t % unitsPerDay
+ if remainder < 0 {
+ days--
+ remainder += unitsPerDay
}
- julianDays := (int64(d.Hours()) / 24) + julianEpochOffsetDays
- lastDayNanos := t % (nanoSecondsPerDay)
+ julianDays := days + julianEpochOffsetDays
+ lastDayNanos := remainder * nanosPerUnit
binary.LittleEndian.PutUint64((*out)[:8], uint64(lastDayNanos))
binary.LittleEndian.PutUint32((*out)[8:], uint32(julianDays))
}
diff --git a/parquet/pqarrow/encode_arrow_int96_test.go
b/parquet/pqarrow/encode_arrow_int96_test.go
new file mode 100644
index 00000000..4e5cbf2d
--- /dev/null
+++ b/parquet/pqarrow/encode_arrow_int96_test.go
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pqarrow
+
+import (
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/parquet"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestArrowTimestampToImpalaTimestamp(t *testing.T) {
+ tests := []struct {
+ name string
+ unit arrow.TimeUnit
+ value arrow.Timestamp
+ }{
+ {"seconds after epoch", arrow.Second, 946_684_801},
+ {"seconds before epoch", arrow.Second, -1},
+ {"milliseconds after epoch", arrow.Millisecond,
946_684_800_001},
+ {"milliseconds before epoch", arrow.Millisecond, -1},
+ {"microseconds after epoch", arrow.Microsecond,
946_684_800_000_001},
+ {"microseconds before epoch", arrow.Microsecond, -1},
+ {"nanoseconds after epoch", arrow.Nanosecond,
946_684_800_000_000_001},
+ {"nanoseconds before epoch", arrow.Nanosecond, -1},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var got parquet.Int96
+ arrowTimestampToImpalaTimestamp(tt.unit,
int64(tt.value), &got)
+
+ assert.Equal(t, tt.value.ToTime(tt.unit), got.ToTime())
+ })
+ }
+}
