This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 904078c3f perf(transforms): speed up year and month partition
transforms (#2263)
904078c3f is described below
commit 904078c3f6810c9ff5f845a4cccd12137c17ebe0
Author: Minh Vu <[email protected]>
AuthorDate: Thu Oct 22 19:58:24 2026 +0200
perf(transforms): speed up year and month partition transforms (#2263)
* perf(transforms): calculate year and month partitions from Unix days
Signed-off-by: Minh Vu <[email protected]>
* test(transforms): cover calendar parity and benchmark reference path
Signed-off-by: Minh Vu <[email protected]>
* style(transforms): keep YearTransform doc next to type
Signed-off-by: Minh Vu <[email protected]>
* bench(transforms): compare inline reference converters
Signed-off-by: Minh Vu <[email protected]>
* lint(transforms): add blank lines before returns
Signed-off-by: Minh Vu <[email protected]>
* lint(transforms): space calendar reference returns
Signed-off-by: Minh Vu <[email protected]>
* style(transforms): fix calendar fast-path benchmark lint
---------
Signed-off-by: Minh Vu <[email protected]>
---
transforms.go | 82 +++++++---
transforms_calendar_fastpath_test.go | 289 +++++++++++++++++++++++++++++++++++
2 files changed, 347 insertions(+), 24 deletions(-)
diff --git a/transforms.go b/transforms.go
index 92895949f..84e8b4fc3 100644
--- a/transforms.go
+++ b/transforms.go
@@ -988,6 +988,44 @@ func projectTimeTransform(t TimeTransform, name string,
pred BoundPredicate) (Un
return nil, nil
}
+// yearMonthFromUnixDays converts days since 1970-01-01 to a Gregorian year
+// and month without constructing a time.Time. The 400-year era calculation
+// handles dates before the Unix epoch and all representable Iceberg dates.
+func yearMonthFromUnixDays(days int64) (year, month int) {
+ z := days + 719468
+ era := z / 146097
+ if z < 0 {
+ era = (z - 146096) / 146097
+ }
+
+ dayOfEra := z - era*146097
+ yearOfEra := (dayOfEra - dayOfEra/1460 + dayOfEra/36524 -
dayOfEra/146096) / 365
+ y := yearOfEra + era*400
+ dayOfYear := dayOfEra - (365*yearOfEra + yearOfEra/4 - yearOfEra/100)
+ monthIndex := (5*dayOfYear + 2) / 153
+ m := monthIndex + 3
+ if monthIndex >= 10 {
+ m = monthIndex - 9
+ }
+ if m <= 2 {
+ y++
+ }
+
+ return int(y), int(m)
+}
+
+func yearPartitionFromDays(days int64) int32 {
+ year, _ := yearMonthFromUnixDays(days)
+
+ return int32(year - 1970)
+}
+
+func monthPartitionFromDays(days int64) int32 {
+ year, month := yearMonthFromUnixDays(days)
+
+ return int32((year-1970)*12 + month - 1)
+}
+
// YearTransform transforms a datetime value into a year value.
type YearTransform struct{}
@@ -1022,7 +1060,7 @@ func (YearTransform) Transformer(src Type) (func(any)
Optional[int32], error) {
return Optional[int32]{
Valid: true,
- Val: int32(v.(Date).ToTime().Year() -
epochTM.Year()),
+ Val: yearPartitionFromDays(int64(v.(Date))),
}
}, nil
case TimestampType, TimestampTzType:
@@ -1033,7 +1071,7 @@ func (YearTransform) Transformer(src Type) (func(any)
Optional[int32], error) {
return Optional[int32]{
Valid: true,
- Val: int32(v.(Timestamp).ToTime().Year() -
epochTM.Year()),
+ Val:
yearPartitionFromDays(int64(v.(Timestamp).ToDate())),
}
}, nil
case TimestampNsType, TimestampTzNsType:
@@ -1044,7 +1082,7 @@ func (YearTransform) Transformer(src Type) (func(any)
Optional[int32], error) {
return Optional[int32]{
Valid: true,
- Val: int32(v.(TimestampNano).ToTime().Year()
- epochTM.Year()),
+ Val:
yearPartitionFromDays(int64(v.(TimestampNano).ToDate())),
}
}, nil
}
@@ -1058,18 +1096,21 @@ func (YearTransform) Apply(value Optional[Literal])
(out Optional[Literal]) {
return out
}
+ var days int64
switch v := value.Val.(type) {
case DateLiteral:
- out.Valid = true
- out.Val = Int32Literal(Date(v).ToTime().Year() - epochTM.Year())
+ days = int64(v)
case TimestampLiteral:
- out.Valid = true
- out.Val = Int32Literal(Timestamp(v).ToTime().Year() -
epochTM.Year())
+ days = int64(Timestamp(v).ToDate())
case TimestampNsLiteral:
- out.Valid = true
- out.Val = Int32Literal(TimestampNano(v).ToTime().Year() -
epochTM.Year())
+ days = int64(TimestampNano(v).ToDate())
+ default:
+ return out
}
+ out.Valid = true
+ out.Val = Int32Literal(yearPartitionFromDays(days))
+
return out
}
@@ -1122,11 +1163,9 @@ func (MonthTransform) Transformer(src Type) (func(any)
Optional[int32], error) {
return Optional[int32]{}
}
- d := v.(Date).ToTime()
-
return Optional[int32]{
Valid: true,
- Val: int32((d.Year()-epochTM.Year())*12 +
(int(d.Month()) - int(epochTM.Month()))),
+ Val: monthPartitionFromDays(int64(v.(Date))),
}
}, nil
case TimestampType, TimestampTzType:
@@ -1135,11 +1174,9 @@ func (MonthTransform) Transformer(src Type) (func(any)
Optional[int32], error) {
return Optional[int32]{}
}
- d := v.(Timestamp).ToTime()
-
return Optional[int32]{
Valid: true,
- Val: int32((d.Year()-epochTM.Year())*12 +
(int(d.Month()) - int(epochTM.Month()))),
+ Val:
monthPartitionFromDays(int64(v.(Timestamp).ToDate())),
}
}, nil
case TimestampNsType, TimestampTzNsType:
@@ -1148,14 +1185,11 @@ func (MonthTransform) Transformer(src Type) (func(any)
Optional[int32], error) {
return Optional[int32]{}
}
- d := v.(TimestampNano).ToTime()
-
return Optional[int32]{
Valid: true,
- Val: int32((d.Year()-epochTM.Year())*12 +
(int(d.Month()) - int(epochTM.Month()))),
+ Val:
monthPartitionFromDays(int64(v.(TimestampNano).ToDate())),
}
}, nil
-
}
return nil, fmt.Errorf("%w: cannot apply month transform for type %s",
@@ -1167,20 +1201,20 @@ func (MonthTransform) Apply(value Optional[Literal])
(out Optional[Literal]) {
return out
}
- var tm time.Time
+ var days int64
switch v := value.Val.(type) {
case DateLiteral:
- tm = Date(v).ToTime()
+ days = int64(v)
case TimestampLiteral:
- tm = Timestamp(v).ToTime()
+ days = int64(Timestamp(v).ToDate())
case TimestampNsLiteral:
- tm = TimestampNano(v).ToTime()
+ days = int64(TimestampNano(v).ToDate())
default:
return out
}
out.Valid = true
- out.Val = Int32Literal(int32((tm.Year()-epochTM.Year())*12 +
(int(tm.Month()) - int(epochTM.Month()))))
+ out.Val = Int32Literal(monthPartitionFromDays(days))
return out
}
diff --git a/transforms_calendar_fastpath_test.go
b/transforms_calendar_fastpath_test.go
new file mode 100644
index 000000000..3316f51c2
--- /dev/null
+++ b/transforms_calendar_fastpath_test.go
@@ -0,0 +1,289 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iceberg
+
+import (
+ "math"
+ "math/rand"
+ "testing"
+ "time"
+)
+
+func checkCalendarDays(t *testing.T, days int64) {
+ t.Helper()
+ expected := epochTM.AddDate(0, 0, int(days))
+ year, month := yearMonthFromUnixDays(days)
+ if year != expected.Year() || month != int(expected.Month()) {
+ t.Fatalf("days=%d: got %d-%02d; expected %d-%02d",
+ days, year, month, expected.Year(), expected.Month())
+ }
+}
+
+func TestYearMonthFromUnixDaysMatchesTime(t *testing.T) {
+ for _, days := range []int64{
+ math.MinInt32, -719469, -719468, -719467,
+ -146097, -146096, -366, -365, -1, 0, 1, 59, 60,
+ 365, 366, 10957, 19782, 20000, 146097, math.MaxInt32,
+ } {
+ checkCalendarDays(t, days)
+ }
+
+ // Exercise Gregorian leap days, century boundaries, pre-epoch dates
+ // and the entire signed Date range with a fixed seed.
+ rng := rand.New(rand.NewSource(19700101))
+ for range 30000 {
+ checkCalendarDays(t, int64(int32(rng.Uint32())))
+ }
+}
+
+func TestYearMonthPartitionTransformParity(t *testing.T) {
+ testCases := []struct {
+ name string
+ typ Type
+ values []any
+ }{
+ {
+ name: "date",
+ typ: DateType{},
+ values: []any{
+ Date(math.MinInt32), Date(-719468), Date(-365),
Date(-1),
+ Date(0), Date(1), Date(18628), Date(20000),
Date(math.MaxInt32),
+ },
+ },
+ {
+ name: "timestamp",
+ typ: TimestampType{},
+ values: []any{
+ Timestamp(math.MinInt64),
Timestamp(-86_400_000_001),
+ Timestamp(-1), Timestamp(0), Timestamp(1),
+ Timestamp(86_399_999_999),
Timestamp(86_400_000_000),
+ Timestamp(math.MaxInt64),
+ },
+ },
+ {
+ name: "timestamp-nanoseconds",
+ typ: TimestampNsType{},
+ values: []any{
+ TimestampNano(math.MinInt64),
TimestampNano(-86_400_000_000_001),
+ TimestampNano(-1), TimestampNano(0),
TimestampNano(1),
+ TimestampNano(86_399_999_999_999),
TimestampNano(86_400_000_000_000),
+ TimestampNano(math.MaxInt64),
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ for _, tr := range []struct {
+ name string
+ transform TimeTransform
+ }{
+ {"year", YearTransform{}},
+ {"month", MonthTransform{}},
+ } {
+ t.Run(tc.name+"/"+tr.name, func(t *testing.T) {
+ fn, err := tr.transform.Transformer(tc.typ)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := fn(nil); got.Valid {
+ t.Fatal("nil input should return
invalid optional")
+ }
+
+ check := func(value any) {
+ t.Helper()
+ var tm time.Time
+ var literal Literal
+ switch v := value.(type) {
+ case Date:
+ tm, literal = v.ToTime(),
DateLiteral(v)
+ case Timestamp:
+ tm, literal = v.ToTime(),
TimestampLiteral(v)
+ case TimestampNano:
+ tm, literal = v.ToTime(),
TimestampNsLiteral(v)
+ }
+ expected := int32(tm.Year() -
epochTM.Year())
+ if tr.name == "month" {
+ expected =
int32((tm.Year()-epochTM.Year())*12 + int(tm.Month()) - int(epochTM.Month()))
+ }
+
+ if result := fn(value); !result.Valid
|| result.Val != expected {
+ t.Fatalf("%s of %v:
Transformer=%v, expected=%d",
+ tr.name, value, result,
expected)
+ }
+
+ if applied :=
tr.transform.Apply(Optional[Literal]{Valid: true, Val: literal});
!applied.Valid ||
+ applied.Val !=
Int32Literal(expected) {
+ t.Fatalf("%s of %v: Apply=%v,
expected=%d",
+ tr.name, value,
applied, expected)
+ }
+ }
+
+ for _, value := range tc.values {
+ check(value)
+ }
+ rng := rand.New(rand.NewSource(934871))
+ for range 5000 {
+ switch tc.typ.(type) {
+ case DateType:
+ check(Date(int32(rng.Uint32())))
+ case TimestampType:
+
check(Timestamp(int64(rng.Uint64())))
+ case TimestampNsType:
+
check(TimestampNano(int64(rng.Uint64())))
+ }
+ }
+ })
+ }
+ }
+}
+
+// referenceCalendarTransformer preserves the previous time.Time-based
+// implementations for paired benchmarks. Keep each source type assertion
+// inside its closure to compare equivalent work.
+func referenceCalendarTransformer(src Type, month bool) func(any)
Optional[int32] {
+ switch src.(type) {
+ case DateType:
+ if month {
+ return func(value any) Optional[int32] {
+ if value == nil {
+ return Optional[int32]{}
+ }
+ tm := value.(Date).ToTime()
+
+ return Optional[int32]{
+ Valid: true,
+ Val:
int32((tm.Year()-epochTM.Year())*12 + int(tm.Month()) - int(epochTM.Month())),
+ }
+ }
+ }
+
+ return func(value any) Optional[int32] {
+ if value == nil {
+ return Optional[int32]{}
+ }
+ tm := value.(Date).ToTime()
+
+ return Optional[int32]{Valid: true, Val:
int32(tm.Year() - epochTM.Year())}
+ }
+ case TimestampType:
+ if month {
+ return func(value any) Optional[int32] {
+ if value == nil {
+ return Optional[int32]{}
+ }
+ tm := value.(Timestamp).ToTime()
+
+ return Optional[int32]{
+ Valid: true,
+ Val:
int32((tm.Year()-epochTM.Year())*12 + int(tm.Month()) - int(epochTM.Month())),
+ }
+ }
+ }
+
+ return func(value any) Optional[int32] {
+ if value == nil {
+ return Optional[int32]{}
+ }
+ tm := value.(Timestamp).ToTime()
+
+ return Optional[int32]{Valid: true, Val:
int32(tm.Year() - epochTM.Year())}
+ }
+ case TimestampNsType:
+ if month {
+ return func(value any) Optional[int32] {
+ if value == nil {
+ return Optional[int32]{}
+ }
+ tm := value.(TimestampNano).ToTime()
+
+ return Optional[int32]{
+ Valid: true,
+ Val:
int32((tm.Year()-epochTM.Year())*12 + int(tm.Month()) - int(epochTM.Month())),
+ }
+ }
+ }
+
+ return func(value any) Optional[int32] {
+ if value == nil {
+ return Optional[int32]{}
+ }
+ tm := value.(TimestampNano).ToTime()
+
+ return Optional[int32]{Valid: true, Val:
int32(tm.Year() - epochTM.Year())}
+ }
+ }
+
+ return nil
+}
+
+var calendarBenchmarkSink int32
+
+func BenchmarkCalendarPartitionTransform(b *testing.B) {
+ cases := []struct {
+ name string
+ typ Type
+ values []any
+ }{
+ {
+ name: "date",
+ typ: DateType{},
+ values: []any{Date(0), Date(-10000), Date(19782),
Date(20000), Date(math.MinInt32), Date(math.MaxInt32)},
+ },
+ {
+ name: "timestamp",
+ typ: TimestampType{},
+ values: []any{Timestamp(0), Timestamp(-1),
Timestamp(-1682982932131), Timestamp(17282931234567), Timestamp(math.MaxInt64)},
+ },
+ {
+ name: "timestamp-nanoseconds",
+ typ: TimestampNsType{},
+ values: []any{TimestampNano(0), TimestampNano(-1),
TimestampNano(-1682982932131), TimestampNano(17282931234567),
TimestampNano(math.MaxInt64)},
+ },
+ }
+ for _, tc := range cases {
+ for _, tr := range []struct {
+ name string
+ transform TimeTransform
+ isMonth bool
+ }{
+ {name: "year", transform: YearTransform{}},
+ {name: "month", transform: MonthTransform{}, isMonth:
true},
+ } {
+ optimized, err := tr.transform.Transformer(tc.typ)
+ if err != nil {
+ b.Fatal(err)
+ }
+ reference := referenceCalendarTransformer(tc.typ,
tr.isMonth)
+ for _, candidate := range []struct {
+ name string
+ fn func(any) Optional[int32]
+ }{
+ {name: "reference", fn: reference},
+ {name: "optimized", fn: optimized},
+ } {
+ b.Run(tc.name+"/"+tr.name+"/"+candidate.name,
func(b *testing.B) {
+ b.ReportAllocs()
+ var sum int32
+ for i := range b.N {
+ sum +=
candidate.fn(tc.values[i%len(tc.values)]).Val
+ }
+ calendarBenchmarkSink = sum
+ })
+ }
+ }
+ }
+}
