This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 8eff0730 fix(table): map nanosecond Arrow timestamps to v3
TimestampNs/TimestampTzNs types (#1081)
8eff0730 is described below
commit 8eff0730e13ecdcfa72fb11c674083fb74dfd209
Author: Jacob Marble <[email protected]>
AuthorDate: Fri May 15 12:18:18 2026 -0700
fix(table): map nanosecond Arrow timestamps to v3 TimestampNs/TimestampTzNs
types (#1081)
`table/Transaction.AddFiles()` and `table/WriteRecords()` panic with
'ns' timestamp precision not supported (#695). I've managed to work
around this with `table/Transaction.AddDataFiles()`.
The fix is pretty simple, just map the new type in
`table/ArrowTypeToIceberg()`. I've left the downcast path
`downcastTimestamp=true` unchanged.
**Behavior change for v1/v2 callers:** Previously `UTC-tz` and
`empty-tz` nanosecond timestamps with `downcastTimestamp=false` failed
immediately at the conversion step with 'ns' timestamp precision not
supported. They will now produce a `TimestampTzNs`/`TimestampNs` Iceberg
type and fail later at `minFormatVersionForType` if the table is format
version 1 or 2. The failure is preserved — only the site moves.
Closes #695 (again)
Helps #589
---------
Signed-off-by: Jacob Marble <[email protected]>
---
table/arrow_utils.go | 11 ++++++++++-
table/arrow_utils_test.go | 5 +++--
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/table/arrow_utils.go b/table/arrow_utils.go
index 873eb727..ed3ff5d3 100644
--- a/table/arrow_utils.go
+++ b/table/arrow_utils.go
@@ -344,7 +344,16 @@ func (c convertToIceberg) Primitive(dt arrow.DataType)
(result iceberg.NestedFie
case *arrow.TimestampType:
if dt.Unit == arrow.Nanosecond {
if !c.downcastTimestamp {
- panic(fmt.Errorf("%w: 'ns' timestamp precision
not supported", iceberg.ErrType))
+ switch {
+ case slices.Contains(utcAliases, dt.TimeZone):
+ result.Type =
iceberg.PrimitiveTypes.TimestampTzNs
+ case dt.TimeZone == "":
+ result.Type =
iceberg.PrimitiveTypes.TimestampNs
+ default:
+ panic(fmt.Errorf("%w: unsupported arrow
type for conversion - %s", iceberg.ErrInvalidSchema, dt))
+ }
+
+ return result
}
slog.Warn("downcasting nanosecond timestamp to
microsecond, precision loss may occur")
}
diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go
index e3f23718..2e4eef49 100644
--- a/table/arrow_utils_test.go
+++ b/table/arrow_utils_test.go
@@ -76,12 +76,13 @@ func TestArrowToIceberg(t *testing.T) {
{arrow.FixedWidthTypes.Timestamp_s,
iceberg.PrimitiveTypes.TimestampTz, false, ""},
{arrow.FixedWidthTypes.Timestamp_ms,
iceberg.PrimitiveTypes.TimestampTz, false, ""},
{arrow.FixedWidthTypes.Timestamp_us,
iceberg.PrimitiveTypes.TimestampTz, true, ""},
- {arrow.FixedWidthTypes.Timestamp_ns, nil, false, "'ns'
timestamp precision not supported"},
+ {arrow.FixedWidthTypes.Timestamp_ns,
iceberg.PrimitiveTypes.TimestampTzNs, true, ""},
{&arrow.TimestampType{Unit: arrow.Second},
iceberg.PrimitiveTypes.Timestamp, false, ""},
{&arrow.TimestampType{Unit: arrow.Millisecond},
iceberg.PrimitiveTypes.Timestamp, false, ""},
{&arrow.TimestampType{Unit: arrow.Microsecond},
iceberg.PrimitiveTypes.Timestamp, true, ""},
- {&arrow.TimestampType{Unit: arrow.Nanosecond}, nil, false,
"'ns' timestamp precision not supported"},
+ {&arrow.TimestampType{Unit: arrow.Nanosecond},
iceberg.PrimitiveTypes.TimestampNs, true, ""},
{&arrow.TimestampType{Unit: arrow.Microsecond, TimeZone:
"US/Pacific"}, nil, false, "unsupported arrow type for conversion -
timestamp[us, tz=US/Pacific]"},
+ {&arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone:
"US/Pacific"}, nil, false, "unsupported arrow type for conversion -
timestamp[ns, tz=US/Pacific]"},
{arrow.BinaryTypes.String, iceberg.PrimitiveTypes.String, true,
""},
{arrow.BinaryTypes.LargeString, iceberg.PrimitiveTypes.String,
false, ""},
{arrow.BinaryTypes.StringView, nil, false, "unsupported arrow
type for conversion - string_view"},
