This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 63ced822c fix(manifest): support nanosecond timestamp partitions
(#2034)
63ced822c is described below
commit 63ced822ca9c7d6534306e58dda3ed6daad8e115
Author: Jeremy Schoemaker <[email protected]>
AuthorDate: Tue Sep 22 16:45:07 2026 -0500
fix(manifest): support nanosecond timestamp partitions (#2034)
* fix(manifest): support nanosecond timestamp partition types in Avro schema
partitionTypeToAvroSchema had no arm for TimestampNsType or
TimestampTzNsType, so any v3 table partitioned on a timestamp_ns or
timestamptz_ns column fell through to "unsupported partition type" and
could not have a manifest written at all. NewManifestWriter calls
partitionTypeToAvroSchema before anything else, so the failure is
unconditional.
The read side already handles these types: the manifest decoder switches
on atype.TimestampNanos, and catalog/rest/scan_task_decoder.go maps both
ns types to atype.TimestampNanos. The library could read partitions it
could not write.
Add TimestampNsNode and TimestampTzNsNode to the internal Avro schema
nodes, using atype.TimestampNanos with adjust-to-utc false and true
respectively, mirroring the existing micros nodes, and wire both ns
types into the partition type switch.
Signed-off-by: Jeremy Schoemaker <[email protected]>
* fix(manifest): finalize nanosecond partition statistics
Signed-off-by: Jeremy Schoemaker <[email protected]>
---------
Signed-off-by: Jeremy Schoemaker <[email protected]>
---
internal/avro_schemas.go | 6 ++-
manifest.go | 8 ++++
schema_conversions.go | 4 ++
schema_conversions_test.go | 108 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 125 insertions(+), 1 deletion(-)
diff --git a/internal/avro_schemas.go b/internal/avro_schemas.go
index fe6c87dd0..d67d8191b 100644
--- a/internal/avro_schemas.go
+++ b/internal/avro_schemas.go
@@ -101,7 +101,11 @@ var (
TimeNode = avro.SchemaNode{Type: atype.Long, LogicalType:
atype.TimeMicros}
TimestampNode = avro.SchemaNode{Type: atype.Long, LogicalType:
atype.TimestampMicros, Props: map[string]any{"adjust-to-utc": false}}
TimestampTzNode = avro.SchemaNode{Type: atype.Long, LogicalType:
atype.TimestampMicros, Props: map[string]any{"adjust-to-utc": true}}
- UUIDNode = avro.SchemaNode{Type: atype.Fixed, Name:
"uuid_fixed", Size: 16, LogicalType: atype.UUID}
+
+ TimestampNsNode = avro.SchemaNode{Type: atype.Long, LogicalType:
atype.TimestampNanos, Props: map[string]any{"adjust-to-utc": false}}
+ TimestampTzNsNode = avro.SchemaNode{Type: atype.Long, LogicalType:
atype.TimestampNanos, Props: map[string]any{"adjust-to-utc": true}}
+
+ UUIDNode = avro.SchemaNode{Type: atype.Fixed, Name: "uuid_fixed", Size:
16, LogicalType: atype.UUID}
)
// Compiled schemas for direct encoding/decoding use.
diff --git a/manifest.go b/manifest.go
index 67705f498..259c96457 100644
--- a/manifest.go
+++ b/manifest.go
@@ -1572,6 +1572,12 @@ func convertPartitionTimestamp(value any) (Timestamp,
bool) {
return Timestamp(converted), ok
}
+func convertPartitionTimestampNano(value any) (TimestampNano, bool) {
+ converted, ok := convertPartitionInt64(value)
+
+ return TimestampNano(converted), ok
+}
+
func newPartitionFieldStat(typ PrimitiveType) (fieldStats, error) {
switch typ.(type) {
case UnknownType:
@@ -1596,6 +1602,8 @@ func newPartitionFieldStat(typ PrimitiveType)
(fieldStats, error) {
return
makePartitionFieldStats[Timestamp](convertPartitionTimestamp), nil
case TimestampTzType:
return
makePartitionFieldStats[Timestamp](convertPartitionTimestamp), nil
+ case TimestampNsType, TimestampTzNsType:
+ return
makePartitionFieldStats[TimestampNano](convertPartitionTimestampNano), nil
case UUIDType:
return
makePartitionFieldStats[uuid.UUID](convertPartitionUUID), nil
case BinaryType:
diff --git a/schema_conversions.go b/schema_conversions.go
index d7a7ab2cc..e23df9961 100644
--- a/schema_conversions.go
+++ b/schema_conversions.go
@@ -48,6 +48,10 @@ func partitionTypeToAvroSchema(t *StructType) (*avro.Schema,
error) {
node = internal.NullableNode(internal.TimestampNode)
case TimestampTzType:
node = internal.NullableNode(internal.TimestampTzNode)
+ case TimestampNsType:
+ node = internal.NullableNode(internal.TimestampNsNode)
+ case TimestampTzNsType:
+ node = internal.NullableNode(internal.TimestampTzNsNode)
case UUIDType:
node = internal.NullableNode(internal.UUIDNode)
case BooleanType:
diff --git a/schema_conversions_test.go b/schema_conversions_test.go
index a57cab2c1..516a8d51e 100644
--- a/schema_conversions_test.go
+++ b/schema_conversions_test.go
@@ -18,7 +18,9 @@
package iceberg
import (
+ "encoding/binary"
"fmt"
+ "io"
"testing"
"time"
@@ -51,6 +53,10 @@ func partitionTypeToAvroSchemaNonNullable(t *StructType)
(*avro.Schema, error) {
node = internal.TimestampNode
case TimestampTzType:
node = internal.TimestampTzNode
+ case TimestampNsType:
+ node = internal.TimestampNsNode
+ case TimestampTzNsType:
+ node = internal.TimestampTzNsNode
case UUIDType:
node = internal.UUIDNode
case BooleanType:
@@ -282,3 +288,105 @@ func TestNonDayInt32PartitionAvroIntEncoding(t
*testing.T) {
assert.IsType(t, int32(0), got, "plain Int32 partition must decode as
int32, not time.Time")
assert.Equal(t, int32(42), got)
}
+
+// TestNanosecondPartitionAvroSchema verifies that v3 nanosecond timestamp
+// types are accepted as partition types and round trip through Avro at
+// nanosecond precision. The read path in manifest.go already decodes
+// atype.TimestampNanos, so the write path has to emit it.
+func TestNanosecondPartitionAvroSchema(t *testing.T) {
+ cases := []struct {
+ name string
+ typ Type
+ }{
+ {"timestamp_ns", TimestampNsType{}},
+ {"timestamptz_ns", TimestampTzNsType{}},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ partitionType := &StructType{FieldList: []NestedField{
+ {ID: 1000, Name: "ts_ns", Type: tc.typ,
Required: false},
+ }}
+
+ avroSchema, err :=
partitionTypeToAvroSchema(partitionType)
+ require.NoError(t, err)
+ require.NotNil(t, avroSchema)
+ nonNullableSchema, err :=
partitionTypeToAvroSchemaNonNullable(partitionType)
+ require.NoError(t, err)
+
+ // A value that is not a whole number of microseconds,
so a
+ // micros-based logical type would lose the trailing
123 ns.
+ want := time.Date(2024, time.March, 1, 12, 0, 0,
456789123, time.UTC)
+
+ nonNullableEncoded, err :=
nonNullableSchema.Encode(map[string]any{"ts_ns": want})
+ require.NoError(t, err)
+ var nonNullableDecoded map[string]any
+ _, err = nonNullableSchema.Decode(nonNullableEncoded,
&nonNullableDecoded)
+ require.NoError(t, err)
+ assert.Equal(t, want, nonNullableDecoded["ts_ns"])
+
+ encoded, err :=
avroSchema.Encode(map[string]any{"ts_ns": want})
+ require.NoError(t, err)
+
+ var decoded map[string]any
+ _, err = avroSchema.Decode(encoded, &decoded)
+ require.NoError(t, err)
+
+ got, ok := decoded["ts_ns"]
+ require.True(t, ok)
+ gotTime, isTime := got.(time.Time)
+ require.True(t, isTime, "nanosecond partition field
must decode as time.Time, got %T", got)
+ assert.Equal(t, want.UnixNano(),
gotTime.UTC().UnixNano(),
+ "nanosecond partition value must round trip
without precision loss")
+ })
+ }
+}
+
+// TestNewManifestWriterV3NanosecondPartition verifies that a v3 manifest can
+// be written for a table with an identity partition on a nanosecond timestamp
+// column. Before the nanosecond arms existed in partitionTypeToAvroSchema,
+// this failed with "unsupported partition type: timestamp_ns".
+func TestNewManifestWriterV3NanosecondPartition(t *testing.T) {
+ cases := []struct {
+ name string
+ typ Type
+ }{
+ {"timestamp_ns", TimestampNsType{}},
+ {"timestamptz_ns", TimestampTzNsType{}},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ sc := NewSchema(0,
+ NestedField{ID: 1, Name: "ts", Type: tc.typ,
Required: true},
+ )
+ spec := NewPartitionSpecID(0,
+ PartitionField{FieldID: 1000, SourceIDs:
[]int{1}, Name: "ts_identity", Transform: IdentityTransform{}},
+ )
+
+ writer, err := NewManifestWriter(3, io.Discard, spec,
sc, 1)
+ require.NoError(t, err)
+ require.NotNil(t, writer)
+
+ values := []TimestampNano{1709294400456789123,
-123456789}
+ for i, value := range values {
+ builder, err := NewDataFileBuilder(spec,
EntryContentData,
+ fmt.Sprintf("data-%d.parquet", i),
ParquetFile,
+ map[int]any{1000: value}, nil, nil, 1,
1)
+ require.NoError(t, err)
+ require.NoError(t, writer.Add(NewManifestEntry(
+ EntryStatusADDED, nil, nil, nil,
builder.Build())))
+ }
+
+ manifest, err := writer.ToManifestFile("manifest.avro",
0)
+ require.NoError(t, err)
+ summaries := manifest.Partitions()
+ require.Len(t, summaries, 1)
+ assert.False(t, summaries[0].ContainsNull)
+ require.NotNil(t, summaries[0].LowerBound)
+ require.NotNil(t, summaries[0].UpperBound)
+ assert.Equal(t, binary.LittleEndian.AppendUint64(nil,
uint64(values[1])), *summaries[0].LowerBound)
+ assert.Equal(t, binary.LittleEndian.AppendUint64(nil,
uint64(values[0])), *summaries[0].UpperBound)
+ })
+ }
+}
