This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new d65c69add perf(partitions): avoid temporary fields in PartitionType
(#2121)
d65c69add is described below
commit d65c69add21ef84e40b64b3accf229c5c8135430
Author: Minh Vu <[email protected]>
AuthorDate: Tue Oct 6 18:33:12 2026 +0200
perf(partitions): avoid temporary fields in PartitionType (#2121)
---
partitions.go | 24 +++++++------
partitions_test.go | 84 +++++++++++++++++++++++++++++++++++++++++++
partitions_type_bench_test.go | 82 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 180 insertions(+), 10 deletions(-)
diff --git a/partitions.go b/partitions.go
index 58014bd41..efd11e621 100644
--- a/partitions.go
+++ b/partitions.go
@@ -867,8 +867,8 @@ func (ps *PartitionSpec) resolvedPartitionFields(schema
*Schema) []resolvedParti
fields := make([]resolvedPartitionField, 0, len(ps.fields))
for _, field := range ps.fields {
sourceType := Type(UnknownType{})
- if typ, ok := schema.FindTypeByID(field.SourceID()); ok {
- sourceType = typ
+ if sourceField, ok := schema.FindFieldByIDRef(field.SourceID(),
internal.SchemaRef{}); ok {
+ sourceType = sourceField.Type
}
fields = append(fields, resolvedPartitionField{
@@ -896,15 +896,19 @@ func (ps *PartitionSpec) resolvedPartitionFields(schema
*Schema) []resolvedParti
// retains the field's position and lets transforms with fixed result types,
// such as bucket, continue to resolve their result type.
func (ps *PartitionSpec) PartitionType(schema *Schema) *StructType {
- resolvedFields := ps.resolvedPartitionFields(schema)
- nestedFields := make([]NestedField, 0, len(resolvedFields))
- for _, field := range resolvedFields {
- nestedFields = append(nestedFields, NestedField{
- ID: field.field.FieldID,
- Name: field.field.Name,
- Type: field.resultType,
+ nestedFields := make([]NestedField, len(ps.fields))
+ for i, field := range ps.fields {
+ sourceType := Type(UnknownType{})
+ if typ, ok := schema.FindTypeByID(field.SourceID()); ok {
+ sourceType = typ
+ }
+
+ nestedFields[i] = NestedField{
+ ID: field.FieldID,
+ Name: field.Name,
+ Type: field.Transform.ResultType(sourceType),
Required: false,
- })
+ }
}
return &StructType{FieldList: nestedFields}
diff --git a/partitions_test.go b/partitions_test.go
index a1b5a4b74..542b3e87d 100644
--- a/partitions_test.go
+++ b/partitions_test.go
@@ -523,6 +523,90 @@ func TestPartitionType(t *testing.T) {
"expected: %s, got: %s", expectedWithDroppedSources, actual)
}
+func TestPartitionTypePreservesFieldOrderAndResultTypes(t *testing.T) {
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "record", Type:
&iceberg.StructType{FieldList: []iceberg.NestedField{
+ {ID: 2, Name: "nested", Type: iceberg.Int64Type{},
Required: true},
+ }}},
+ iceberg.NestedField{ID: 3, Name: "ts", Type:
iceberg.TimestampTzNsType{}, Required: true},
+ iceberg.NestedField{ID: 4, Name: "amount", Type:
iceberg.DecimalTypeOf(12, 2)},
+ )
+ unknown, err := iceberg.ParseTransform("custom_transform[42]")
+ require.NoError(t, err)
+
+ fields := []iceberg.PartitionField{
+ {SourceIDs: []int{2}, FieldID: 1007, Name: "nested", Transform:
iceberg.IdentityTransform{}},
+ {SourceIDs: []int{3}, FieldID: 1001, Name: "ts", Transform:
iceberg.IdentityTransform{}},
+ {SourceIDs: []int{4}, FieldID: 1006, Name: "amount", Transform:
iceberg.TruncateTransform{Width: 10}},
+ {SourceIDs: []int{99}, FieldID: 1002, Name: "missing_bucket",
Transform: iceberg.BucketTransform{NumBuckets: 16}},
+ {SourceIDs: []int{99}, FieldID: 1005, Name: "missing_day",
Transform: iceberg.DayTransform{}},
+ {SourceIDs: []int{99}, FieldID: 1003, Name: "missing_identity",
Transform: iceberg.IdentityTransform{}},
+ {SourceIDs: []int{0}, FieldID: 1004, Name: "void", Transform:
iceberg.VoidTransform{}},
+ {SourceIDs: []int{99}, FieldID: 1000, Name: "custom",
Transform: unknown},
+ }
+ spec := iceberg.NewPartitionSpec(fields...)
+
+ want := []iceberg.NestedField{
+ {ID: 1007, Name: "nested", Type: iceberg.Int64Type{}},
+ {ID: 1001, Name: "ts", Type: iceberg.TimestampTzNsType{}},
+ {ID: 1006, Name: "amount", Type: iceberg.DecimalTypeOf(12, 2)},
+ {ID: 1002, Name: "missing_bucket", Type: iceberg.Int32Type{}},
+ {ID: 1005, Name: "missing_day", Type: iceberg.DateType{}},
+ {ID: 1003, Name: "missing_identity", Type:
iceberg.UnknownType{}},
+ {ID: 1004, Name: "void", Type: iceberg.UnknownType{}},
+ {ID: 1000, Name: "custom", Type: iceberg.StringType{}},
+ }
+ require.Equal(t, want, spec.PartitionType(schema).FieldList)
+
+ for i, field := range fields {
+ require.True(t, field.Equals(spec.Field(i)))
+ }
+}
+
+func TestPartitionTypeResolvesEachSchema(t *testing.T) {
+ spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
+ SourceIDs: []int{1}, FieldID: 1000, Name: "value", Transform:
iceberg.IdentityTransform{},
+ })
+ schemas := []*iceberg.Schema{
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "value",
Type: iceberg.StringType{}}),
+ iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "value",
Type: iceberg.Int64Type{}}),
+ iceberg.NewSchema(0),
+ }
+ types := []iceberg.Type{iceberg.StringType{}, iceberg.Int64Type{},
iceberg.UnknownType{}}
+ for range 2 {
+ for i, schema := range schemas {
+ require.Equal(t, []iceberg.NestedField{
+ {ID: 1000, Name: "value", Type: types[i]},
+ }, spec.PartitionType(schema).FieldList)
+ }
+ }
+}
+
+func TestPartitionTypeReturnsIndependentFields(t *testing.T) {
+ schema := iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name:
"value", Type: iceberg.StringType{}})
+ spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
+ SourceIDs: []int{1}, FieldID: 1000, Name: "value", Transform:
iceberg.IdentityTransform{},
+ })
+ first := spec.PartitionType(schema)
+ second := spec.PartitionType(schema)
+ first.FieldList[0] = iceberg.NestedField{ID: 2000, Name: "changed",
Type: iceberg.Int64Type{}, Required: true}
+
+ want := []iceberg.NestedField{{ID: 1000, Name: "value", Type:
iceberg.StringType{}}}
+ require.Equal(t, want, second.FieldList)
+ require.Equal(t, want, spec.PartitionType(schema).FieldList)
+ require.Equal(t, "value", spec.Field(0).Name)
+ require.Equal(t, 1000, spec.Field(0).FieldID)
+}
+
+func TestPartitionTypeUnpartitioned(t *testing.T) {
+ for _, spec := range []iceberg.PartitionSpec{{},
iceberg.NewPartitionSpec()} {
+ partitionType := spec.PartitionType(nil)
+ require.NotNil(t, partitionType)
+ require.NotNil(t, partitionType.FieldList)
+ require.Empty(t, partitionType.FieldList)
+ }
+}
+
type partitionRecord []any
func (p partitionRecord) Size() int { return len(p) }
diff --git a/partitions_type_bench_test.go b/partitions_type_bench_test.go
new file mode 100644
index 000000000..dcceb26b8
--- /dev/null
+++ b/partitions_type_bench_test.go
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/iceberg-go"
+)
+
+var partitionTypeBenchmarkResult *iceberg.StructType
+
+func BenchmarkPartitionTypeFieldCount(b *testing.B) {
+ for _, numFields := range []int{0, 1, 3, 20, 100} {
+ b.Run(fmt.Sprintf("fields=%d", numFields), func(b *testing.B) {
+ fields := make([]iceberg.NestedField, numFields)
+ partitionFields := make([]iceberg.PartitionField,
numFields)
+ for i := range fields {
+ name := fmt.Sprintf("field_%d", i)
+ fields[i] = iceberg.NestedField{ID: i + 1,
Name: name, Type: iceberg.StringType{}}
+ partitionFields[i] = iceberg.PartitionField{
+ SourceIDs: []int{i + 1}, FieldID: 1000
+ i, Name: name,
+ Transform: iceberg.IdentityTransform{},
+ }
+ }
+ schema := iceberg.NewSchema(0, fields...)
+ spec := iceberg.NewPartitionSpec(partitionFields...)
+ partitionTypeBenchmarkResult =
spec.PartitionType(schema)
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for range b.N {
+ partitionTypeBenchmarkResult =
spec.PartitionType(schema)
+ }
+ })
+ }
+}
+
+func BenchmarkPartitionTypeMultipleSchemasWarmIndexes(b *testing.B) {
+ spec := iceberg.NewPartitionSpec(
+ iceberg.PartitionField{
+ SourceIDs: []int{1}, FieldID: 1000, Name: "name",
+ Transform: iceberg.IdentityTransform{},
+ },
+ iceberg.PartitionField{
+ SourceIDs: []int{2}, FieldID: 1001, Name: "count",
+ Transform: iceberg.IdentityTransform{},
+ },
+ )
+ schemas := make([]*iceberg.Schema, 10)
+ for i := range schemas {
+ schemas[i] = iceberg.NewSchema(i,
+ iceberg.NestedField{ID: 1, Name: "name", Type:
iceberg.StringType{}},
+ iceberg.NestedField{ID: 2, Name: "count", Type:
iceberg.Int64Type{}},
+ )
+ // Warm the schema's field-ID index before timing, matching
normal
+ // repeated planning against already-used metadata.
+ partitionTypeBenchmarkResult = spec.PartitionType(schemas[i])
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; b.Loop(); i++ {
+ partitionTypeBenchmarkResult =
spec.PartitionType(schemas[i%len(schemas)])
+ }
+}
