This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new c6494edbc perf(table): speed up compaction partition tuple keys (#2204)
c6494edbc is described below
commit c6494edbce9e523fca90ef6f9e048ff86e336854
Author: Minh Vu <[email protected]>
AuthorDate: Tue Oct 20 19:14:02 2026 +0200
perf(table): speed up compaction partition tuple keys (#2204)
* perf(table): format compaction partition tuples without repeated fmt calls
* test(table): verify and benchmark compaction partition tuple formatting
* style: format partition tuple test values for gofumpt
* perf(compaction): cover manifest types in tuple formatter
---
table/compaction/eq_delete_decision.go | 35 +++++++-
table/compaction/eq_delete_decision_bench_test.go | 37 +++++----
table/compaction/partition_tuple_test.go | 98 +++++++++++++++++++++++
3 files changed, 153 insertions(+), 17 deletions(-)
diff --git a/table/compaction/eq_delete_decision.go
b/table/compaction/eq_delete_decision.go
index 9fbd068ac..25144289f 100644
--- a/table/compaction/eq_delete_decision.go
+++ b/table/compaction/eq_delete_decision.go
@@ -21,6 +21,7 @@ import (
"fmt"
"math"
"slices"
+ "strconv"
"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/internal"
@@ -246,7 +247,8 @@ func partitionBucketKey(specID int32, part map[int]any)
string {
func partitionBucketKeyFromTuple(specID int32, tuple []byte) string {
key := make([]byte, 0, len(tuple)+12)
- key = fmt.Appendf(key, "%d:", specID)
+ key = strconv.AppendInt(key, int64(specID), 10)
+ key = append(key, ':')
return string(append(key, tuple...))
}
@@ -254,13 +256,40 @@ func partitionBucketKeyFromTuple(specID int32, tuple
[]byte) string {
// appendPartitionTuple emits the sorted "id=value;" tuple into dst.
// Caller is responsible for any leading prefix (e.g. specID).
func appendPartitionTuple(dst []byte, part map[int]any) []byte {
- ids := make([]int, 0, len(part))
+ var inlineIDs [8]int
+ ids := inlineIDs[:0]
+ if len(part) > len(inlineIDs) {
+ ids = make([]int, 0, len(part))
+ }
for id := range part {
ids = append(ids, id)
}
slices.Sort(ids)
+
+ if cap(dst) == 0 && len(part) != 0 {
+ dst = make([]byte, 0, len(part)*16)
+ }
for _, id := range ids {
- dst = fmt.Appendf(dst, "%d=%v;", id, part[id])
+ dst = strconv.AppendInt(dst, int64(id), 10)
+ dst = append(dst, '=')
+ // Match fmt's %v output exactly; these partition keys are
externally visible.
+ switch value := part[id].(type) {
+ case nil:
+ dst = append(dst, "<nil>"...)
+ case string:
+ dst = append(dst, value...)
+ case int32:
+ dst = strconv.AppendInt(dst, int64(value), 10)
+ case iceberg.Date:
+ dst = strconv.AppendInt(dst, int64(value), 10)
+ case int64:
+ dst = strconv.AppendInt(dst, value, 10)
+ case bool:
+ dst = strconv.AppendBool(dst, value)
+ default:
+ dst = fmt.Append(dst, value)
+ }
+ dst = append(dst, ';')
}
return dst
diff --git a/table/compaction/eq_delete_decision_bench_test.go
b/table/compaction/eq_delete_decision_bench_test.go
index bd814cd25..35290c109 100644
--- a/table/compaction/eq_delete_decision_bench_test.go
+++ b/table/compaction/eq_delete_decision_bench_test.go
@@ -32,21 +32,30 @@ var (
func BenchmarkAddSurvivorWithSpec(b *testing.B) {
for _, fields := range []int{1, 8, 32} {
- b.Run(fmt.Sprintf("fields=%d", fields), func(b *testing.B) {
- partition := make(map[int]any, fields)
- for i := range fields {
- partition[1000+i] = i
- }
+ for _, kind := range []string{"int32", "string", "date"} {
+ b.Run(fmt.Sprintf("fields=%d/%s", fields, kind), func(b
*testing.B) {
+ partition := make(map[int]any, fields)
+ for i := range fields {
+ switch kind {
+ case "int32":
+ partition[1000+i] = int32(i * 5)
+ case "string":
+ partition[1000+i] =
fmt.Sprintf("partition-%d", i)
+ case "date":
+ partition[1000+i] =
iceberg.Date(20000 + i)
+ }
+ }
- survey := compaction.NewSurvivorSurvey()
- b.ReportAllocs()
- b.ResetTimer()
- for range b.N {
- survey.AddSurvivorWithSpec(1, partition, 10)
- }
- b.StopTimer()
- benchmarkSurvivorSurvey = survey
- })
+ survey := compaction.NewSurvivorSurvey()
+ b.ReportAllocs()
+ b.ResetTimer()
+ for range b.N {
+ survey.AddSurvivorWithSpec(1,
partition, 10)
+ }
+ b.StopTimer()
+ benchmarkSurvivorSurvey = survey
+ })
+ }
}
}
diff --git a/table/compaction/partition_tuple_test.go
b/table/compaction/partition_tuple_test.go
new file mode 100644
index 000000000..ca2cffddc
--- /dev/null
+++ b/table/compaction/partition_tuple_test.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compaction
+
+import (
+ "bytes"
+ "fmt"
+ "math"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/apache/iceberg-go"
+)
+
+type partitionTupleStringer string
+
+func (s partitionTupleStringer) String() string { return "custom:" + string(s)
}
+
+func TestAppendPartitionTupleMatchesFmt(t *testing.T) {
+ values := []any{
+ nil,
+ "",
+ "a=b;:é",
+ -123,
+ int32(-45),
+ iceberg.Date(-1),
+ iceberg.Date(20500),
+ int64(1234567890123),
+ true,
+ false,
+ uint64(1 << 63),
+ math.NaN(),
+ math.Inf(1),
+ math.Copysign(0, -1),
+ float32(3.5),
+ []byte{0, 255},
+ []int{1, 2},
+ partitionTupleStringer("x"),
+ time.Unix(0, 0).UTC(),
+ (*int)(nil),
+ }
+
+ for _, fields := range []int{0, 1, 3, 8, 9, 16, 32} {
+ for seed := range 32 {
+ partition := make(map[int]any, fields)
+ for i := range fields {
+ partition[1000+i*7] =
values[(i*3+seed)%len(values)]
+ }
+
+ ids := make([]int, 0, len(partition))
+ for id := range partition {
+ ids = append(ids, id)
+ }
+ slices.Sort(ids)
+
+ for _, prefix := range []string{"", "42:"} {
+ want := []byte(prefix)
+ for _, id := range ids {
+ want = fmt.Appendf(want, "%d=%v;", id,
partition[id])
+ }
+ got := appendPartitionTuple([]byte(prefix),
partition)
+ if !bytes.Equal(got, want) {
+ t.Fatalf("fields=%d seed=%d prefix=%q:
got %q, want %q",
+ fields, seed, prefix, got, want)
+ }
+ }
+ }
+ }
+}
+
+func TestPartitionBucketKeyFromTupleMatchesFmt(t *testing.T) {
+ for _, specID := range []int32{math.MinInt32, -1, 0, 1, math.MaxInt32} {
+ for _, tuple := range []string{"", "1000=abc;",
strings.Repeat("x", 128)} {
+ got := partitionBucketKeyFromTuple(specID,
[]byte(tuple))
+ want := fmt.Sprintf("%d:%s", specID, tuple)
+ if got != want {
+ t.Fatalf("spec ID %d: got %q, want %q", specID,
got, want)
+ }
+ }
+ }
+}
