This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 03c4bab5a perf(table): specialize binary partition lookups in
partitions metadata (#2199)
03c4bab5a is described below
commit 03c4bab5a4594eede844801fbf85e7ec36bc1af4
Author: Minh Vu <[email protected]>
AuthorDate: Mon Oct 19 19:32:54 2026 +0200
perf(table): specialize binary partition lookups in partitions metadata
(#2199)
* perf(table): specialize binary partition lookups in partitions metadata
* fix(table): satisfy nlreturn lint in binary partition lookup
---
table/inspect_partitions.go | 47 ++++++++++++---
table/inspect_partitions_bench_test.go | 42 +++++++++++++
table/inspect_partitions_internal_test.go | 99 +++++++++++++++++++++++++++++++
3 files changed, 179 insertions(+), 9 deletions(-)
diff --git a/table/inspect_partitions.go b/table/inspect_partitions.go
index 53f5aad5b..c63dc4877 100644
--- a/table/inspect_partitions.go
+++ b/table/inspect_partitions.go
@@ -46,20 +46,50 @@ type inspectPartitionAggregate struct {
}
type inspectPartitionAggregateTree struct {
- children map[any]*inspectPartitionAggregateTree
- aggregate *inspectPartitionAggregate
+ children map[any]*inspectPartitionAggregateTree
+ binaryChildren map[string]*inspectPartitionAggregateTree
+ aggregate *inspectPartitionAggregate
}
func newInspectPartitionAggregateTree() *inspectPartitionAggregateTree {
- return &inspectPartitionAggregateTree{
- children: make(map[any]*inspectPartitionAggregateTree),
+ return &inspectPartitionAggregateTree{}
+}
+
+// child uses a typed map for binary values to avoid interface-backed keys
+// during repeated lookups. Other values retain their existing key types.
+func (t *inspectPartitionAggregateTree) child(value any)
(*inspectPartitionAggregateTree, bool) {
+ if binary, ok := value.([]byte); ok {
+ child, found := t.binaryChildren[string(binary)]
+
+ return child, found
+ }
+
+ child, found := t.children[comparablePartitionKey(value)]
+
+ return child, found
+}
+
+func (t *inspectPartitionAggregateTree) setChild(value any, child
*inspectPartitionAggregateTree) {
+ if binary, ok := value.([]byte); ok {
+ if t.binaryChildren == nil {
+ t.binaryChildren =
make(map[string]*inspectPartitionAggregateTree)
+ }
+ // A stored string key owns its bytes even if the input buffer
changes.
+ t.binaryChildren[string(binary)] = child
+
+ return
+ }
+
+ if t.children == nil {
+ t.children = make(map[any]*inspectPartitionAggregateTree)
}
+ t.children[comparablePartitionKey(value)] = child
}
func (t *inspectPartitionAggregateTree) lookup(record partitionRecord)
*inspectPartitionAggregate {
node := t
for _, value := range record {
- child, ok := node.children[comparablePartitionKey(value)]
+ child, ok := node.child(value)
if !ok {
return nil
}
@@ -75,7 +105,7 @@ func (t *inspectPartitionAggregateTree) lookupPartition(
) *inspectPartitionAggregate {
node := t
for _, field := range partitionType.FieldList {
- child, ok :=
node.children[comparablePartitionKey(partition[field.ID])]
+ child, ok := node.child(partition[field.ID])
if !ok {
return nil
}
@@ -91,11 +121,10 @@ func (t *inspectPartitionAggregateTree) lookupPartition(
func (t *inspectPartitionAggregateTree) insert(record partitionRecord,
aggregate *inspectPartitionAggregate) {
node := t
for _, value := range record {
- key := comparablePartitionKey(value)
- child, ok := node.children[key]
+ child, ok := node.child(value)
if !ok {
child = newInspectPartitionAggregateTree()
- node.children[key] = child
+ node.setChild(value, child)
}
node = child
}
diff --git a/table/inspect_partitions_bench_test.go
b/table/inspect_partitions_bench_test.go
index cee48c384..aedf0c1ba 100644
--- a/table/inspect_partitions_bench_test.go
+++ b/table/inspect_partitions_bench_test.go
@@ -151,3 +151,45 @@ func benchmarkInspectPartitionFiles(
return partitionType, files
}
+
+func BenchmarkInspectPartitionBinaryLookup(b *testing.B) {
+ const (
+ partitionCount = 1024
+ fieldCount = 8
+ )
+
+ fields := make([]iceberg.NestedField, fieldCount)
+ for i := range fields {
+ fields[i] = iceberg.NestedField{
+ ID: 1000 + i, Name: fmt.Sprintf("part_%d", i), Type:
iceberg.PrimitiveTypes.Binary,
+ }
+ }
+ partitionType := &iceberg.StructType{FieldList: fields}
+
+ tree := newInspectPartitionAggregateTree()
+ partitions := make([]map[int]any, partitionCount)
+ for i := range partitions {
+ record := make(partitionRecord, fieldCount)
+ partition := make(map[int]any, fieldCount)
+ for j, field := range fields {
+ value := []byte(fmt.Sprintf("partition-%d-field-%d", i,
j))
+ record[j] = value
+ partition[field.ID] = value
+ }
+ tree.insert(record, &inspectPartitionAggregate{})
+ partitions[i] = partition
+ }
+
+ index := 0
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ if tree.lookupPartition(partitions[index], partitionType) ==
nil {
+ b.Fatal("partition not found")
+ }
+ index++
+ if index == len(partitions) {
+ index = 0
+ }
+ }
+}
diff --git a/table/inspect_partitions_internal_test.go
b/table/inspect_partitions_internal_test.go
new file mode 100644
index 000000000..5d0d0f320
--- /dev/null
+++ b/table/inspect_partitions_internal_test.go
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "math"
+ "testing"
+
+ "github.com/apache/iceberg-go"
+)
+
+func TestInspectPartitionAggregateTreeBinaryKeys(t *testing.T) {
+ partitionType := &iceberg.StructType{FieldList: []iceberg.NestedField{
+ {ID: 1000, Name: "first", Type: iceberg.PrimitiveTypes.Binary},
+ {ID: 1001, Name: "second", Type: iceberg.PrimitiveTypes.Binary},
+ }}
+ tree := newInspectPartitionAggregateTree()
+
+ check := func(first, second any, want *inspectPartitionAggregate) {
+ t.Helper()
+ if got := tree.lookup(partitionRecord{first, second}); got !=
want {
+ t.Fatalf("positional lookup (%v, %v) = %p, want %p",
first, second, got, want)
+ }
+ partition := map[int]any{1000: first, 1001: second}
+ if got := tree.lookupPartition(partition, partitionType); got
!= want {
+ t.Fatalf("map lookup (%v, %v) = %p, want %p", first,
second, got, want)
+ }
+ }
+
+ check([]byte("missing"), []byte("value"), nil)
+
+ first := []byte{0, 0xff}
+ second := []byte("x/y")
+ original := &inspectPartitionAggregate{}
+ tree.insert(partitionRecord{first, second}, original)
+ first[0] = 1
+ second[0] = 'z'
+ check([]byte{0, 0xff}, []byte("x/y"), original)
+ check(first, second, nil)
+
+ nullFirst := &inspectPartitionAggregate{}
+ emptyFirst := &inspectPartitionAggregate{}
+ tree.insert(partitionRecord{nil, []byte("x")}, nullFirst)
+ tree.insert(partitionRecord{[]byte{}, []byte("x")}, emptyFirst)
+ check(nil, []byte("x"), nullFirst)
+ check([]byte{}, []byte("x"), emptyFirst)
+ check([]byte(nil), []byte("x"), emptyFirst)
+
+ binaryFirst := &inspectPartitionAggregate{}
+ stringFirst := &inspectPartitionAggregate{}
+ tree.insert(partitionRecord{[]byte("same"), []byte("tail")},
binaryFirst)
+ tree.insert(partitionRecord{"same", []byte("tail")}, stringFirst)
+ check([]byte("same"), []byte("tail"), binaryFirst)
+ check("same", []byte("tail"), stringFirst)
+
+ otherNested := &inspectPartitionAggregate{}
+ tree.insert(partitionRecord{[]byte("same"), []byte("other")},
otherNested)
+ check([]byte("same"), []byte("other"), otherNested)
+ check([]byte("same"), []byte("tail"), binaryFirst)
+}
+
+func TestInspectPartitionAggregateTreeMixedKeys(t *testing.T) {
+ tree := newInspectPartitionAggregateTree()
+ negativeZero := &inspectPartitionAggregate{}
+ positiveZero := &inspectPartitionAggregate{}
+ nan := &inspectPartitionAggregate{}
+
+ tree.insert(partitionRecord{[]byte("scope"), math.Copysign(0, -1)},
negativeZero)
+ tree.insert(partitionRecord{[]byte("scope"), float64(0)}, positiveZero)
+ tree.insert(partitionRecord{[]byte("scope"),
math.Float64frombits(0x7ff8000000000001)}, nan)
+
+ for _, test := range []struct {
+ value float64
+ want *inspectPartitionAggregate
+ }{
+ {math.Copysign(0, -1), negativeZero},
+ {0, positiveZero},
+ {math.Float64frombits(0x7ff8000000000002), nan},
+ } {
+ if got := tree.lookup(partitionRecord{[]byte("scope"),
test.value}); got != test.want {
+ t.Fatalf("lookup (%v) = %p, want %p", test.value, got,
test.want)
+ }
+ }
+}
