This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new be73f9bfe perf(io): avoid WalkDir comparator allocations (#2138)
be73f9bfe is described below
commit be73f9bfe5052a77df5ed4012833814c5304118f
Author: Minh Vu <[email protected]>
AuthorDate: Wed Oct 7 13:54:29 2026 +0200
perf(io): avoid WalkDir comparator allocations (#2138)
---
io/mem.go | 51 +++++++++++++++++++++++++++++++++++----------------
io/mem_bench_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 85 insertions(+), 16 deletions(-)
diff --git a/io/mem.go b/io/mem.go
index e55b671cb..e8687a6e9 100644
--- a/io/mem.go
+++ b/io/mem.go
@@ -167,22 +167,7 @@ func (m *MemFS) WalkDir(root string, fn fs.WalkDirFunc)
error {
paths = append(paths, entryPath)
}
sort.Slice(paths, func(i, j int) bool {
- if paths[i] == root {
- return true
- }
- if paths[j] == root {
- return false
- }
-
- left := strings.Split(strings.TrimPrefix(paths[i], root+"/"),
"/")
- right := strings.Split(strings.TrimPrefix(paths[j], root+"/"),
"/")
- for index := range min(len(left), len(right)) {
- if left[index] != right[index] {
- return left[index] < right[index]
- }
- }
-
- return len(left) < len(right)
+ return memWalkPathLess(root, paths[i], paths[j])
})
var skipPrefix string
@@ -216,6 +201,40 @@ func memPathInRoot(path, root string) bool {
return root == "" || path == root || strings.HasPrefix(path, root+"/")
}
+func memWalkPathLess(root, left, right string) bool {
+ if left == root {
+ return true
+ }
+ if right == root {
+ return false
+ }
+
+ left = strings.TrimPrefix(strings.TrimPrefix(left, root), "/")
+ right = strings.TrimPrefix(strings.TrimPrefix(right, root), "/")
+ for {
+ leftEnd := strings.IndexByte(left, '/')
+ rightEnd := strings.IndexByte(right, '/')
+
+ leftPart, rightPart := left, right
+ if leftEnd >= 0 {
+ leftPart = left[:leftEnd]
+ }
+ if rightEnd >= 0 {
+ rightPart = right[:rightEnd]
+ }
+
+ if leftPart != rightPart {
+ return leftPart < rightPart
+ }
+ if leftEnd < 0 || rightEnd < 0 {
+ return leftEnd < 0 && rightEnd >= 0
+ }
+
+ left = left[leftEnd+1:]
+ right = right[rightEnd+1:]
+ }
+}
+
type memFile struct {
data []byte
name string
diff --git a/io/mem_bench_test.go b/io/mem_bench_test.go
new file mode 100644
index 000000000..50ef091fc
--- /dev/null
+++ b/io/mem_bench_test.go
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io_test
+
+import (
+ "fmt"
+ "io/fs"
+ "testing"
+
+ icebergio "github.com/apache/iceberg-go/io"
+)
+
+func BenchmarkMemFSWalkDir(b *testing.B) {
+ for _, fileCount := range []int{100, 1_000, 5_000} {
+ b.Run(fmt.Sprintf("files=%d", fileCount), func(b *testing.B) {
+ memIO := icebergio.NewMemFS()
+ for i := range fileCount {
+ name :=
fmt.Sprintf("mem://bucket/root/dir-%03d/file-%05d.parquet", i%128, i)
+ if err := memIO.WriteFile(name, nil); err !=
nil {
+ b.Fatal(err)
+ }
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ if err := memIO.WalkDir("mem://bucket/root",
func(_ string, _ fs.DirEntry, err error) error {
+ return err
+ }); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
