This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new c18cb4fdb perf(io): avoid rebuilding MemFS WalkDir path prefixes
(#2205)
c18cb4fdb is described below
commit c18cb4fdb8461255ae34ee7738a1801959a5974f
Author: Minh Vu <[email protected]>
AuthorDate: Mon Oct 19 14:00:33 2026 +0200
perf(io): avoid rebuilding MemFS WalkDir path prefixes (#2205)
---
io/mem.go | 20 ++++++++++----------
io/mem_bench_test.go | 23 ++++++++++++++++++++---
io/mem_test.go | 41 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 71 insertions(+), 13 deletions(-)
diff --git a/io/mem.go b/io/mem.go
index e8687a6e9..e7244fb40 100644
--- a/io/mem.go
+++ b/io/mem.go
@@ -136,18 +136,18 @@ func (m *MemFS) WalkDir(root string, fn fs.WalkDirFunc)
error {
addEntry(walkEntry{path: root, isDir: true})
relative := strings.TrimPrefix(key, root+"/")
- parts := strings.Split(relative, "/")
- for i := range parts {
- entryPath := strings.Join(parts[:i+1], "/")
- if root != "" {
- entryPath = root + "/" + entryPath
- }
- entry := walkEntry{path: entryPath, isDir: i <
len(parts)-1}
- if !entry.isDir {
- entry.size = int64(len(data))
+ // Keep empty-root paths relative (e.g. "/a/b" is walked as
"a/b").
+ filePath, start := relative, 0
+ if root != "" {
+ filePath = key
+ start = len(key) - len(relative)
+ }
+ for i := range len(relative) {
+ if relative[i] == '/' {
+ addEntry(walkEntry{path: filePath[:start+i],
isDir: true})
}
- addEntry(entry)
}
+ addEntry(walkEntry{path: filePath, size: int64(len(data))})
}
m.mu.RUnlock()
if len(entriesByPath) == 0 {
diff --git a/io/mem_bench_test.go b/io/mem_bench_test.go
index 50ef091fc..a631094ef 100644
--- a/io/mem_bench_test.go
+++ b/io/mem_bench_test.go
@@ -20,24 +20,41 @@ package io_test
import (
"fmt"
"io/fs"
+ "strings"
"testing"
icebergio "github.com/apache/iceberg-go/io"
)
func BenchmarkMemFSWalkDir(b *testing.B) {
+ benchmarkMemFSWalkDir(b, func(i int) string {
+ return
fmt.Sprintf("mem://bucket/root/dir-%03d/file-%05d.parquet", i%128, i)
+ })
+}
+
+func BenchmarkMemFSWalkDirDeep(b *testing.B) {
+ benchmarkMemFSWalkDir(b, func(i int) string {
+ parts := make([]string, 8)
+ for j := range parts {
+ parts[j] = fmt.Sprintf("dir-%03d", (i+j*13)%128)
+ }
+
+ return fmt.Sprintf("mem://bucket/root/%s/file-%05d.parquet",
strings.Join(parts, "/"), i)
+ })
+}
+
+func benchmarkMemFSWalkDir(b *testing.B, nameForFile func(int) string) {
+ b.Helper()
for _, fileCount := range []int{100, 1_000, 5_000} {
b.Run(fmt.Sprintf("files=%d", fileCount), func(b *testing.B) {
memIO := icebergio.NewMemFS()
for i := range fileCount {
- name :=
fmt.Sprintf("mem://bucket/root/dir-%03d/file-%05d.parquet", i%128, i)
- if err := memIO.WriteFile(name, nil); err !=
nil {
+ if err := memIO.WriteFile(nameForFile(i), nil);
err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
- b.ResetTimer()
for b.Loop() {
if err := memIO.WalkDir("mem://bucket/root",
func(_ string, _ fs.DirEntry, err error) error {
return err
diff --git a/io/mem_test.go b/io/mem_test.go
index f396acbaa..cdc62dba5 100644
--- a/io/mem_test.go
+++ b/io/mem_test.go
@@ -353,3 +353,44 @@ func TestMemIO_WalkDirHonorsSkipAtRootAndSkipAll(t
*testing.T) {
assert.Equal(t, []string{"mem://bucket/root"}, walked)
}
}
+
+func TestMemIO_WalkDirNestedPrefixes(t *testing.T) {
+ memIO := icebergio.NewMemFS()
+ for name, data := range map[string][]byte{
+ "mem://bucket/root/a/b/c/deep.parquet": []byte("deep"),
+ "mem://bucket/root/a/first.parquet": []byte("first!"),
+ "mem://bucket/root/ab/second.parquet": []byte("second file"),
+ } {
+ require.NoError(t, memIO.WriteFile(name, data))
+ }
+
+ type visitedEntry struct {
+ path string
+ isDir bool
+ size int64
+ }
+ var got []visitedEntry
+ err := memIO.WalkDir("mem://bucket/root", func(path string, d
fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ info, err := d.Info()
+ if err != nil {
+ return err
+ }
+ got = append(got, visitedEntry{path: path, isDir: d.IsDir(),
size: info.Size()})
+
+ return nil
+ })
+ require.NoError(t, err)
+ assert.Equal(t, []visitedEntry{
+ {path: "mem://bucket/root", isDir: true},
+ {path: "mem://bucket/root/a", isDir: true},
+ {path: "mem://bucket/root/a/b", isDir: true},
+ {path: "mem://bucket/root/a/b/c", isDir: true},
+ {path: "mem://bucket/root/a/b/c/deep.parquet", size: 4},
+ {path: "mem://bucket/root/a/first.parquet", size: 6},
+ {path: "mem://bucket/root/ab", isDir: true},
+ {path: "mem://bucket/root/ab/second.parquet", size: 11},
+ }, got)
+}
