This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new cc260be95 refactor(io)!: consolidate fs adapters behind options (#2300)
cc260be95 is described below
commit cc260be95386c91ab55b7cfc322265034d168bea
Author: Minh Vu <[email protected]>
AuthorDate: Mon Oct 26 18:16:31 2026 +0100
refactor(io)!: consolidate fs adapters behind options (#2300)
* refactor(io)!: consolidate fs adapters behind options
Signed-off-by: Minh Vu <[email protected]>
* docs(release): note v1 API compatibility change
* chore: sync release notes with main to resolve conflicts
---------
Signed-off-by: Minh Vu <[email protected]>
Co-authored-by: Matt Topol <[email protected]>
---
io/io.go | 35 ++++++++++++++++++++-----------
io/io_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++++++---
website/src/releases.md | 1 +
3 files changed, 77 insertions(+), 15 deletions(-)
diff --git a/io/io.go b/io/io.go
index 1e45cb80f..b7e53a1db 100644
--- a/io/io.go
+++ b/io/io.go
@@ -222,24 +222,35 @@ type MkdirAllIO interface {
MkdirAll(path string) error
}
-// FS wraps an io/fs.FS as an IO interface.
-func FS(fsys fs.FS) IO {
+// FS wraps an io/fs.FS as an IO interface. It implements ReadFileIO only
+// if the underlying fsys supports fs.ReadFileFS. Options can configure name
+// processing; see WithNamePreprocessor.
+func FS(fsys fs.FS, opts ...FSOption) IO {
+ adapter := ioFS{fsys: fsys}
+ for _, opt := range opts {
+ if opt != nil {
+ opt(&adapter)
+ }
+ }
+
if _, ok := fsys.(fs.ReadFileFS); ok {
- return readFileFS{ioFS{fsys, nil}}
+ return readFileFS{adapter}
}
- return ioFS{fsys, nil}
+ return adapter
}
-// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then
-// it is called to preprocess any filenames before they are passed to
-// the underlying fsys.
-func FSPreProcName(fsys fs.FS, fn func(string) string) IO {
- if _, ok := fsys.(fs.ReadFileFS); ok {
- return readFileFS{ioFS{fsys, fn}}
- }
+// FSOption configures an IO adapter created by FS.
+type FSOption func(*ioFS)
- return ioFS{fsys, fn}
+// WithNamePreprocessor transforms filenames before FS normalizes leading
+// slashes and passes them to the underlying fs.FS. It applies to Open, Remove,
+// and ReadFile when available. A nil preprocessor leaves names unchanged.
+// fn must be safe for concurrent use by multiple goroutines.
+func WithNamePreprocessor(fn func(string) string) FSOption {
+ return func(adapter *ioFS) {
+ adapter.preProcessName = fn
+ }
}
type readFileFS struct {
diff --git a/io/io_test.go b/io/io_test.go
index 3d870dd96..2a0f7d24d 100644
--- a/io/io_test.go
+++ b/io/io_test.go
@@ -52,13 +52,13 @@ func (f *recordingFS) Remove(name string) error {
return nil
}
-func TestFSPreProcNameTransformsAllOperations(t *testing.T) {
+func TestFSWithNamePreprocessorTransformsAllOperations(t *testing.T) {
underlying := &recordingFS{files: fstest.MapFS{
"rewritten/file.txt": &fstest.MapFile{Data: []byte("data")},
}}
- fsys := iceio.FSPreProcName(underlying, func(string) string {
+ fsys := iceio.FS(underlying, iceio.WithNamePreprocessor(func(string)
string {
return "/rewritten/file.txt"
- })
+ }))
file, err := fsys.Open(context.Background(),
"file://bucket/original.txt")
require.NoError(t, err)
@@ -71,3 +71,53 @@ func TestFSPreProcNameTransformsAllOperations(t *testing.T) {
require.Equal(t, underlying.opened, underlying.read)
require.Equal(t, underlying.opened, underlying.removed)
}
+
+func TestFSDefaultAndNilOptions(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ opts []iceio.FSOption
+ }{
+ {name: "default"},
+ {name: "nil option", opts: []iceio.FSOption{nil}},
+ {name: "nil preprocessor", opts:
[]iceio.FSOption{iceio.WithNamePreprocessor(nil)}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ underlying := &recordingFS{files: fstest.MapFS{
+ "plain.txt": &fstest.MapFile{Data:
[]byte("data")},
+ }}
+ fsys := iceio.FS(underlying, tc.opts...)
+
+ file, err := fsys.Open(context.Background(),
"/plain.txt")
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+
+ reader, ok := fsys.(iceio.ReadFileIO)
+ require.True(t, ok)
+ data, err := reader.ReadFile("/plain.txt")
+ require.NoError(t, err)
+ require.Equal(t, []byte("data"), data)
+
+ require.NoError(t, fsys.Remove(context.Background(),
"/plain.txt"))
+ require.Equal(t, "plain.txt", underlying.opened)
+ require.Equal(t, underlying.opened, underlying.read)
+ require.Equal(t, underlying.opened, underlying.removed)
+ })
+ }
+}
+
+type openOnlyFS struct{ fs.FS }
+
+func TestFSDoesNotAdvertiseUnsupportedReadFile(t *testing.T) {
+ fsys := iceio.FS(openOnlyFS{fstest.MapFS{
+ "plain.txt": &fstest.MapFile{Data: []byte("data")},
+ }}, iceio.WithNamePreprocessor(func(string) string {
+ return "/plain.txt"
+ }))
+
+ _, ok := fsys.(iceio.ReadFileIO)
+ require.False(t, ok)
+
+ file, err := fsys.Open(context.Background(), "unused")
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+}
diff --git a/website/src/releases.md b/website/src/releases.md
index 67f72328d..723138fd8 100644
--- a/website/src/releases.md
+++ b/website/src/releases.md
@@ -42,6 +42,7 @@ Per-release notes (highlights, breaking changes,
contributors) are published on
### Unreleased compatibility notes
+- **Filesystem adapter construction consolidated.** Replace
`io.FSPreProcName(fsys, fn)` with `io.FS(fsys, io.WithNamePreprocessor(fn))`.
Plain `io.FS(fsys)` remains supported.
- **Table purge is required on catalogs.** The `catalog.Catalog` interface now
embeds `catalog.PurgeableCatalog`; every custom catalog implementation must
provide `PurgeTable(ctx, identifier) error`. The narrow
`catalog.PurgeableTable` interface is renamed to `catalog.PurgeableCatalog`.
Built-in purge behavior is unchanged.
- **Unify statistics blob types with Puffin.** Replace `table.BlobType` with
`puffin.BlobType` when constructing `table.BlobMetadata`. Replace
`table.BlobTypeApacheDatasketchesThetaV1` with
`puffin.BlobTypeDataSketchesTheta` and `table.BlobTypeDeletionVectorV1` with
`puffin.BlobTypeDeletionVector`. Use `puffin.BlobType.IsKnown()` to identify
the types defined by the Puffin specification; the redundant
`table.BlobType.IsValid()` method is removed. Unknown non-empty blob type names
still r [...]
- **View representation field uses the Go SQL initialism.** Replace
`view.Representation.Sql` with `view.Representation.SQL`. Serialized view
metadata still uses the `sql` JSON field.
