This is an automated email from the ASF dual-hosted git repository.
liurenjie1024 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 7c2d5d199 refactor(storage): Reorganize storage code into a new module
(#2109)
7c2d5d199 is described below
commit 7c2d5d199e50eb9a142bdefe632ac6109fcb2006
Author: Shawn Chang <[email protected]>
AuthorDate: Wed Feb 11 19:03:00 2026 -0800
refactor(storage): Reorganize storage code into a new module (#2109)
## Which issue does this PR close?
- Closes #.
## What changes are included in this PR?
- Add a new module `storage` within `io` and put traits there
## Are these changes tested?
Relying on the existing tests
---
crates/iceberg/src/io/file_io.rs | 3 +--
crates/iceberg/src/io/mod.rs | 10 +---------
crates/iceberg/src/io/{ => storage}/config/azdls.rs | 0
crates/iceberg/src/io/{ => storage}/config/gcs.rs | 0
crates/iceberg/src/io/{ => storage}/config/mod.rs | 0
crates/iceberg/src/io/{ => storage}/config/oss.rs | 0
crates/iceberg/src/io/{ => storage}/config/s3.rs | 0
crates/iceberg/src/io/{ => storage}/local_fs.rs | 2 +-
crates/iceberg/src/io/{ => storage}/memory.rs | 2 +-
crates/iceberg/src/io/{storage.rs => storage/mod.rs} | 12 +++++++++++-
crates/iceberg/src/io/{ => storage}/opendal/azdls.rs | 2 +-
crates/iceberg/src/io/{ => storage}/opendal/fs.rs | 0
crates/iceberg/src/io/{ => storage}/opendal/gcs.rs | 5 ++---
crates/iceberg/src/io/{ => storage}/opendal/memory.rs | 0
crates/iceberg/src/io/{ => storage}/opendal/mod.rs | 2 +-
crates/iceberg/src/io/{ => storage}/opendal/oss.rs | 2 +-
crates/iceberg/src/io/{ => storage}/opendal/s3.rs | 5 ++---
17 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs
index 1ad71d853..267f29536 100644
--- a/crates/iceberg/src/io/file_io.rs
+++ b/crates/iceberg/src/io/file_io.rs
@@ -23,8 +23,7 @@ use std::sync::Arc;
use bytes::Bytes;
use url::Url;
-use super::opendal::OpenDalStorage;
-use super::storage::Storage;
+use super::storage::{OpenDalStorage, Storage};
use crate::{Error, ErrorKind, Result};
/// FileIO implementation, used to manipulate files in underlying storage.
diff --git a/crates/iceberg/src/io/mod.rs b/crates/iceberg/src/io/mod.rs
index 5f65a15fc..07f40d104 100644
--- a/crates/iceberg/src/io/mod.rs
+++ b/crates/iceberg/src/io/mod.rs
@@ -66,19 +66,11 @@
//! - `new_input`: Create input file for reading.
//! - `new_output`: Create output file for writing.
-mod config;
mod file_io;
-mod local_fs;
-mod memory;
-mod opendal;
mod storage;
-pub use config::*;
pub use file_io::*;
-#[cfg(feature = "storage-s3")]
-pub use opendal::CustomAwsCredentialLoader;
-pub use opendal::{OpenDalStorage, OpenDalStorageFactory};
-pub use storage::{Storage, StorageConfig, StorageFactory};
+pub use storage::*;
pub(crate) mod object_cache;
diff --git a/crates/iceberg/src/io/config/azdls.rs
b/crates/iceberg/src/io/storage/config/azdls.rs
similarity index 100%
rename from crates/iceberg/src/io/config/azdls.rs
rename to crates/iceberg/src/io/storage/config/azdls.rs
diff --git a/crates/iceberg/src/io/config/gcs.rs
b/crates/iceberg/src/io/storage/config/gcs.rs
similarity index 100%
rename from crates/iceberg/src/io/config/gcs.rs
rename to crates/iceberg/src/io/storage/config/gcs.rs
diff --git a/crates/iceberg/src/io/config/mod.rs
b/crates/iceberg/src/io/storage/config/mod.rs
similarity index 100%
rename from crates/iceberg/src/io/config/mod.rs
rename to crates/iceberg/src/io/storage/config/mod.rs
diff --git a/crates/iceberg/src/io/config/oss.rs
b/crates/iceberg/src/io/storage/config/oss.rs
similarity index 100%
rename from crates/iceberg/src/io/config/oss.rs
rename to crates/iceberg/src/io/storage/config/oss.rs
diff --git a/crates/iceberg/src/io/config/s3.rs
b/crates/iceberg/src/io/storage/config/s3.rs
similarity index 100%
rename from crates/iceberg/src/io/config/s3.rs
rename to crates/iceberg/src/io/storage/config/s3.rs
diff --git a/crates/iceberg/src/io/local_fs.rs
b/crates/iceberg/src/io/storage/local_fs.rs
similarity index 99%
rename from crates/iceberg/src/io/local_fs.rs
rename to crates/iceberg/src/io/storage/local_fs.rs
index 0a55199f7..d6dd5b433 100644
--- a/crates/iceberg/src/io/local_fs.rs
+++ b/crates/iceberg/src/io/storage/local_fs.rs
@@ -31,7 +31,7 @@ use async_trait::async_trait;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
-use super::{
+use crate::io::{
FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage,
StorageConfig,
StorageFactory,
};
diff --git a/crates/iceberg/src/io/memory.rs
b/crates/iceberg/src/io/storage/memory.rs
similarity index 99%
rename from crates/iceberg/src/io/memory.rs
rename to crates/iceberg/src/io/storage/memory.rs
index 39f1f5db9..cb01ee470 100644
--- a/crates/iceberg/src/io/memory.rs
+++ b/crates/iceberg/src/io/storage/memory.rs
@@ -30,7 +30,7 @@ use async_trait::async_trait;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
-use super::{
+use crate::io::{
FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage,
StorageConfig,
StorageFactory,
};
diff --git a/crates/iceberg/src/io/storage.rs
b/crates/iceberg/src/io/storage/mod.rs
similarity index 93%
rename from crates/iceberg/src/io/storage.rs
rename to crates/iceberg/src/io/storage/mod.rs
index 15cc85ab1..31ceaf243 100644
--- a/crates/iceberg/src/io/storage.rs
+++ b/crates/iceberg/src/io/storage/mod.rs
@@ -17,15 +17,25 @@
//! Storage interfaces for Iceberg.
+mod config;
+mod local_fs;
+mod memory;
+mod opendal;
+
use std::fmt::Debug;
use std::sync::Arc;
use async_trait::async_trait;
use bytes::Bytes;
+pub use config::*;
+pub use local_fs::{LocalFsStorage, LocalFsStorageFactory};
+pub use memory::{MemoryStorage, MemoryStorageFactory};
+#[cfg(feature = "storage-s3")]
+pub use opendal::CustomAwsCredentialLoader;
+pub use opendal::{OpenDalStorage, OpenDalStorageFactory};
use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile};
use crate::Result;
-pub use crate::io::config::StorageConfig;
/// Trait for storage operations in Iceberg.
///
diff --git a/crates/iceberg/src/io/opendal/azdls.rs
b/crates/iceberg/src/io/storage/opendal/azdls.rs
similarity index 99%
rename from crates/iceberg/src/io/opendal/azdls.rs
rename to crates/iceberg/src/io/storage/opendal/azdls.rs
index c957fd62a..759ff301e 100644
--- a/crates/iceberg/src/io/opendal/azdls.rs
+++ b/crates/iceberg/src/io/storage/opendal/azdls.rs
@@ -24,7 +24,7 @@ use opendal::services::AzdlsConfig;
use serde::{Deserialize, Serialize};
use url::Url;
-use crate::io::config::{
+use crate::io::{
ADLS_ACCOUNT_KEY, ADLS_ACCOUNT_NAME, ADLS_AUTHORITY_HOST, ADLS_CLIENT_ID,
ADLS_CLIENT_SECRET,
ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, ADLS_TENANT_ID,
};
diff --git a/crates/iceberg/src/io/opendal/fs.rs
b/crates/iceberg/src/io/storage/opendal/fs.rs
similarity index 100%
rename from crates/iceberg/src/io/opendal/fs.rs
rename to crates/iceberg/src/io/storage/opendal/fs.rs
diff --git a/crates/iceberg/src/io/opendal/gcs.rs
b/crates/iceberg/src/io/storage/opendal/gcs.rs
similarity index 96%
rename from crates/iceberg/src/io/opendal/gcs.rs
rename to crates/iceberg/src/io/storage/opendal/gcs.rs
index 5c6145d32..4cb8aa859 100644
--- a/crates/iceberg/src/io/opendal/gcs.rs
+++ b/crates/iceberg/src/io/storage/opendal/gcs.rs
@@ -22,11 +22,10 @@ use opendal::Operator;
use opendal::services::GcsConfig;
use url::Url;
-use crate::io::config::{
+use crate::io::{
GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD,
GCS_DISABLE_VM_METADATA,
- GCS_NO_AUTH, GCS_SERVICE_PATH, GCS_TOKEN,
+ GCS_NO_AUTH, GCS_SERVICE_PATH, GCS_TOKEN, is_truthy,
};
-use crate::io::is_truthy;
use crate::{Error, ErrorKind, Result};
/// Parse iceberg properties to [`GcsConfig`].
diff --git a/crates/iceberg/src/io/opendal/memory.rs
b/crates/iceberg/src/io/storage/opendal/memory.rs
similarity index 100%
rename from crates/iceberg/src/io/opendal/memory.rs
rename to crates/iceberg/src/io/storage/opendal/memory.rs
diff --git a/crates/iceberg/src/io/opendal/mod.rs
b/crates/iceberg/src/io/storage/opendal/mod.rs
similarity index 99%
rename from crates/iceberg/src/io/opendal/mod.rs
rename to crates/iceberg/src/io/storage/opendal/mod.rs
index fb49dc9e3..ee0df20fc 100644
--- a/crates/iceberg/src/io/opendal/mod.rs
+++ b/crates/iceberg/src/io/storage/opendal/mod.rs
@@ -38,7 +38,7 @@ use opendal::{Operator, Scheme};
pub use s3::CustomAwsCredentialLoader;
use serde::{Deserialize, Serialize};
-use super::{
+use crate::io::{
FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, OutputFile,
Storage,
StorageConfig, StorageFactory,
};
diff --git a/crates/iceberg/src/io/opendal/oss.rs
b/crates/iceberg/src/io/storage/opendal/oss.rs
similarity index 96%
rename from crates/iceberg/src/io/opendal/oss.rs
rename to crates/iceberg/src/io/storage/opendal/oss.rs
index 83fc1424a..98b68f7cd 100644
--- a/crates/iceberg/src/io/opendal/oss.rs
+++ b/crates/iceberg/src/io/storage/opendal/oss.rs
@@ -21,7 +21,7 @@ use opendal::services::OssConfig;
use opendal::{Configurator, Operator};
use url::Url;
-use crate::io::config::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET,
OSS_ENDPOINT};
+use crate::io::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_ENDPOINT};
use crate::{Error, ErrorKind, Result};
/// Parse iceberg props to oss config.
diff --git a/crates/iceberg/src/io/opendal/s3.rs
b/crates/iceberg/src/io/storage/opendal/s3.rs
similarity index 98%
rename from crates/iceberg/src/io/opendal/s3.rs
rename to crates/iceberg/src/io/storage/opendal/s3.rs
index bf7399e01..6b4d64e3a 100644
--- a/crates/iceberg/src/io/opendal/s3.rs
+++ b/crates/iceberg/src/io/storage/opendal/s3.rs
@@ -25,13 +25,12 @@ pub use reqsign::{AwsCredential, AwsCredentialLoad};
use reqwest::Client;
use url::Url;
-use crate::io::config::{
+use crate::io::{
CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN,
S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME,
S3_DISABLE_CONFIG_LOAD,
S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION,
S3_SECRET_ACCESS_KEY,
- S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE,
+ S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, is_truthy,
};
-use crate::io::is_truthy;
use crate::{Error, ErrorKind, Result};
/// Parse iceberg props to s3 config.
