This is an automated email from the ASF dual-hosted git repository. koushiro pushed a commit to branch extract-lakefs-service in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 3137ba72f5f2d12f98fbf539081aec904929fbfc Author: koushiro <[email protected]> AuthorDate: Wed Dec 17 12:32:58 2025 +0800 refactor: Split lakefs service to new crate --- core/Cargo.lock | 15 ++++++++ core/Cargo.toml | 3 +- core/core/Cargo.toml | 1 - core/core/src/services/mod.rs | 5 --- core/services/lakefs/Cargo.toml | 26 ++++++++++++++ .../lakefs => services/lakefs/src}/backend.rs | 4 +-- .../lakefs => services/lakefs/src}/config.rs | 40 ++++++++++------------ .../lakefs => services/lakefs/src}/core.rs | 6 ++-- .../lakefs => services/lakefs/src}/deleter.rs | 5 +-- .../lakefs => services/lakefs/src}/docs.md | 4 +-- .../lakefs => services/lakefs/src}/error.rs | 8 ++--- .../lakefs/mod.rs => services/lakefs/src/lib.rs} | 11 +++--- .../lakefs => services/lakefs/src}/lister.rs | 8 +++-- .../lakefs => services/lakefs/src}/writer.rs | 5 +-- core/src/lib.rs | 4 +++ 15 files changed, 93 insertions(+), 52 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index e03ff0760..e02df9797 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5569,6 +5569,7 @@ dependencies = [ "opendal-service-github", "opendal-service-hdfs-native", "opendal-service-ipfs", + "opendal-service-lakefs", "opendal-service-memcached", "opendal-service-mini-moka", "opendal-service-moka", @@ -6153,6 +6154,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-lakefs" +version = "0.55.0" +dependencies = [ + "bytes", + "ctor", + "http 1.4.0", + "log", + "opendal-core", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "opendal-service-memcached" version = "0.55.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index e6231e25f..08d23ec6f 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -131,7 +131,7 @@ services-huggingface = ["opendal-core/services-huggingface"] services-ipfs = ["dep:opendal-service-ipfs"] services-ipmfs = ["opendal-core/services-ipmfs"] services-koofr = ["opendal-core/services-koofr"] -services-lakefs = ["opendal-core/services-lakefs"] +services-lakefs = ["dep:opendal-service-lakefs"] services-memcached = ["dep:opendal-service-memcached"] services-memory = ["opendal-core/services-memory"] services-mini-moka = ["dep:opendal-service-mini-moka"] @@ -221,6 +221,7 @@ opendal-service-ghac = { path = "services/ghac", version = "0.55.0", optional = opendal-service-github = { path = "services/github", version = "0.55.0", optional = true, default-features = false } opendal-service-hdfs-native = { path = "services/hdfs-native", version = "0.55.0", optional = true, default-features = false } opendal-service-ipfs = { path = "services/ipfs", version = "0.55.0", optional = true, default-features = false } +opendal-service-lakefs = { path = "services/lakefs", version = "0.55.0", optional = true, default-features = false } opendal-service-memcached = { path = "services/memcached", version = "0.55.0", optional = true, default-features = false } opendal-service-mini-moka = { path = "services/mini_moka", version = "0.55.0", optional = true, default-features = false } opendal-service-moka = { path = "services/moka", version = "0.55.0", optional = true, default-features = false } diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml index edfaefc7d..e7f0af46e 100644 --- a/core/core/Cargo.toml +++ b/core/core/Cargo.toml @@ -69,7 +69,6 @@ services-http = [] services-huggingface = [] services-ipmfs = [] services-koofr = [] -services-lakefs = [] services-memory = [] services-mongodb = ["dep:mongodb", "dep:mongodb-internal-macros"] services-monoiofs = ["dep:monoio", "dep:flume"] diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs index e1553e71d..3292db0a4 100644 --- a/core/core/src/services/mod.rs +++ b/core/core/src/services/mod.rs @@ -79,11 +79,6 @@ mod koofr; #[cfg(feature = "services-koofr")] pub use koofr::*; -#[cfg(feature = "services-lakefs")] -mod lakefs; -#[cfg(feature = "services-lakefs")] -pub use lakefs::*; - #[cfg(feature = "services-memory")] mod memory; #[cfg(feature = "services-memory")] diff --git a/core/services/lakefs/Cargo.toml b/core/services/lakefs/Cargo.toml new file mode 100644 index 000000000..d89c3f785 --- /dev/null +++ b/core/services/lakefs/Cargo.toml @@ -0,0 +1,26 @@ +[package] +description = "Apache OpenDAL lakeFS service implementation" +name = "opendal-service-lakefs" + +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +bytes = { workspace = true } +ctor = { workspace = true } +http = { workspace = true } +log = { workspace = true } +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/core/core/src/services/lakefs/backend.rs b/core/services/lakefs/src/backend.rs similarity index 99% rename from core/core/src/services/lakefs/backend.rs rename to core/services/lakefs/src/backend.rs index c4ffc6b6e..e789cc33c 100644 --- a/core/core/src/services/lakefs/backend.rs +++ b/core/services/lakefs/src/backend.rs @@ -21,6 +21,8 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; +use opendal_core::raw::*; +use opendal_core::*; use super::LAKEFS_SCHEME; use super::config::LakefsConfig; @@ -30,8 +32,6 @@ use super::deleter::LakefsDeleter; use super::error::parse_error; use super::lister::LakefsLister; use super::writer::LakefsWriter; -use crate::raw::*; -use crate::*; /// [Lakefs](https://docs.lakefs.io/reference/api.html#/)'s API support. #[doc = include_str!("docs.md")] diff --git a/core/core/src/services/lakefs/config.rs b/core/services/lakefs/src/config.rs similarity index 83% rename from core/core/src/services/lakefs/config.rs rename to core/services/lakefs/src/config.rs index dff70ec74..04a338808 100644 --- a/core/core/src/services/lakefs/config.rs +++ b/core/services/lakefs/src/config.rs @@ -17,6 +17,7 @@ use std::fmt::Debug; +use opendal_core::*; use serde::Deserialize; use serde::Serialize; @@ -66,16 +67,13 @@ impl Debug for LakefsConfig { } } -impl crate::Configurator for LakefsConfig { +impl Configurator for LakefsConfig { type Builder = LakefsBuilder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> { + fn from_uri(uri: &OperatorUri) -> Result<Self> { let raw_path = uri.root().ok_or_else(|| { - crate::Error::new( - crate::ErrorKind::ConfigInvalid, - "uri path must contain repository", - ) - .with_context("service", LAKEFS_SCHEME) + Error::new(ErrorKind::ConfigInvalid, "uri path must contain repository") + .with_context("service", LAKEFS_SCHEME) })?; let (repository, remainder) = match raw_path.split_once('/') { @@ -89,8 +87,8 @@ impl crate::Configurator for LakefsConfig { Some(repository) } .ok_or_else(|| { - crate::Error::new( - crate::ErrorKind::ConfigInvalid, + Error::new( + ErrorKind::ConfigInvalid, "repository is required in uri path", ) .with_context("service", LAKEFS_SCHEME) @@ -136,42 +134,40 @@ impl crate::Configurator for LakefsConfig { #[cfg(test)] mod tests { use super::*; - use crate::Configurator; - use crate::types::OperatorUri; #[test] - fn from_uri_sets_endpoint_repository_branch_and_root() { + fn from_uri_sets_endpoint_repository_branch_and_root() -> Result<()> { let uri = OperatorUri::new( "lakefs://api.example.com/sample/main/data/dir", Vec::<(String, String)>::new(), - ) - .unwrap(); + )?; - let cfg = LakefsConfig::from_uri(&uri).unwrap(); + let cfg = LakefsConfig::from_uri(&uri)?; assert_eq!(cfg.endpoint.as_deref(), Some("https://api.example.com";)); assert_eq!(cfg.repository.as_deref(), Some("sample")); assert_eq!(cfg.branch.as_deref(), Some("main")); assert_eq!(cfg.root.as_deref(), Some("data/dir")); + Ok(()) } #[test] - fn from_uri_requires_repository() { - let uri = - OperatorUri::new("lakefs://api.example.com", Vec::<(String, String)>::new()).unwrap(); + fn from_uri_requires_repository() -> Result<()> { + let uri = OperatorUri::new("lakefs://api.example.com", Vec::<(String, String)>::new())?; assert!(LakefsConfig::from_uri(&uri).is_err()); + Ok(()) } #[test] - fn from_uri_respects_branch_override_and_sets_root() { + fn from_uri_respects_branch_override_and_sets_root() -> Result<()> { let uri = OperatorUri::new( "lakefs://api.example.com/sample/content", vec![("branch".to_string(), "develop".to_string())], - ) - .unwrap(); + )?; - let cfg = LakefsConfig::from_uri(&uri).unwrap(); + let cfg = LakefsConfig::from_uri(&uri)?; assert_eq!(cfg.branch.as_deref(), Some("develop")); assert_eq!(cfg.root.as_deref(), Some("content")); + Ok(()) } } diff --git a/core/core/src/services/lakefs/core.rs b/core/services/lakefs/src/core.rs similarity index 99% rename from core/core/src/services/lakefs/core.rs rename to core/services/lakefs/src/core.rs index 86051acd4..425184516 100644 --- a/core/core/src/services/lakefs/core.rs +++ b/core/services/lakefs/src/core.rs @@ -22,11 +22,11 @@ use std::sync::Arc; use http::Request; use http::Response; use http::header; +use opendal_core::raw::*; +use opendal_core::*; use serde::Deserialize; -use crate::raw::*; -use crate::*; - +/// Core shared state for lakefs service. pub struct LakefsCore { pub info: Arc<AccessorInfo>, pub endpoint: String, diff --git a/core/core/src/services/lakefs/deleter.rs b/core/services/lakefs/src/deleter.rs similarity index 94% rename from core/core/src/services/lakefs/deleter.rs rename to core/services/lakefs/src/deleter.rs index a75820919..ae0c7a6c5 100644 --- a/core/core/src/services/lakefs/deleter.rs +++ b/core/services/lakefs/src/deleter.rs @@ -18,12 +18,13 @@ use std::sync::Arc; use http::StatusCode; +use opendal_core::raw::*; +use opendal_core::*; use super::core::*; use super::error::parse_error; -use crate::raw::*; -use crate::*; +/// Deleter for lakefs service operations. pub struct LakefsDeleter { core: Arc<LakefsCore>, } diff --git a/core/core/src/services/lakefs/docs.md b/core/services/lakefs/src/docs.md similarity index 96% rename from core/core/src/services/lakefs/docs.md rename to core/services/lakefs/src/docs.md index 20348c7fd..cb23d4f85 100644 --- a/core/core/src/services/lakefs/docs.md +++ b/core/services/lakefs/src/docs.md @@ -34,8 +34,8 @@ Refer to [`LakefsBuilder`]'s public API docs for more information. ```rust,no_run use opendal_core::Operator; -use opendal_core::services::Lakefs; -use anyhow::Result; +use opendal_core::Result; +use opendal_service_lakefs::Lakefs; #[tokio::main] async fn main() -> Result<()> { diff --git a/core/core/src/services/lakefs/error.rs b/core/services/lakefs/src/error.rs similarity index 96% rename from core/core/src/services/lakefs/error.rs rename to core/services/lakefs/src/error.rs index 3e64b0eea..f8bd43f57 100644 --- a/core/core/src/services/lakefs/error.rs +++ b/core/services/lakefs/src/error.rs @@ -19,11 +19,10 @@ use std::fmt::Debug; use http::Response; use http::StatusCode; +use opendal_core::raw::*; +use opendal_core::*; use serde::Deserialize; -use crate::raw::*; -use crate::*; - /// LakefsError is the error returned by Lakefs File System. #[derive(Default, Deserialize)] struct LakefsError { @@ -72,8 +71,7 @@ pub(super) fn parse_error(resp: Response<Buffer>) -> Error { #[cfg(test)] mod test { use super::*; - use crate::raw::new_json_deserialize_error; - use crate::types::Result; + use opendal_core::raw::new_json_deserialize_error; #[test] fn test_parse_error() -> Result<()> { diff --git a/core/core/src/services/lakefs/mod.rs b/core/services/lakefs/src/lib.rs similarity index 84% rename from core/core/src/services/lakefs/mod.rs rename to core/services/lakefs/src/lib.rs index cc55ed939..b9e0076f8 100644 --- a/core/core/src/services/lakefs/mod.rs +++ b/core/services/lakefs/src/lib.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -/// Default scheme for lakefs service. -pub const LAKEFS_SCHEME: &str = "lakefs"; +//! lakeFS service implementation for Apache OpenDAL. -use crate::types::DEFAULT_OPERATOR_REGISTRY; +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(missing_docs)] mod backend; mod config; @@ -31,7 +31,10 @@ mod writer; pub use backend::LakefsBuilder as Lakefs; pub use config::LakefsConfig; +/// Default scheme for lakefs service. +pub const LAKEFS_SCHEME: &str = "lakefs"; + #[ctor::ctor] fn register_lakefs_service() { - DEFAULT_OPERATOR_REGISTRY.register::<Lakefs>(LAKEFS_SCHEME); + opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<Lakefs>(LAKEFS_SCHEME); } diff --git a/core/core/src/services/lakefs/lister.rs b/core/services/lakefs/src/lister.rs similarity index 97% rename from core/core/src/services/lakefs/lister.rs rename to core/services/lakefs/src/lister.rs index fe49f1529..76e207160 100644 --- a/core/core/src/services/lakefs/lister.rs +++ b/core/services/lakefs/src/lister.rs @@ -17,13 +17,15 @@ use std::sync::Arc; +use bytes::Buf; +use opendal_core::raw::*; +use opendal_core::*; + use super::core::LakefsCore; use super::core::LakefsListResponse; use super::error::parse_error; -use crate::raw::*; -use crate::*; -use bytes::Buf; +/// Lister for lakefs service operations. pub struct LakefsLister { core: Arc<LakefsCore>, path: String, diff --git a/core/core/src/services/lakefs/writer.rs b/core/services/lakefs/src/writer.rs similarity index 97% rename from core/core/src/services/lakefs/writer.rs rename to core/services/lakefs/src/writer.rs index 8699cc07e..d736eebbb 100644 --- a/core/core/src/services/lakefs/writer.rs +++ b/core/services/lakefs/src/writer.rs @@ -19,13 +19,14 @@ use std::sync::Arc; use bytes::Buf; use http::StatusCode; +use opendal_core::raw::*; +use opendal_core::*; use super::core::LakefsCore; use super::core::LakefsStatus; use super::error::parse_error; -use crate::raw::*; -use crate::*; +/// Writer for lakefs service operations. pub struct LakefsWriter { core: Arc<LakefsCore>, op: OpWrite, diff --git a/core/src/lib.rs b/core/src/lib.rs index 055936fe5..66be1e9cc 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -66,6 +66,10 @@ pub mod services { pub use opendal_service_hdfs_native::*; #[cfg(feature = "services-ipfs")] pub use opendal_service_ipfs::*; + #[cfg(feature = "services-koofr")] + pub use opendal_service_koofr::*; + #[cfg(feature = "services-lakefs")] + pub use opendal_service_lakefs::*; #[cfg(feature = "services-memcached")] pub use opendal_service_memcached::*; #[cfg(feature = "services-mini-moka")]
