This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch xuanwo/s3-crate-extract in repository https://gitbox.apache.org/repos/asf/opendal.git
commit c0c740b1db76b8ef690c021ade4f7293d973066c Author: Xuanwo <[email protected]> AuthorDate: Thu Dec 4 23:27:12 2025 +0800 refactor: Move services s3 out as a crate Signed-off-by: Xuanwo <[email protected]> --- core/Cargo.lock | 31 +++++++++++-- core/Cargo.toml | 6 ++- core/core/Cargo.toml | 15 ------ core/core/src/blocking/operator.rs | 6 +-- core/core/src/docs/concepts.rs | 20 +++----- core/core/src/layers/http_client.rs | 2 +- core/core/src/lib.rs | 8 ++-- core/core/src/services/mod.rs | 5 -- core/core/src/types/builder.rs | 5 +- core/core/src/types/operator/operator.rs | 6 +-- core/services/s3/Cargo.toml | 33 ++++++++++++++ .../src/services/s3 => services/s3/src}/backend.rs | 53 +++++++++++++--------- .../s3 => services/s3/src}/compatible_services.md | 0 .../src/services/s3 => services/s3/src}/config.rs | 13 ++++-- .../src/services/s3 => services/s3/src}/core.rs | 4 +- .../src/services/s3 => services/s3/src}/deleter.rs | 12 ++--- .../src/services/s3 => services/s3/src}/docs.md | 10 ++-- .../src/services/s3 => services/s3/src}/error.rs | 4 +- .../services/s3/mod.rs => services/s3/src/lib.rs} | 12 +++-- .../src/services/s3 => services/s3/src}/lister.rs | 37 ++++++++++----- .../src/services/s3 => services/s3/src}/mod.rs | 2 +- .../src/services/s3 => services/s3/src}/writer.rs | 27 ++++++----- core/src/lib.rs | 9 +++- 23 files changed, 195 insertions(+), 125 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 9c7b5927e..0dd01bee2 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5406,6 +5406,7 @@ dependencies = [ "libtest-mimic", "log", "opendal-core", + "opendal-service-s3", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", @@ -5458,7 +5459,6 @@ dependencies = [ "bytes", "cacache", "compio", - "crc32c", "criterion", "ctor", "dashmap 6.1.0", @@ -5512,10 +5512,6 @@ dependencies = [ "redb", "redis", "reqsign", - "reqsign-aws-v4", - "reqsign-core", - "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "reqwest", "rocksdb", "rustls-native-certs 0.8.2", @@ -5580,6 +5576,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "opendal-service-s3" +version = "0.55.0" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "ctor", + "http 1.3.1", + "log", + "md-5", + "opendal-core", + "pretty_assertions", + "quick-xml", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", + "reqwest", + "serde", + "serde_json", + "tokio", + "tracing-subscriber", +] + [[package]] name = "openssh" version = "0.11.5" diff --git a/core/Cargo.toml b/core/Cargo.toml index f1c5aaf4a..5c8c363a9 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -17,7 +17,7 @@ [workspace] default-members = [".", "core"] -members = [".", "core", "examples/*", "fuzz", "edge/*", "benches/vs_*"] +members = [".", "core", "examples/*", "fuzz", "edge/*", "benches/vs_*", "services/s3"] [workspace.package] edition = "2024" @@ -116,7 +116,8 @@ services-redb = ["opendal-core/services-redb"] services-redis = ["opendal-core/services-redis"] services-redis-native-tls = ["opendal-core/services-redis-native-tls"] services-rocksdb = ["opendal-core/services-rocksdb"] -services-s3 = ["opendal-core/services-s3"] +services-s3 = ["dep:opendal-service-s3"] +service-s3 = ["services-s3"] services-seafile = ["opendal-core/services-seafile"] services-sftp = ["opendal-core/services-sftp"] services-sled = ["opendal-core/services-sled"] @@ -152,6 +153,7 @@ required-features = ["tests"] [dependencies] opendal-core = { path = "core", version = "0.55.0", default-features = false } +opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional = true, default-features = false } [dev-dependencies] anyhow = { version = "1.0.100", features = ["std"] } diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml index 4082d9185..0f6edc6d5 100644 --- a/core/core/Cargo.toml +++ b/core/core/Cargo.toml @@ -62,7 +62,6 @@ tests = [ "services-http", "services-memory", "internal-tokio-rt", - "services-s3", ] # Enable path cache. @@ -191,13 +190,6 @@ services-redb = ["dep:redb", "internal-tokio-rt"] services-redis = ["dep:redis", "dep:fastpool", "redis?/tokio-rustls-comp"] services-redis-native-tls = ["services-redis", "redis?/tokio-native-tls-comp"] services-rocksdb = ["dep:rocksdb", "internal-tokio-rt"] -services-s3 = [ - "dep:reqsign-core", - "dep:reqsign-aws-v4", - "dep:reqsign-file-read-tokio", - "dep:reqsign-http-send-reqwest", - "dep:crc32c", -] services-seafile = [] services-sftp = ["dep:openssh", "dep:openssh-sftp-client", "dep:fastpool"] services-sled = ["dep:sled", "internal-tokio-rt"] @@ -262,11 +254,6 @@ sqlx = { version = "0.8.0", features = [ # For http based services. reqsign = { version = "0.16.5", default-features = false, optional = true } -# For S3 service migration to v1 -reqsign-aws-v4 = { version = "2.0.1", default-features = false, optional = true } -reqsign-core = { version = "2.0.1", default-features = false, optional = true } -reqsign-file-read-tokio = { version = "2.0.1", default-features = false, optional = true } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false, optional = true } # for self-referencing structs ouroboros = { version = "0.18.4", optional = true } @@ -339,8 +326,6 @@ compio = { version = "0.16.0", optional = true, features = [ "polling", "dispatcher", ] } -# for services-s3 -crc32c = { version = "0.6.6", optional = true } # for services-monoiofs flume = { version = "0.11", optional = true } monoio = { version = "0.2.4", optional = true, features = [ diff --git a/core/core/src/blocking/operator.rs b/core/core/src/blocking/operator.rs index 27f0190be..0f6fa89cb 100644 --- a/core/core/src/blocking/operator.rs +++ b/core/core/src/blocking/operator.rs @@ -45,7 +45,7 @@ use crate::*; /// #[tokio::main] /// async fn main() -> Result<()> { /// // Create fs backend builder. -/// let mut builder = services::S3::default().bucket("test").region("us-east-1"); +/// let builder = services::Memory::default(); /// let op = Operator::new(builder)?.finish(); /// /// // Build an `blocking::Operator` with blocking layer to start operating the storage. @@ -75,7 +75,7 @@ use crate::*; /// /// fn blocking_fn() -> Result<blocking::Operator> { /// // Create fs backend builder. -/// let mut builder = services::S3::default().bucket("test").region("us-east-1"); +/// let builder = services::Memory::default(); /// let op = Operator::new(builder)?.finish(); /// /// let handle = tokio::runtime::Handle::try_current().unwrap(); @@ -109,7 +109,7 @@ use crate::*; /// /// fn main() -> Result<()> { /// // Create fs backend builder. -/// let mut builder = services::S3::default().bucket("test").region("us-east-1"); +/// let builder = services::Memory::default(); /// let op = Operator::new(builder)?.finish(); /// /// // Fetch the `EnterGuard` from global runtime. diff --git a/core/core/src/docs/concepts.rs b/core/core/src/docs/concepts.rs index 9f8a0b179..d208b7a62 100644 --- a/core/core/src/docs/concepts.rs +++ b/core/core/src/docs/concepts.rs @@ -45,18 +45,16 @@ //! └───────────┘ └───────────┘ //! ``` //! -//! All [`Builder`] provided by OpenDAL is under [`services`][crate::services], we can refer to them like `opendal::services::S3`. +//! All [`Builder`] provided by OpenDAL is under [`services`][crate::services], we can refer to them like `opendal::services::Memory`. //! By right the builder will be named like `OneServiceBuilder`, but usually we will export it to public with renaming it as one //! general name. For example, we will rename `S3Builder` to `S3` and developer will use `S3` finally. //! //! For example: //! //! ```no_run -//! use opendal_core::services::S3; +//! use opendal_core::services::Memory; //! -//! let mut builder = S3::default(); -//! builder.bucket("example"); -//! builder.root("/path/to/file"); +//! let builder = Memory::default(); //! ``` //! //! # Operator @@ -79,13 +77,11 @@ //! //! ```no_run //! # use opendal_core::Result; -//! use opendal_core::services::S3; +//! use opendal_core::services::Memory; //! use opendal_core::Operator; //! //! # fn test() -> Result<()> { -//! let mut builder = S3::default(); -//! builder.bucket("example"); -//! builder.root("/path/to/file"); +//! let builder = Memory::default(); //! //! let op = Operator::new(builder)?.finish(); //! # Ok(()) @@ -117,13 +113,11 @@ //! //! ```no_run //! # use opendal_core::Result; -//! use opendal_core::services::S3; +//! use opendal_core::services::Memory; //! use opendal_core::Operator; //! //! # async fn test() -> Result<()> { -//! let mut builder = S3::default(); -//! builder.bucket("example"); -//! builder.root("/path/to/file"); +//! let builder = Memory::default(); //! //! let op = Operator::new(builder)?.finish(); //! let bs: Vec<u8> = op.read("abc").await?; diff --git a/core/core/src/layers/http_client.rs b/core/core/src/layers/http_client.rs index 92e57ac44..1b1eed4b0 100644 --- a/core/core/src/layers/http_client.rs +++ b/core/core/src/layers/http_client.rs @@ -38,7 +38,7 @@ use crate::*; /// // Create a custom HTTP client /// let custom_client = HttpClient::new()?; /// -/// let op = Operator::new(services::S3::default())? +/// let op = Operator::new(services::Memory::default())? /// .layer(HttpClientLayer::new(custom_client)) /// .finish(); /// # Ok(()) diff --git a/core/core/src/lib.rs b/core/core/src/lib.rs index 1fb46bc53..fb7b08d05 100644 --- a/core/core/src/lib.rs +++ b/core/core/src/lib.rs @@ -38,7 +38,7 @@ //! The first step is to pick a service and init it with a builder. All supported //! services could be found at [`services`]. //! -//! Let's take [`services::S3`] as an example: +//! Let's take [`services::Memory`] as an example: //! //! ```no_run //! use opendal_core::services; @@ -47,7 +47,7 @@ //! //! fn main() -> Result<()> { //! // Pick a builder and configure it. -//! let mut builder = services::S3::default().bucket("test"); +//! let builder = services::Memory::default(); //! //! // Init an operator //! let op = Operator::new(builder)?.finish(); @@ -72,7 +72,7 @@ //! #[tokio::main] //! async fn main() -> Result<()> { //! // Pick a builder and configure it. -//! let mut builder = services::S3::default().bucket("test"); +//! let builder = services::Memory::default(); //! //! // Init an operator //! let op = Operator::new(builder)? @@ -111,7 +111,7 @@ //! #[tokio::main] //! async fn main() -> Result<()> { //! // Pick a builder and configure it. -//! let mut builder = services::S3::default().bucket("test"); +//! let builder = services::Memory::default(); //! //! // Init an operator //! let op = Operator::new(builder)? diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs index 57cd81b4d..dd232fa2a 100644 --- a/core/core/src/services/mod.rs +++ b/core/core/src/services/mod.rs @@ -254,11 +254,6 @@ mod rocksdb; #[cfg(feature = "services-rocksdb")] pub use self::rocksdb::*; -#[cfg(feature = "services-s3")] -mod s3; -#[cfg(feature = "services-s3")] -pub use s3::*; - #[cfg(feature = "services-seafile")] mod seafile; #[cfg(feature = "services-seafile")] diff --git a/core/core/src/types/builder.rs b/core/core/src/types/builder.rs index 99f84d70a..006704e12 100644 --- a/core/core/src/types/builder.rs +++ b/core/core/src/types/builder.rs @@ -102,14 +102,13 @@ impl Builder for () { /// use std::collections::HashMap; /// /// use opendal_core::raw::HttpClient; -/// use opendal_core::services::S3Config; +/// use opendal_core::services::MemoryConfig; /// use opendal_core::Configurator; /// use opendal_core::Operator; /// /// async fn test() -> Result<()> { -/// let mut cfg = S3Config::default(); +/// let mut cfg = MemoryConfig::default(); /// cfg.root = Some("/".to_string()); -/// cfg.bucket = "test".to_string(); /// /// let builder = cfg.into_builder(); /// let builder = builder.http_client(HttpClient::new()?); diff --git a/core/core/src/types/operator/operator.rs b/core/core/src/types/operator/operator.rs index 5b657fc2a..d9cde1c67 100644 --- a/core/core/src/types/operator/operator.rs +++ b/core/core/src/types/operator/operator.rs @@ -39,8 +39,8 @@ use crate::*; /// /// Users can initialize an `Operator` through the following methods: /// -/// - [`Operator::new`]: Creates an operator using a [`services`] builder, such as [`services::S3`]. -/// - [`Operator::from_config`]: Creates an operator using a [`services`] configuration, such as [`services::S3Config`]. +/// - [`Operator::new`]: Creates an operator using a [`services`] builder, such as [`services::Memory`]. +/// - [`Operator::from_config`]: Creates an operator using a [`services`] configuration, such as [`services::MemoryConfig`]. /// - [`Operator::from_iter`]: Creates an operator from an iterator of configuration key-value pairs. /// /// ``` @@ -110,7 +110,7 @@ use crate::*; /// #[tokio::main] /// async fn main() -> Result<()> { /// // Pick a builder and configure it. -/// let mut builder = services::S3::default().bucket("test"); +/// let builder = services::Memory::default(); /// /// // Init an operator /// let op = Operator::new(builder)? diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml new file mode 100644 index 000000000..cb8655657 --- /dev/null +++ b/core/services/s3/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "opendal-service-s3" +version = "0.55.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/apache/opendal"; +description = "Apache OpenDAL S3 service implementation" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +base64 = "0.22" +bytes = "1.6" +crc32c = "0.6.6" +ctor = "0.6" +http = "1.1" +log = "0.4" +md-5 = "0.10" +quick-xml = { version = "0.38", features = ["serialize", "overlapped-lists"] } +reqsign-aws-v4 = { version = "2.0.1", default-features = false } +reqsign-core = { version = "2.0.1", default-features = false } +reqsign-file-read-tokio = { version = "2.0.1", default-features = false } +reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } +reqwest = { version = "0.12.24", features = ["stream"], default-features = false } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1.48", features = ["macros", "rt-multi-thread", "io-util"] } + +[dev-dependencies] +pretty_assertions = "1" +tracing-subscriber = "0.3" diff --git a/core/core/src/services/s3/backend.rs b/core/services/s3/src/backend.rs similarity index 97% rename from core/core/src/services/s3/backend.rs rename to core/services/s3/src/backend.rs index 3c619055a..dc3f0295d 100644 --- a/core/core/src/services/s3/backend.rs +++ b/core/services/s3/src/backend.rs @@ -45,19 +45,21 @@ use reqsign_file_read_tokio::TokioFileRead; use reqsign_http_send_reqwest::ReqwestHttpSend; use reqwest::Url; -use super::S3_SCHEME; -use super::config::S3Config; -use super::core::*; -use super::deleter::S3Deleter; -use super::error::parse_error; -use super::lister::S3ListerV1; -use super::lister::S3ListerV2; -use super::lister::S3Listers; -use super::lister::S3ObjectVersionsLister; -use super::writer::S3Writer; -use super::writer::S3Writers; -use crate::raw::*; -use crate::*; +use crate::S3_SCHEME; +use crate::config::S3Config; +use crate::core::*; +use crate::deleter::S3Deleter; +use crate::error::parse_error; +use crate::lister::S3ListerV1; +use crate::lister::S3ListerV2; +use crate::lister::S3Listers; +use crate::lister::S3ObjectVersionsLister; +use crate::writer::S3Writer; +use crate::writer::S3Writers; +use opendal_core::raw::*; +use opendal_core::*; + +static GLOBAL_REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(reqwest::Client::new); /// Allow constructing correct region endpoint if user gives a global endpoint. static ENDPOINT_TEMPLATES: LazyLock<HashMap<&'static str, &'static str>> = LazyLock::new(|| { @@ -604,7 +606,7 @@ impl S3Builder { /// # Examples /// /// ```no_run - /// use opendal_core::services::S3; + /// use opendal_service_s3::S3; /// /// # async fn example() { /// let region: Option<String> = S3::detect_region("https://s3.amazonaws.com";, "example").await; @@ -642,10 +644,11 @@ impl S3Builder { } // If this bucket is AWS, we can try to match the endpoint. - if let Some(v) = endpoint.strip_prefix("https://s3.";) { - if let Some(region) = v.strip_suffix(".amazonaws.com") { - return Some(region.to_string()); - } + if let Some(region) = endpoint + .strip_prefix("https://s3.";) + .and_then(|v| v.strip_suffix(".amazonaws.com")) + { + return Some(region.to_string()); } // If this bucket is OSS, we can try to match the endpoint. @@ -679,10 +682,12 @@ impl S3Builder { ); // Get region from response header no matter status code. - if let Some(header) = res.headers().get("x-amz-bucket-region") { - if let Ok(regin) = header.to_str() { - return Some(regin.to_string()); - } + if let Some(region) = res + .headers() + .get("x-amz-bucket-region") + .and_then(|header| header.to_str().ok()) + { + return Some(region.to_string()); } // Status code is 403 or 200 means we already visit the correct @@ -1119,6 +1124,10 @@ impl Access for S3Backend { ErrorKind::Unsupported, "operation is not supported", )), + _ => Err(Error::new( + ErrorKind::Unsupported, + "operation is not supported", + )), }; let req = req?; diff --git a/core/core/src/services/s3/compatible_services.md b/core/services/s3/src/compatible_services.md similarity index 100% rename from core/core/src/services/s3/compatible_services.md rename to core/services/s3/src/compatible_services.md diff --git a/core/core/src/services/s3/config.rs b/core/services/s3/src/config.rs similarity index 98% rename from core/core/src/services/s3/config.rs rename to core/services/s3/src/config.rs index e23c1bce2..9b2761e01 100644 --- a/core/core/src/services/s3/config.rs +++ b/core/services/s3/src/config.rs @@ -17,10 +17,13 @@ use std::fmt::Debug; +use opendal_core::Configurator; +use opendal_core::OperatorUri; +use opendal_core::Result; use serde::Deserialize; use serde::Serialize; -use super::backend::S3Builder; +use crate::backend::S3Builder; /// Config for Aws S3 and compatible services (including minio, digitalocean space, /// Tencent Cloud Object Storage(COS) and so on) support. @@ -231,10 +234,10 @@ impl Debug for S3Config { } } -impl crate::Configurator for S3Config { +impl Configurator for S3Config { type Builder = S3Builder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> { + fn from_uri(uri: &OperatorUri) -> Result<Self> { let mut map = uri.options().clone(); if let Some(name) = uri.name() { @@ -263,8 +266,8 @@ mod tests { use std::iter; use super::*; - use crate::Configurator; - use crate::types::OperatorUri; + use opendal_core::Configurator; + use opendal_core::OperatorUri; #[test] fn test_s3_config_original_field_names() { diff --git a/core/core/src/services/s3/core.rs b/core/services/s3/src/core.rs similarity index 99% rename from core/core/src/services/s3/core.rs rename to core/services/s3/src/core.rs index 6699dd2bf..03e14483a 100644 --- a/core/core/src/services/s3/core.rs +++ b/core/services/s3/src/core.rs @@ -43,8 +43,8 @@ use reqsign_core::Signer; use serde::Deserialize; use serde::Serialize; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub mod constants { pub const X_AMZ_COPY_SOURCE: &str = "x-amz-copy-source"; diff --git a/core/core/src/services/s3/deleter.rs b/core/services/s3/src/deleter.rs similarity index 95% rename from core/core/src/services/s3/deleter.rs rename to core/services/s3/src/deleter.rs index 9d8ae601d..feb9ff90b 100644 --- a/core/core/src/services/s3/deleter.rs +++ b/core/services/s3/src/deleter.rs @@ -20,12 +20,12 @@ use std::sync::Arc; use bytes::Buf; use http::StatusCode; -use super::core::*; -use super::error::parse_error; -use super::error::parse_s3_error_code; -use crate::raw::oio::BatchDeleteResult; -use crate::raw::*; -use crate::*; +use crate::core::*; +use crate::error::parse_error; +use crate::error::parse_s3_error_code; +use opendal_core::raw::oio::BatchDeleteResult; +use opendal_core::raw::*; +use opendal_core::*; pub struct S3Deleter { core: Arc<S3Core>, diff --git a/core/core/src/services/s3/docs.md b/core/services/s3/src/docs.md similarity index 97% rename from core/core/src/services/s3/docs.md rename to core/services/s3/src/docs.md index 5a7ab9686..66c2b1d3f 100644 --- a/core/core/src/services/s3/docs.md +++ b/core/services/s3/src/docs.md @@ -81,7 +81,7 @@ Reference: [Protecting data using server-side encryption](https://docs.aws.amazo use std::sync::Arc; use anyhow::Result; -use opendal_core::services::S3; +use opendal_service_s3::S3; use opendal_core::Operator; #[tokio::main] @@ -126,7 +126,7 @@ async fn main() -> Result<()> { ```rust,no_run use anyhow::Result; use log::info; -use opendal_core::services::S3; +use opendal_service_s3::S3; use opendal_core::Operator; #[tokio::main] @@ -155,7 +155,7 @@ async fn main() -> Result<()> { ```rust,no_run use anyhow::Result; use log::info; -use opendal_core::services::S3; +use opendal_service_s3::S3; use opendal_core::Operator; #[tokio::main] @@ -185,7 +185,7 @@ async fn main() -> Result<()> { ```rust,no_run use anyhow::Result; use log::info; -use opendal_core::services::S3; +use opendal_service_s3::S3; use opendal_core::Operator; #[tokio::main] @@ -215,7 +215,7 @@ async fn main() -> Result<()> { ```rust,no_run use anyhow::Result; use log::info; -use opendal_core::services::S3; +use opendal_service_s3::S3; use opendal_core::Operator; #[tokio::main] diff --git a/core/core/src/services/s3/error.rs b/core/services/s3/src/error.rs similarity index 99% rename from core/core/src/services/s3/error.rs rename to core/services/s3/src/error.rs index 3244abb44..da675c6de 100644 --- a/core/core/src/services/s3/error.rs +++ b/core/services/s3/src/error.rs @@ -21,8 +21,8 @@ use http::response::Parts; use quick_xml::de; use serde::Deserialize; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; /// S3Error is the error returned by s3 service. #[derive(Default, Debug, Deserialize, PartialEq, Eq)] diff --git a/core/core/src/services/s3/mod.rs b/core/services/s3/src/lib.rs similarity index 84% copy from core/core/src/services/s3/mod.rs copy to core/services/s3/src/lib.rs index 432f8f45e..57b39c419 100644 --- a/core/core/src/services/s3/mod.rs +++ b/core/services/s3/src/lib.rs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -/// Default scheme for s3 service. -pub const S3_SCHEME: &str = "s3"; - -use crate::types::DEFAULT_OPERATOR_REGISTRY; +#![cfg_attr(docsrs, feature(doc_cfg))] +//! Amazon S3 service implementation for Apache OpenDAL. +#![deny(missing_docs)] mod backend; mod config; @@ -31,7 +30,10 @@ mod writer; pub use backend::S3Builder as S3; pub use config::S3Config; +/// Default scheme for s3 service. +pub const S3_SCHEME: &str = "s3"; + #[ctor::ctor] fn register_s3_service() { - DEFAULT_OPERATOR_REGISTRY.register::<S3>(S3_SCHEME); + opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<S3>(S3_SCHEME); } diff --git a/core/core/src/services/s3/lister.rs b/core/services/s3/src/lister.rs similarity index 94% rename from core/core/src/services/s3/lister.rs rename to core/services/s3/src/lister.rs index ae15788ff..55a7c1821 100644 --- a/core/core/src/services/s3/lister.rs +++ b/core/services/s3/src/lister.rs @@ -20,14 +20,14 @@ use std::sync::Arc; use bytes::Buf; use quick_xml::de; -use super::core::*; -use super::error::parse_error; -use crate::EntryMode; -use crate::Error; -use crate::Metadata; -use crate::Result; -use crate::raw::oio::PageContext; -use crate::raw::*; +use crate::core::*; +use crate::error::parse_error; +use opendal_core::EntryMode; +use opendal_core::Error; +use opendal_core::Metadata; +use opendal_core::Result; +use opendal_core::raw::oio::PageContext; +use opendal_core::raw::*; pub type S3Listers = ThreeWays< oio::PageLister<S3ListerV1>, @@ -138,7 +138,12 @@ impl oio::PageList for S3ListerV1 { path = "/".to_string(); } - let mut meta = Metadata::new(EntryMode::from_path(&path)); + let mode = if path.ends_with('/') { + EntryMode::DIR + } else { + EntryMode::FILE + }; + let mut meta = Metadata::new(mode); meta.set_is_current(true); if let Some(etag) = &object.etag { meta.set_etag(etag); @@ -248,7 +253,12 @@ impl oio::PageList for S3ListerV2 { path = "/".to_string(); } - let mut meta = Metadata::new(EntryMode::from_path(&path)); + let mode = if path.ends_with('/') { + EntryMode::DIR + } else { + EntryMode::FILE + }; + let mut meta = Metadata::new(mode); meta.set_is_current(true); if let Some(etag) = &object.etag { meta.set_etag(etag); @@ -364,7 +374,12 @@ impl oio::PageList for S3ObjectVersionsLister { path = "/".to_owned(); } - let mut meta = Metadata::new(EntryMode::from_path(&path)); + let mode = if path.ends_with('/') { + EntryMode::DIR + } else { + EntryMode::FILE + }; + let mut meta = Metadata::new(mode); meta.set_version(&version_object.version_id); meta.set_is_current(version_object.is_latest); meta.set_content_length(version_object.size); diff --git a/core/core/src/services/s3/mod.rs b/core/services/s3/src/mod.rs similarity index 96% rename from core/core/src/services/s3/mod.rs rename to core/services/s3/src/mod.rs index 432f8f45e..5407b1653 100644 --- a/core/core/src/services/s3/mod.rs +++ b/core/services/s3/src/mod.rs @@ -18,7 +18,7 @@ /// Default scheme for s3 service. pub const S3_SCHEME: &str = "s3"; -use crate::types::DEFAULT_OPERATOR_REGISTRY; +use opendal_core::DEFAULT_OPERATOR_REGISTRY; mod backend; mod config; diff --git a/core/core/src/services/s3/writer.rs b/core/services/s3/src/writer.rs similarity index 93% rename from core/core/src/services/s3/writer.rs rename to core/services/s3/src/writer.rs index bd73388fb..1732e0979 100644 --- a/core/core/src/services/s3/writer.rs +++ b/core/services/s3/src/writer.rs @@ -22,12 +22,12 @@ use constants::X_AMZ_OBJECT_SIZE; use constants::X_AMZ_VERSION_ID; use http::StatusCode; -use super::core::*; -use super::error::S3Error; -use super::error::from_s3_error; -use super::error::parse_error; -use crate::raw::*; -use crate::*; +use crate::core::*; +use crate::error::S3Error; +use crate::error::from_s3_error; +use crate::error::parse_error; +use opendal_core::raw::*; +use opendal_core::*; pub type S3Writers = TwoWays<oio::MultipartWriter<S3Writer>, oio::AppendWriter<S3Writer>>; @@ -48,17 +48,22 @@ impl S3Writer { } fn parse_header_into_meta(path: &str, headers: &http::HeaderMap) -> Result<Metadata> { - let mut meta = Metadata::new(EntryMode::from_path(path)); + let mode = if path.ends_with('/') { + EntryMode::DIR + } else { + EntryMode::FILE + }; + let mut meta = Metadata::new(mode); if let Some(etag) = parse_etag(headers)? { meta.set_etag(etag); } if let Some(version) = parse_header_to_str(headers, X_AMZ_VERSION_ID)? { meta.set_version(version); } - if let Some(size) = parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)? { - if let Ok(value) = size.parse() { - meta.set_content_length(value); - } + if let Some(value) = + parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)?.and_then(|size| size.parse().ok()) + { + meta.set_content_length(value); } Ok(meta) } diff --git a/core/src/lib.rs b/core/src/lib.rs index 381539fb8..e02ba8c9c 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -19,7 +19,14 @@ html_logo_url = "https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"; )] #![cfg_attr(docsrs, feature(doc_cfg))] -//! Facade crate that re-exports all public APIs from `opendal-core`. +//! Facade crate that re-exports all public APIs from `opendal-core` and optional services/layers. #![deny(missing_docs)] pub use opendal_core::*; + +/// Re-export of service implementations. +pub mod services { + pub use opendal_core::services::*; + #[cfg(feature = "services-s3")] + pub use opendal_service_s3::*; +}
![https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"](https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"){kind=link}