This is an automated email from the ASF dual-hosted git repository.
asukaminato pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new fa51d69a3 refactor: Move services s3 out as a crate (#6866)
fa51d69a3 is described below
commit fa51d69a302540f4d28ec478efc7f76c9dfcae87
Author: Xuanwo <[email protected]>
AuthorDate: Fri Dec 5 02:03:41 2025 +0800
refactor: Move services s3 out as a crate (#6866)
* refactor: Move services s3 out as a crate
Signed-off-by: Xuanwo <[email protected]>
* Remove wrong features
Signed-off-by: Xuanwo <[email protected]>
* Fix build
Signed-off-by: Xuanwo <[email protected]>
* Fix license
Signed-off-by: Xuanwo <[email protected]>
* Fix tests
Signed-off-by: Xuanwo <[email protected]>
* revert not needed changes
Signed-off-by: Xuanwo <[email protected]>
* remove not used tests
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
core/Cargo.lock | 31 +++++++++++--
core/Cargo.toml | 5 +-
core/core/Cargo.toml | 15 ------
core/core/src/blocking/operator.rs | 6 +--
core/core/src/docs/concepts.rs | 20 +++-----
core/core/src/layers/http_client.rs | 2 +-
core/core/src/lib.rs | 8 ++--
core/core/src/services/mod.rs | 5 --
core/core/src/types/builder.rs | 27 -----------
core/core/src/types/mode.rs | 2 +-
core/core/src/types/operator/operator.rs | 6 +--
.../Cargo.toml | 2 +-
core/services/s3/Cargo.toml | 50 ++++++++++++++++++++
.../src/services/s3 => services/s3/src}/backend.rs | 53 +++++++++++++---------
.../s3 => services/s3/src}/compatible_services.md | 0
.../src/services/s3 => services/s3/src}/config.rs | 13 ++++--
.../src/services/s3 => services/s3/src}/core.rs | 4 +-
.../src/services/s3 => services/s3/src}/deleter.rs | 12 ++---
.../src/services/s3 => services/s3/src}/docs.md | 10 ++--
.../src/services/s3 => services/s3/src}/error.rs | 4 +-
.../services/s3/mod.rs => services/s3/src/lib.rs} | 12 +++--
.../src/services/s3 => services/s3/src}/lister.rs | 16 +++----
.../src/services/s3 => services/s3/src}/mod.rs | 2 +-
.../src/services/s3 => services/s3/src}/writer.rs | 20 ++++----
core/src/lib.rs | 9 +++-
25 files changed, 187 insertions(+), 147 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 9c7b5927e..0dd01bee2 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5406,6 +5406,7 @@ dependencies = [
"libtest-mimic",
"log",
"opendal-core",
+ "opendal-service-s3",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
@@ -5458,7 +5459,6 @@ dependencies = [
"bytes",
"cacache",
"compio",
- "crc32c",
"criterion",
"ctor",
"dashmap 6.1.0",
@@ -5512,10 +5512,6 @@ dependencies = [
"redb",
"redis",
"reqsign",
- "reqsign-aws-v4",
- "reqsign-core",
- "reqsign-file-read-tokio",
- "reqsign-http-send-reqwest",
"reqwest",
"rocksdb",
"rustls-native-certs 0.8.2",
@@ -5580,6 +5576,31 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "opendal-service-s3"
+version = "0.55.0"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "crc32c",
+ "ctor",
+ "http 1.3.1",
+ "log",
+ "md-5",
+ "opendal-core",
+ "pretty_assertions",
+ "quick-xml",
+ "reqsign-aws-v4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-http-send-reqwest",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tracing-subscriber",
+]
+
[[package]]
name = "openssh"
version = "0.11.5"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index f1c5aaf4a..0ca702199 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -17,7 +17,7 @@
[workspace]
default-members = [".", "core"]
-members = [".", "core", "examples/*", "fuzz", "edge/*", "benches/vs_*"]
+members = [".", "core", "examples/*", "fuzz", "edge/*", "benches/vs_*",
"services/*"]
[workspace.package]
edition = "2024"
@@ -116,7 +116,7 @@ services-redb = ["opendal-core/services-redb"]
services-redis = ["opendal-core/services-redis"]
services-redis-native-tls = ["opendal-core/services-redis-native-tls"]
services-rocksdb = ["opendal-core/services-rocksdb"]
-services-s3 = ["opendal-core/services-s3"]
+services-s3 = ["dep:opendal-service-s3"]
services-seafile = ["opendal-core/services-seafile"]
services-sftp = ["opendal-core/services-sftp"]
services-sled = ["opendal-core/services-sled"]
@@ -152,6 +152,7 @@ required-features = ["tests"]
[dependencies]
opendal-core = { path = "core", version = "0.55.0", default-features = false }
+opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional =
true, default-features = false }
[dev-dependencies]
anyhow = { version = "1.0.100", features = ["std"] }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index 4082d9185..0f6edc6d5 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -62,7 +62,6 @@ tests = [
"services-http",
"services-memory",
"internal-tokio-rt",
- "services-s3",
]
# Enable path cache.
@@ -191,13 +190,6 @@ services-redb = ["dep:redb", "internal-tokio-rt"]
services-redis = ["dep:redis", "dep:fastpool", "redis?/tokio-rustls-comp"]
services-redis-native-tls = ["services-redis", "redis?/tokio-native-tls-comp"]
services-rocksdb = ["dep:rocksdb", "internal-tokio-rt"]
-services-s3 = [
- "dep:reqsign-core",
- "dep:reqsign-aws-v4",
- "dep:reqsign-file-read-tokio",
- "dep:reqsign-http-send-reqwest",
- "dep:crc32c",
-]
services-seafile = []
services-sftp = ["dep:openssh", "dep:openssh-sftp-client", "dep:fastpool"]
services-sled = ["dep:sled", "internal-tokio-rt"]
@@ -262,11 +254,6 @@ sqlx = { version = "0.8.0", features = [
# For http based services.
reqsign = { version = "0.16.5", default-features = false, optional = true }
-# For S3 service migration to v1
-reqsign-aws-v4 = { version = "2.0.1", default-features = false, optional =
true }
-reqsign-core = { version = "2.0.1", default-features = false, optional = true }
-reqsign-file-read-tokio = { version = "2.0.1", default-features = false,
optional = true }
-reqsign-http-send-reqwest = { version = "2.0.1", default-features = false,
optional = true }
# for self-referencing structs
ouroboros = { version = "0.18.4", optional = true }
@@ -339,8 +326,6 @@ compio = { version = "0.16.0", optional = true, features = [
"polling",
"dispatcher",
] }
-# for services-s3
-crc32c = { version = "0.6.6", optional = true }
# for services-monoiofs
flume = { version = "0.11", optional = true }
monoio = { version = "0.2.4", optional = true, features = [
diff --git a/core/core/src/blocking/operator.rs
b/core/core/src/blocking/operator.rs
index 27f0190be..0f6fa89cb 100644
--- a/core/core/src/blocking/operator.rs
+++ b/core/core/src/blocking/operator.rs
@@ -45,7 +45,7 @@ use crate::*;
/// #[tokio::main]
/// async fn main() -> Result<()> {
/// // Create fs backend builder.
-/// let mut builder =
services::S3::default().bucket("test").region("us-east-1");
+/// let builder = services::Memory::default();
/// let op = Operator::new(builder)?.finish();
///
/// // Build an `blocking::Operator` with blocking layer to start
operating the storage.
@@ -75,7 +75,7 @@ use crate::*;
///
/// fn blocking_fn() -> Result<blocking::Operator> {
/// // Create fs backend builder.
-/// let mut builder =
services::S3::default().bucket("test").region("us-east-1");
+/// let builder = services::Memory::default();
/// let op = Operator::new(builder)?.finish();
///
/// let handle = tokio::runtime::Handle::try_current().unwrap();
@@ -109,7 +109,7 @@ use crate::*;
///
/// fn main() -> Result<()> {
/// // Create fs backend builder.
-/// let mut builder =
services::S3::default().bucket("test").region("us-east-1");
+/// let builder = services::Memory::default();
/// let op = Operator::new(builder)?.finish();
///
/// // Fetch the `EnterGuard` from global runtime.
diff --git a/core/core/src/docs/concepts.rs b/core/core/src/docs/concepts.rs
index 9f8a0b179..d208b7a62 100644
--- a/core/core/src/docs/concepts.rs
+++ b/core/core/src/docs/concepts.rs
@@ -45,18 +45,16 @@
//! └───────────┘ └───────────┘
//! ```
//!
-//! All [`Builder`] provided by OpenDAL is under
[`services`][crate::services], we can refer to them like
`opendal::services::S3`.
+//! All [`Builder`] provided by OpenDAL is under
[`services`][crate::services], we can refer to them like
`opendal::services::Memory`.
//! By right the builder will be named like `OneServiceBuilder`, but usually
we will export it to public with renaming it as one
//! general name. For example, we will rename `S3Builder` to `S3` and
developer will use `S3` finally.
//!
//! For example:
//!
//! ```no_run
-//! use opendal_core::services::S3;
+//! use opendal_core::services::Memory;
//!
-//! let mut builder = S3::default();
-//! builder.bucket("example");
-//! builder.root("/path/to/file");
+//! let builder = Memory::default();
//! ```
//!
//! # Operator
@@ -79,13 +77,11 @@
//!
//! ```no_run
//! # use opendal_core::Result;
-//! use opendal_core::services::S3;
+//! use opendal_core::services::Memory;
//! use opendal_core::Operator;
//!
//! # fn test() -> Result<()> {
-//! let mut builder = S3::default();
-//! builder.bucket("example");
-//! builder.root("/path/to/file");
+//! let builder = Memory::default();
//!
//! let op = Operator::new(builder)?.finish();
//! # Ok(())
@@ -117,13 +113,11 @@
//!
//! ```no_run
//! # use opendal_core::Result;
-//! use opendal_core::services::S3;
+//! use opendal_core::services::Memory;
//! use opendal_core::Operator;
//!
//! # async fn test() -> Result<()> {
-//! let mut builder = S3::default();
-//! builder.bucket("example");
-//! builder.root("/path/to/file");
+//! let builder = Memory::default();
//!
//! let op = Operator::new(builder)?.finish();
//! let bs: Vec<u8> = op.read("abc").await?;
diff --git a/core/core/src/layers/http_client.rs
b/core/core/src/layers/http_client.rs
index 92e57ac44..1b1eed4b0 100644
--- a/core/core/src/layers/http_client.rs
+++ b/core/core/src/layers/http_client.rs
@@ -38,7 +38,7 @@ use crate::*;
/// // Create a custom HTTP client
/// let custom_client = HttpClient::new()?;
///
-/// let op = Operator::new(services::S3::default())?
+/// let op = Operator::new(services::Memory::default())?
/// .layer(HttpClientLayer::new(custom_client))
/// .finish();
/// # Ok(())
diff --git a/core/core/src/lib.rs b/core/core/src/lib.rs
index 1fb46bc53..fb7b08d05 100644
--- a/core/core/src/lib.rs
+++ b/core/core/src/lib.rs
@@ -38,7 +38,7 @@
//! The first step is to pick a service and init it with a builder. All
supported
//! services could be found at [`services`].
//!
-//! Let's take [`services::S3`] as an example:
+//! Let's take [`services::Memory`] as an example:
//!
//! ```no_run
//! use opendal_core::services;
@@ -47,7 +47,7 @@
//!
//! fn main() -> Result<()> {
//! // Pick a builder and configure it.
-//! let mut builder = services::S3::default().bucket("test");
+//! let builder = services::Memory::default();
//!
//! // Init an operator
//! let op = Operator::new(builder)?.finish();
@@ -72,7 +72,7 @@
//! #[tokio::main]
//! async fn main() -> Result<()> {
//! // Pick a builder and configure it.
-//! let mut builder = services::S3::default().bucket("test");
+//! let builder = services::Memory::default();
//!
//! // Init an operator
//! let op = Operator::new(builder)?
@@ -111,7 +111,7 @@
//! #[tokio::main]
//! async fn main() -> Result<()> {
//! // Pick a builder and configure it.
-//! let mut builder = services::S3::default().bucket("test");
+//! let builder = services::Memory::default();
//!
//! // Init an operator
//! let op = Operator::new(builder)?
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index 57cd81b4d..dd232fa2a 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -254,11 +254,6 @@ mod rocksdb;
#[cfg(feature = "services-rocksdb")]
pub use self::rocksdb::*;
-#[cfg(feature = "services-s3")]
-mod s3;
-#[cfg(feature = "services-s3")]
-pub use s3::*;
-
#[cfg(feature = "services-seafile")]
mod seafile;
#[cfg(feature = "services-seafile")]
diff --git a/core/core/src/types/builder.rs b/core/core/src/types/builder.rs
index 99f84d70a..2c0ec5e6c 100644
--- a/core/core/src/types/builder.rs
+++ b/core/core/src/types/builder.rs
@@ -93,33 +93,6 @@ impl Builder for () {
/// Ok(())
/// }
/// ```
-///
-/// Some service builder might contain in memory options like `http_client` .
Users can call
-/// `into_builder` to convert the configuration into a builder instead.
-///
-/// ```
-/// # use anyhow::Result;
-/// use std::collections::HashMap;
-///
-/// use opendal_core::raw::HttpClient;
-/// use opendal_core::services::S3Config;
-/// use opendal_core::Configurator;
-/// use opendal_core::Operator;
-///
-/// async fn test() -> Result<()> {
-/// let mut cfg = S3Config::default();
-/// cfg.root = Some("/".to_string());
-/// cfg.bucket = "test".to_string();
-///
-/// let builder = cfg.into_builder();
-/// let builder = builder.http_client(HttpClient::new()?);
-///
-/// // Build an `Operator` to start operating the storage.
-/// let op: Operator = Operator::new(builder)?.finish();
-///
-/// Ok(())
-/// }
-/// ```
pub trait Configurator: Serialize + DeserializeOwned + Debug + 'static {
/// Associated builder for this configuration.
type Builder: Builder;
diff --git a/core/core/src/types/mode.rs b/core/core/src/types/mode.rs
index bbb464c75..26468c082 100644
--- a/core/core/src/types/mode.rs
+++ b/core/core/src/types/mode.rs
@@ -43,7 +43,7 @@ impl EntryMode {
/// Create entry mode from given path.
#[allow(dead_code)]
- pub(crate) fn from_path(path: &str) -> Self {
+ pub fn from_path(path: &str) -> Self {
if path.ends_with('/') {
EntryMode::DIR
} else {
diff --git a/core/core/src/types/operator/operator.rs
b/core/core/src/types/operator/operator.rs
index 5b657fc2a..d9cde1c67 100644
--- a/core/core/src/types/operator/operator.rs
+++ b/core/core/src/types/operator/operator.rs
@@ -39,8 +39,8 @@ use crate::*;
///
/// Users can initialize an `Operator` through the following methods:
///
-/// - [`Operator::new`]: Creates an operator using a [`services`] builder,
such as [`services::S3`].
-/// - [`Operator::from_config`]: Creates an operator using a [`services`]
configuration, such as [`services::S3Config`].
+/// - [`Operator::new`]: Creates an operator using a [`services`] builder,
such as [`services::Memory`].
+/// - [`Operator::from_config`]: Creates an operator using a [`services`]
configuration, such as [`services::MemoryConfig`].
/// - [`Operator::from_iter`]: Creates an operator from an iterator of
configuration key-value pairs.
///
/// ```
@@ -110,7 +110,7 @@ use crate::*;
/// #[tokio::main]
/// async fn main() -> Result<()> {
/// // Pick a builder and configure it.
-/// let mut builder = services::S3::default().bucket("test");
+/// let builder = services::Memory::default();
///
/// // Init an operator
/// let op = Operator::new(builder)?
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
index fe3fd7696..85a5ae6d7 100644
--- a/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
+++ b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
@@ -25,7 +25,7 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
-opendal = { path = "../..", features = ["tests"] }
+opendal = { path = "../..", features = ["tests", "services-s3"] }
tokio = { version = "1", features = ["full"] }
tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }
uuid = { version = "1", features = ["serde", "v4"] }
diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml
new file mode 100644
index 000000000..8b9c1aaae
--- /dev/null
+++ b/core/services/s3/Cargo.toml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "opendal-service-s3"
+version = "0.55.0"
+edition = "2024"
+license = "Apache-2.0"
+repository = "https://github.com/apache/opendal";
+description = "Apache OpenDAL S3 service implementation"
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+base64 = "0.22"
+bytes = "1.6"
+crc32c = "0.6.6"
+ctor = "0.6"
+http = "1.1"
+log = "0.4"
+md-5 = "0.10"
+quick-xml = { version = "0.38", features = ["serialize", "overlapped-lists"] }
+reqsign-aws-v4 = { version = "2.0.1", default-features = false }
+reqsign-core = { version = "2.0.1", default-features = false }
+reqsign-file-read-tokio = { version = "2.0.1", default-features = false }
+reqsign-http-send-reqwest = { version = "2.0.1", default-features = false }
+reqwest = { version = "0.12.24", features = ["stream"], default-features =
false }
+serde = { version = "1", features = ["derive"] }
+
+[dev-dependencies]
+pretty_assertions = "1"
+tracing-subscriber = "0.3"
+tokio = { version = "1.48", features = ["macros", "rt-multi-thread",
"io-util"] }
+serde_json = "1"
diff --git a/core/core/src/services/s3/backend.rs
b/core/services/s3/src/backend.rs
similarity index 97%
rename from core/core/src/services/s3/backend.rs
rename to core/services/s3/src/backend.rs
index 3c619055a..dc3f0295d 100644
--- a/core/core/src/services/s3/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -45,19 +45,21 @@ use reqsign_file_read_tokio::TokioFileRead;
use reqsign_http_send_reqwest::ReqwestHttpSend;
use reqwest::Url;
-use super::S3_SCHEME;
-use super::config::S3Config;
-use super::core::*;
-use super::deleter::S3Deleter;
-use super::error::parse_error;
-use super::lister::S3ListerV1;
-use super::lister::S3ListerV2;
-use super::lister::S3Listers;
-use super::lister::S3ObjectVersionsLister;
-use super::writer::S3Writer;
-use super::writer::S3Writers;
-use crate::raw::*;
-use crate::*;
+use crate::S3_SCHEME;
+use crate::config::S3Config;
+use crate::core::*;
+use crate::deleter::S3Deleter;
+use crate::error::parse_error;
+use crate::lister::S3ListerV1;
+use crate::lister::S3ListerV2;
+use crate::lister::S3Listers;
+use crate::lister::S3ObjectVersionsLister;
+use crate::writer::S3Writer;
+use crate::writer::S3Writers;
+use opendal_core::raw::*;
+use opendal_core::*;
+
+static GLOBAL_REQWEST_CLIENT: LazyLock<reqwest::Client> =
LazyLock::new(reqwest::Client::new);
/// Allow constructing correct region endpoint if user gives a global endpoint.
static ENDPOINT_TEMPLATES: LazyLock<HashMap<&'static str, &'static str>> =
LazyLock::new(|| {
@@ -604,7 +606,7 @@ impl S3Builder {
/// # Examples
///
/// ```no_run
- /// use opendal_core::services::S3;
+ /// use opendal_service_s3::S3;
///
/// # async fn example() {
/// let region: Option<String> =
S3::detect_region("https://s3.amazonaws.com";, "example").await;
@@ -642,10 +644,11 @@ impl S3Builder {
}
// If this bucket is AWS, we can try to match the endpoint.
- if let Some(v) = endpoint.strip_prefix("https://s3.";) {
- if let Some(region) = v.strip_suffix(".amazonaws.com") {
- return Some(region.to_string());
- }
+ if let Some(region) = endpoint
+ .strip_prefix("https://s3.";)
+ .and_then(|v| v.strip_suffix(".amazonaws.com"))
+ {
+ return Some(region.to_string());
}
// If this bucket is OSS, we can try to match the endpoint.
@@ -679,10 +682,12 @@ impl S3Builder {
);
// Get region from response header no matter status code.
- if let Some(header) = res.headers().get("x-amz-bucket-region") {
- if let Ok(regin) = header.to_str() {
- return Some(regin.to_string());
- }
+ if let Some(region) = res
+ .headers()
+ .get("x-amz-bucket-region")
+ .and_then(|header| header.to_str().ok())
+ {
+ return Some(region.to_string());
}
// Status code is 403 or 200 means we already visit the correct
@@ -1119,6 +1124,10 @@ impl Access for S3Backend {
ErrorKind::Unsupported,
"operation is not supported",
)),
+ _ => Err(Error::new(
+ ErrorKind::Unsupported,
+ "operation is not supported",
+ )),
};
let req = req?;
diff --git a/core/core/src/services/s3/compatible_services.md
b/core/services/s3/src/compatible_services.md
similarity index 100%
rename from core/core/src/services/s3/compatible_services.md
rename to core/services/s3/src/compatible_services.md
diff --git a/core/core/src/services/s3/config.rs
b/core/services/s3/src/config.rs
similarity index 98%
rename from core/core/src/services/s3/config.rs
rename to core/services/s3/src/config.rs
index e23c1bce2..9b2761e01 100644
--- a/core/core/src/services/s3/config.rs
+++ b/core/services/s3/src/config.rs
@@ -17,10 +17,13 @@
use std::fmt::Debug;
+use opendal_core::Configurator;
+use opendal_core::OperatorUri;
+use opendal_core::Result;
use serde::Deserialize;
use serde::Serialize;
-use super::backend::S3Builder;
+use crate::backend::S3Builder;
/// Config for Aws S3 and compatible services (including minio, digitalocean
space,
/// Tencent Cloud Object Storage(COS) and so on) support.
@@ -231,10 +234,10 @@ impl Debug for S3Config {
}
}
-impl crate::Configurator for S3Config {
+impl Configurator for S3Config {
type Builder = S3Builder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
let mut map = uri.options().clone();
if let Some(name) = uri.name() {
@@ -263,8 +266,8 @@ mod tests {
use std::iter;
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
#[test]
fn test_s3_config_original_field_names() {
diff --git a/core/core/src/services/s3/core.rs b/core/services/s3/src/core.rs
similarity index 99%
rename from core/core/src/services/s3/core.rs
rename to core/services/s3/src/core.rs
index 6699dd2bf..03e14483a 100644
--- a/core/core/src/services/s3/core.rs
+++ b/core/services/s3/src/core.rs
@@ -43,8 +43,8 @@ use reqsign_core::Signer;
use serde::Deserialize;
use serde::Serialize;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub mod constants {
pub const X_AMZ_COPY_SOURCE: &str = "x-amz-copy-source";
diff --git a/core/core/src/services/s3/deleter.rs
b/core/services/s3/src/deleter.rs
similarity index 95%
rename from core/core/src/services/s3/deleter.rs
rename to core/services/s3/src/deleter.rs
index 9d8ae601d..feb9ff90b 100644
--- a/core/core/src/services/s3/deleter.rs
+++ b/core/services/s3/src/deleter.rs
@@ -20,12 +20,12 @@ use std::sync::Arc;
use bytes::Buf;
use http::StatusCode;
-use super::core::*;
-use super::error::parse_error;
-use super::error::parse_s3_error_code;
-use crate::raw::oio::BatchDeleteResult;
-use crate::raw::*;
-use crate::*;
+use crate::core::*;
+use crate::error::parse_error;
+use crate::error::parse_s3_error_code;
+use opendal_core::raw::oio::BatchDeleteResult;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct S3Deleter {
core: Arc<S3Core>,
diff --git a/core/core/src/services/s3/docs.md b/core/services/s3/src/docs.md
similarity index 97%
rename from core/core/src/services/s3/docs.md
rename to core/services/s3/src/docs.md
index 5a7ab9686..66c2b1d3f 100644
--- a/core/core/src/services/s3/docs.md
+++ b/core/services/s3/src/docs.md
@@ -81,7 +81,7 @@ Reference: [Protecting data using server-side
encryption](https://docs.aws.amazo
use std::sync::Arc;
use anyhow::Result;
-use opendal_core::services::S3;
+use opendal_service_s3::S3;
use opendal_core::Operator;
#[tokio::main]
@@ -126,7 +126,7 @@ async fn main() -> Result<()> {
```rust,no_run
use anyhow::Result;
use log::info;
-use opendal_core::services::S3;
+use opendal_service_s3::S3;
use opendal_core::Operator;
#[tokio::main]
@@ -155,7 +155,7 @@ async fn main() -> Result<()> {
```rust,no_run
use anyhow::Result;
use log::info;
-use opendal_core::services::S3;
+use opendal_service_s3::S3;
use opendal_core::Operator;
#[tokio::main]
@@ -185,7 +185,7 @@ async fn main() -> Result<()> {
```rust,no_run
use anyhow::Result;
use log::info;
-use opendal_core::services::S3;
+use opendal_service_s3::S3;
use opendal_core::Operator;
#[tokio::main]
@@ -215,7 +215,7 @@ async fn main() -> Result<()> {
```rust,no_run
use anyhow::Result;
use log::info;
-use opendal_core::services::S3;
+use opendal_service_s3::S3;
use opendal_core::Operator;
#[tokio::main]
diff --git a/core/core/src/services/s3/error.rs b/core/services/s3/src/error.rs
similarity index 99%
rename from core/core/src/services/s3/error.rs
rename to core/services/s3/src/error.rs
index 3244abb44..da675c6de 100644
--- a/core/core/src/services/s3/error.rs
+++ b/core/services/s3/src/error.rs
@@ -21,8 +21,8 @@ use http::response::Parts;
use quick_xml::de;
use serde::Deserialize;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
/// S3Error is the error returned by s3 service.
#[derive(Default, Debug, Deserialize, PartialEq, Eq)]
diff --git a/core/core/src/services/s3/mod.rs b/core/services/s3/src/lib.rs
similarity index 84%
copy from core/core/src/services/s3/mod.rs
copy to core/services/s3/src/lib.rs
index 432f8f45e..57b39c419 100644
--- a/core/core/src/services/s3/mod.rs
+++ b/core/services/s3/src/lib.rs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-/// Default scheme for s3 service.
-pub const S3_SCHEME: &str = "s3";
-
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+#![cfg_attr(docsrs, feature(doc_cfg))]
+//! Amazon S3 service implementation for Apache OpenDAL.
+#![deny(missing_docs)]
mod backend;
mod config;
@@ -31,7 +30,10 @@ mod writer;
pub use backend::S3Builder as S3;
pub use config::S3Config;
+/// Default scheme for s3 service.
+pub const S3_SCHEME: &str = "s3";
+
#[ctor::ctor]
fn register_s3_service() {
- DEFAULT_OPERATOR_REGISTRY.register::<S3>(S3_SCHEME);
+ opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<S3>(S3_SCHEME);
}
diff --git a/core/core/src/services/s3/lister.rs
b/core/services/s3/src/lister.rs
similarity index 98%
rename from core/core/src/services/s3/lister.rs
rename to core/services/s3/src/lister.rs
index ae15788ff..8c9605296 100644
--- a/core/core/src/services/s3/lister.rs
+++ b/core/services/s3/src/lister.rs
@@ -20,14 +20,14 @@ use std::sync::Arc;
use bytes::Buf;
use quick_xml::de;
-use super::core::*;
-use super::error::parse_error;
-use crate::EntryMode;
-use crate::Error;
-use crate::Metadata;
-use crate::Result;
-use crate::raw::oio::PageContext;
-use crate::raw::*;
+use crate::core::*;
+use crate::error::parse_error;
+use opendal_core::EntryMode;
+use opendal_core::Error;
+use opendal_core::Metadata;
+use opendal_core::Result;
+use opendal_core::raw::oio::PageContext;
+use opendal_core::raw::*;
pub type S3Listers = ThreeWays<
oio::PageLister<S3ListerV1>,
diff --git a/core/core/src/services/s3/mod.rs b/core/services/s3/src/mod.rs
similarity index 96%
rename from core/core/src/services/s3/mod.rs
rename to core/services/s3/src/mod.rs
index 432f8f45e..5407b1653 100644
--- a/core/core/src/services/s3/mod.rs
+++ b/core/services/s3/src/mod.rs
@@ -18,7 +18,7 @@
/// Default scheme for s3 service.
pub const S3_SCHEME: &str = "s3";
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+use opendal_core::DEFAULT_OPERATOR_REGISTRY;
mod backend;
mod config;
diff --git a/core/core/src/services/s3/writer.rs
b/core/services/s3/src/writer.rs
similarity index 95%
rename from core/core/src/services/s3/writer.rs
rename to core/services/s3/src/writer.rs
index bd73388fb..5a01e9237 100644
--- a/core/core/src/services/s3/writer.rs
+++ b/core/services/s3/src/writer.rs
@@ -22,12 +22,12 @@ use constants::X_AMZ_OBJECT_SIZE;
use constants::X_AMZ_VERSION_ID;
use http::StatusCode;
-use super::core::*;
-use super::error::S3Error;
-use super::error::from_s3_error;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
+use crate::core::*;
+use crate::error::S3Error;
+use crate::error::from_s3_error;
+use crate::error::parse_error;
+use opendal_core::raw::*;
+use opendal_core::*;
pub type S3Writers = TwoWays<oio::MultipartWriter<S3Writer>,
oio::AppendWriter<S3Writer>>;
@@ -55,10 +55,10 @@ impl S3Writer {
if let Some(version) = parse_header_to_str(headers, X_AMZ_VERSION_ID)?
{
meta.set_version(version);
}
- if let Some(size) = parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)? {
- if let Ok(value) = size.parse() {
- meta.set_content_length(value);
- }
+ if let Some(value) =
+ parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)?.and_then(|size|
size.parse().ok())
+ {
+ meta.set_content_length(value);
}
Ok(meta)
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 381539fb8..e02ba8c9c 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -19,7 +19,14 @@
html_logo_url =
"https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg";
)]
#![cfg_attr(docsrs, feature(doc_cfg))]
-//! Facade crate that re-exports all public APIs from `opendal-core`.
+//! Facade crate that re-exports all public APIs from `opendal-core` and
optional services/layers.
#![deny(missing_docs)]
pub use opendal_core::*;
+
+/// Re-export of service implementations.
+pub mod services {
+ pub use opendal_core::services::*;
+ #[cfg(feature = "services-s3")]
+ pub use opendal_service_s3::*;
+}
![https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"](https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"){kind=link}