This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch registry-more in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 505ca44d8345a1158010607cf9b42adfb3ef8fcf Author: Xuanwo <[email protected]> AuthorDate: Wed Oct 1 17:00:18 2025 +0800 feat: Add from_uri support for all object storage services Signed-off-by: Xuanwo <[email protected]> --- core/src/services/azblob/backend.rs | 90 ++++++++++++++++++++++++++++++++++++- core/src/services/azblob/mod.rs | 2 +- core/src/services/b2/backend.rs | 81 ++++++++++++++++++++++++++++++++- core/src/services/b2/mod.rs | 2 +- core/src/services/cos/backend.rs | 78 +++++++++++++++++++++++++++++++- core/src/services/cos/mod.rs | 2 +- core/src/services/fs/backend.rs | 15 +++++++ core/src/services/gcs/backend.rs | 79 +++++++++++++++++++++++++++++++- core/src/services/gcs/mod.rs | 2 +- core/src/services/memory/backend.rs | 10 +++++ core/src/services/obs/backend.rs | 77 ++++++++++++++++++++++++++++++- core/src/services/obs/mod.rs | 2 +- core/src/services/oss/backend.rs | 78 +++++++++++++++++++++++++++++++- core/src/services/oss/mod.rs | 2 +- core/src/services/s3/backend.rs | 11 +++++ core/src/services/upyun/backend.rs | 79 +++++++++++++++++++++++++++++++- core/src/services/upyun/mod.rs | 2 +- core/src/types/operator/registry.rs | 14 ++++++ 18 files changed, 605 insertions(+), 21 deletions(-) diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index 780caee62..72b7436cc 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,7 +24,9 @@ use base64::prelude::BASE64_STANDARD; use base64::Engine; use http::Response; use http::StatusCode; +use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; @@ -38,7 +41,7 @@ use super::error::parse_error; use super::lister::AzblobLister; use super::writer::AzblobWriter; use super::writer::AzblobWriters; -use super::DEFAULT_SCHEME; +use super::AZBLOB_SCHEME; use crate::raw::*; use crate::services::AzblobConfig; use crate::*; @@ -59,6 +62,65 @@ impl From<AzureStorageConfig> for AzblobConfig { impl Configurator for AzblobConfig { type Builder = AzblobBuilder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_container = map.get("container").map(|v| !v.is_empty()).unwrap_or(false); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_container { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if host.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "azblob uri requires container", + )); + } + map.insert("container".to_string(), host.to_string()); + + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "azblob uri requires container", + )); + } + + let mut segments = trimmed.splitn(2, '/'); + let container = segments.next().unwrap(); + if container.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "azblob uri requires container", + )); + } + map.insert("container".to_string(), container.to_string()); + + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { AzblobBuilder { @@ -397,7 +459,7 @@ impl Builder for AzblobBuilder { core: Arc::new(AzblobCore { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(AZBLOB_SCHEME) .set_root(&root) .set_name(container) .set_native_capability(Capability { @@ -592,3 +654,27 @@ impl Access for AzblobBackend { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_with_host_container() { + let uri: Uri = "azblob://my-container/path/to/root".parse().unwrap(); + let cfg = AzblobConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.container, "my-container"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } + + #[test] + fn from_uri_with_path_container() { + let uri: Uri = "azblob:///my-container/nested/root".parse().unwrap(); + let cfg = AzblobConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.container, "my-container"); + assert_eq!(cfg.root.as_deref(), Some("nested/root")); + } +} diff --git a/core/src/services/azblob/mod.rs b/core/src/services/azblob/mod.rs index 818ed4628..f014a29ef 100644 --- a/core/src/services/azblob/mod.rs +++ b/core/src/services/azblob/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for azblob service. #[cfg(feature = "services-azblob")] -pub(super) const DEFAULT_SCHEME: &str = "azblob"; +pub const AZBLOB_SCHEME: &str = "azblob"; #[cfg(feature = "services-azblob")] pub(crate) mod core; #[cfg(feature = "services-azblob")] diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs index f1674a03b..868039c8f 100644 --- a/core/src/services/b2/backend.rs +++ b/core/src/services/b2/backend.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -22,7 +23,9 @@ use std::sync::Arc; use http::Request; use http::Response; use http::StatusCode; +use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use tokio::sync::RwLock; use super::core::constants; @@ -34,13 +37,67 @@ use super::error::parse_error; use super::lister::B2Lister; use super::writer::B2Writer; use super::writer::B2Writers; -use super::DEFAULT_SCHEME; +use super::B2_SCHEME; use crate::raw::*; use crate::services::B2Config; use crate::*; impl Configurator for B2Config { type Builder = B2Builder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_bucket { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if !host.is_empty() { + map.insert("bucket".to_string(), host.to_string()); + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } + } + + if !map.contains_key("bucket") { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "b2 uri requires bucket", + )); + } + let mut segments = trimmed.splitn(2, '/'); + let bucket = segments.next().unwrap(); + if bucket.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "b2 uri requires bucket", + )); + } + map.insert("bucket".to_string(), bucket.to_string()); + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { B2Builder { @@ -191,7 +248,7 @@ impl Builder for B2Builder { core: Arc::new(B2Core { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(B2_SCHEME) .set_root(&root) .set_native_capability(Capability { stat: true, @@ -432,3 +489,23 @@ impl Access for B2Backend { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "b2://example-bucket/path/to/root".parse().unwrap(); + let mut options = HashMap::new(); + options.insert("bucket_id".to_string(), "bucket-id".to_string()); + + let cfg = B2Config::from_uri(&uri, &options).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + assert_eq!(cfg.bucket_id, "bucket-id"); + } +} diff --git a/core/src/services/b2/mod.rs b/core/src/services/b2/mod.rs index 09c6b4bfe..6a4778994 100644 --- a/core/src/services/b2/mod.rs +++ b/core/src/services/b2/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for b2 service. #[cfg(feature = "services-b2")] -pub(super) const DEFAULT_SCHEME: &str = "b2"; +pub const B2_SCHEME: &str = "b2"; #[cfg(feature = "services-b2")] mod core; #[cfg(feature = "services-b2")] diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index da2d29804..d1750fb5a 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -22,6 +23,7 @@ use http::Response; use http::StatusCode; use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use reqsign::TencentCosConfig; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; @@ -34,7 +36,7 @@ use super::lister::CosListers; use super::lister::CosObjectVersionsLister; use super::writer::CosWriter; use super::writer::CosWriters; -use super::DEFAULT_SCHEME; +use super::COS_SCHEME; use crate::raw::oio::PageLister; use crate::raw::*; use crate::services::CosConfig; @@ -42,6 +44,62 @@ use crate::*; impl Configurator for CosConfig { type Builder = CosBuilder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_bucket { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if !host.is_empty() { + map.insert("bucket".to_string(), host.to_string()); + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } + } + + let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false); + + if !has_bucket { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "cos uri requires bucket", + )); + } + let mut segments = trimmed.splitn(2, '/'); + let bucket = segments.next().unwrap(); + if bucket.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "cos uri requires bucket", + )); + } + map.insert("bucket".to_string(), bucket.to_string()); + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { CosBuilder { @@ -221,7 +279,7 @@ impl Builder for CosBuilder { core: Arc::new(CosCore { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(COS_SCHEME) .set_root(&root) .set_name(&bucket) .set_native_capability(Capability { @@ -440,3 +498,19 @@ impl Access for CosBackend { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "cos://example-bucket/path/to/root".parse().unwrap(); + let cfg = CosConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.bucket.as_deref(), Some("example-bucket")); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } +} diff --git a/core/src/services/cos/mod.rs b/core/src/services/cos/mod.rs index 486597aa2..0de2ff00d 100644 --- a/core/src/services/cos/mod.rs +++ b/core/src/services/cos/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for cos service. #[cfg(feature = "services-cos")] -pub(super) const DEFAULT_SCHEME: &str = "cos"; +pub const COS_SCHEME: &str = "cos"; #[cfg(feature = "services-cos")] mod core; #[cfg(feature = "services-cos")] diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index ab0332020..93d0e40fd 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -297,3 +297,18 @@ impl Access for FsBackend { Ok(RpRename::default()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_root() { + let uri: Uri = "fs:///tmp/data".parse().unwrap(); + let cfg = FsConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.root.as_deref(), Some("/tmp/data")); + } +} diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index d93efbce5..00848905d 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use http::Response; use http::StatusCode; +use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use reqsign::GoogleCredentialLoader; use reqsign::GoogleSigner; use reqsign::GoogleTokenLoad; @@ -33,7 +36,7 @@ use super::error::parse_error; use super::lister::GcsLister; use super::writer::GcsWriter; use super::writer::GcsWriters; -use super::DEFAULT_SCHEME; +use super::GCS_SCHEME; use crate::raw::oio::BatchDeleter; use crate::raw::*; use crate::services::GcsConfig; @@ -44,6 +47,62 @@ const DEFAULT_GCS_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read impl Configurator for GcsConfig { type Builder = GcsBuilder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_bucket { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if !host.is_empty() { + map.insert("bucket".to_string(), host.to_string()); + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } + } + + let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true); + + if !has_bucket { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "gcs uri requires bucket", + )); + } + let mut segments = trimmed.splitn(2, '/'); + let bucket = segments.next().unwrap(); + if bucket.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "gcs uri requires bucket", + )); + } + map.insert("bucket".to_string(), bucket.to_string()); + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { GcsBuilder { @@ -307,7 +366,7 @@ impl Builder for GcsBuilder { core: Arc::new(GcsCore { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(GCS_SCHEME) .set_root(&root) .set_name(bucket) .set_native_capability(Capability { @@ -497,3 +556,19 @@ impl Access for GcsBackend { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "gcs://example-bucket/path/to/root".parse().unwrap(); + let cfg = GcsConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } +} diff --git a/core/src/services/gcs/mod.rs b/core/src/services/gcs/mod.rs index a4d5d9712..0c8427158 100644 --- a/core/src/services/gcs/mod.rs +++ b/core/src/services/gcs/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for gcs service. #[cfg(feature = "services-gcs")] -pub(super) const DEFAULT_SCHEME: &str = "gcs"; +pub const GCS_SCHEME: &str = "gcs"; #[cfg(feature = "services-gcs")] mod core; #[cfg(feature = "services-gcs")] diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs index 8c1d0d1cb..b1108fd63 100644 --- a/core/src/services/memory/backend.rs +++ b/core/src/services/memory/backend.rs @@ -193,6 +193,9 @@ impl Access for MemoryAccessor { #[cfg(test)] mod tests { use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; #[test] fn test_accessor_metadata_name() { @@ -202,4 +205,11 @@ mod tests { let b2 = MemoryBuilder::default().build().unwrap(); assert_ne!(b1.info().name(), b2.info().name()) } + + #[test] + fn from_uri_extracts_root() { + let uri: Uri = "memory://localhost/path/to/root".parse().unwrap(); + let cfg = MemoryConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } } diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 88e5f350d..f61167429 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -24,6 +24,7 @@ use http::Response; use http::StatusCode; use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use reqsign::HuaweicloudObsConfig; use reqsign::HuaweicloudObsCredentialLoader; use reqsign::HuaweicloudObsSigner; @@ -35,13 +36,69 @@ use super::error::parse_error; use super::lister::ObsLister; use super::writer::ObsWriter; use super::writer::ObsWriters; -use super::DEFAULT_SCHEME; +use super::OBS_SCHEME; use crate::raw::*; use crate::services::ObsConfig; use crate::*; impl Configurator for ObsConfig { type Builder = ObsBuilder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_bucket { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if !host.is_empty() { + map.insert("bucket".to_string(), host.to_string()); + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } + } + + let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false); + + if !has_bucket { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "obs uri requires bucket", + )); + } + let mut segments = trimmed.splitn(2, '/'); + let bucket = segments.next().unwrap(); + if bucket.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "obs uri requires bucket", + )); + } + map.insert("bucket".to_string(), bucket.to_string()); + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { ObsBuilder { @@ -232,7 +289,7 @@ impl Builder for ObsBuilder { core: Arc::new(ObsCore { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(OBS_SCHEME) .set_root(&root) .set_name(&bucket) .set_native_capability(Capability { @@ -440,3 +497,19 @@ impl Access for ObsBackend { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "obs://example-bucket/path/to/root".parse().unwrap(); + let cfg = ObsConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.bucket.as_deref(), Some("example-bucket")); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } +} diff --git a/core/src/services/obs/mod.rs b/core/src/services/obs/mod.rs index ed94697a1..c33719f6a 100644 --- a/core/src/services/obs/mod.rs +++ b/core/src/services/obs/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for obs service. #[cfg(feature = "services-obs")] -pub(super) const DEFAULT_SCHEME: &str = "obs"; +pub const OBS_SCHEME: &str = "obs"; #[cfg(feature = "services-obs")] mod core; #[cfg(feature = "services-obs")] diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 6cf1a725b..140738275 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,6 +24,7 @@ use http::Response; use http::StatusCode; use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use reqsign::AliyunConfig; use reqsign::AliyunLoader; use reqsign::AliyunOssSigner; @@ -35,7 +37,7 @@ use super::lister::OssListers; use super::lister::OssObjectVersionsLister; use super::writer::OssWriter; use super::writer::OssWriters; -use super::DEFAULT_SCHEME; +use super::OSS_SCHEME; use crate::raw::*; use crate::services::OssConfig; use crate::*; @@ -44,6 +46,62 @@ const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000; impl Configurator for OssConfig { type Builder = OssBuilder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_bucket { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if !host.is_empty() { + map.insert("bucket".to_string(), host.to_string()); + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } + } + + let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true); + + if !has_bucket { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "oss uri requires bucket", + )); + } + let mut segments = trimmed.splitn(2, '/'); + let bucket = segments.next().unwrap(); + if bucket.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "oss uri requires bucket", + )); + } + map.insert("bucket".to_string(), bucket.to_string()); + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { OssBuilder { @@ -511,7 +569,7 @@ impl Builder for OssBuilder { core: Arc::new(OssCore { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(OSS_SCHEME) .set_root(&root) .set_name(bucket) .set_native_capability(Capability { @@ -732,3 +790,19 @@ impl Access for OssBackend { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "oss://example-bucket/path/to/root".parse().unwrap(); + let cfg = OssConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } +} diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs index e27e6a78b..e3e05c1b6 100644 --- a/core/src/services/oss/mod.rs +++ b/core/src/services/oss/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for oss service. #[cfg(feature = "services-oss")] -pub(super) const DEFAULT_SCHEME: &str = "oss"; +pub const OSS_SCHEME: &str = "oss"; #[cfg(feature = "services-oss")] mod core; #[cfg(feature = "services-oss")] diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index a55c95493..4f3676866 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -1188,6 +1188,9 @@ impl Access for S3Backend { #[cfg(test)] mod tests { use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; #[test] fn test_is_valid_bucket() { @@ -1290,4 +1293,12 @@ mod tests { assert_eq!(region.as_deref(), expected, "{name}"); } } + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "s3://example-bucket/path/to/root".parse().unwrap(); + let cfg = S3Config::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } } diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs index 5154f6f83..c13f421c4 100644 --- a/core/src/services/upyun/backend.rs +++ b/core/src/services/upyun/backend.rs @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use http::Response; use http::StatusCode; +use http::Uri; use log::debug; +use percent_encoding::percent_decode_str; use super::core::*; use super::delete::UpyunDeleter; @@ -29,13 +32,69 @@ use super::error::parse_error; use super::lister::UpyunLister; use super::writer::UpyunWriter; use super::writer::UpyunWriters; -use super::DEFAULT_SCHEME; +use super::UPYUN_SCHEME; use crate::raw::*; use crate::services::UpyunConfig; use crate::*; impl Configurator for UpyunConfig { type Builder = UpyunBuilder; + fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> { + let mut map = options.clone(); + + let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true); + + let path = percent_decode_str(uri.path()).decode_utf8_lossy(); + + if has_bucket { + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } else if let Some(authority) = uri.authority() { + let host = authority.host(); + if !host.is_empty() { + map.insert("bucket".to_string(), host.to_string()); + if !map.contains_key("root") { + let trimmed = path.trim_matches('/'); + if !trimmed.is_empty() { + map.insert("root".to_string(), trimmed.to_string()); + } + } + } + } + + let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true); + + if !has_bucket { + let trimmed = path.trim_matches('/'); + if trimmed.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "upyun uri requires bucket", + )); + } + let mut segments = trimmed.splitn(2, '/'); + let bucket = segments.next().unwrap(); + if bucket.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "upyun uri requires bucket", + )); + } + map.insert("bucket".to_string(), bucket.to_string()); + if let Some(rest) = segments.next() { + if !rest.is_empty() && !map.contains_key("root") { + map.insert("root".to_string(), rest.to_string()); + } + } + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { UpyunBuilder { @@ -169,7 +228,7 @@ impl Builder for UpyunBuilder { core: Arc::new(UpyunCore { info: { let am = AccessorInfo::default(); - am.set_scheme(DEFAULT_SCHEME) + am.set_scheme(UPYUN_SCHEME) .set_root(&root) .set_native_capability(Capability { stat: true, @@ -315,3 +374,19 @@ impl Access for UpyunBackend { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Configurator; + use http::Uri; + use std::collections::HashMap; + + #[test] + fn from_uri_extracts_bucket_and_root() { + let uri: Uri = "upyun://example-bucket/path/to/root".parse().unwrap(); + let cfg = UpyunConfig::from_uri(&uri, &HashMap::new()).unwrap(); + assert_eq!(cfg.bucket, "example-bucket"); + assert_eq!(cfg.root.as_deref(), Some("path/to/root")); + } +} diff --git a/core/src/services/upyun/mod.rs b/core/src/services/upyun/mod.rs index eee374681..50db7493e 100644 --- a/core/src/services/upyun/mod.rs +++ b/core/src/services/upyun/mod.rs @@ -17,7 +17,7 @@ /// Default scheme for upyun service. #[cfg(feature = "services-upyun")] -pub(super) const DEFAULT_SCHEME: &str = "upyun"; +pub const UPYUN_SCHEME: &str = "upyun"; #[cfg(feature = "services-upyun")] mod core; #[cfg(feature = "services-upyun")] diff --git a/core/src/types/operator/registry.rs b/core/src/types/operator/registry.rs index 7f11c003f..22d05c408 100644 --- a/core/src/types/operator/registry.rs +++ b/core/src/types/operator/registry.rs @@ -97,6 +97,20 @@ fn register_builtin_services(registry: &OperatorRegistry) { registry.register::<services::Fs>(services::FS_SCHEME); #[cfg(feature = "services-s3")] registry.register::<services::S3>(services::S3_SCHEME); + #[cfg(feature = "services-azblob")] + registry.register::<services::Azblob>(services::AZBLOB_SCHEME); + #[cfg(feature = "services-b2")] + registry.register::<services::B2>(services::B2_SCHEME); + #[cfg(feature = "services-cos")] + registry.register::<services::Cos>(services::COS_SCHEME); + #[cfg(feature = "services-gcs")] + registry.register::<services::Gcs>(services::GCS_SCHEME); + #[cfg(feature = "services-obs")] + registry.register::<services::Obs>(services::OBS_SCHEME); + #[cfg(feature = "services-oss")] + registry.register::<services::Oss>(services::OSS_SCHEME); + #[cfg(feature = "services-upyun")] + registry.register::<services::Upyun>(services::UPYUN_SCHEME); } /// Factory adapter that builds an operator from a configurator type.
