This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch registry-more in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 7031448901103173af0ad9ffce930b511d7eac4a Author: Xuanwo <[email protected]> AuthorDate: Tue Oct 14 17:32:30 2025 +0800 feat: Add from_uri support for http/webdav/ftp/sftp Signed-off-by: Xuanwo <[email protected]> --- core/src/services/ftp/backend.rs | 53 +++++++++++++++++++++++++++++ core/src/services/http/backend.rs | 66 +++++++++++++++++++++++++++++++++++++ core/src/services/sftp/backend.rs | 55 +++++++++++++++++++++++++++++++ core/src/services/webdav/backend.rs | 64 +++++++++++++++++++++++++++++++++++ core/src/types/operator/uri.rs | 41 +++++++++++++++++++---- 5 files changed, 272 insertions(+), 7 deletions(-) diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index ff7bf662a..4b5362a95 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -39,9 +39,30 @@ use super::reader::FtpReader; use super::writer::FtpWriter; use crate::raw::*; use crate::services::FtpConfig; +use crate::types::OperatorUri; use crate::*; impl Configurator for FtpConfig { type Builder = FtpBuilder; + + fn from_uri(uri: &OperatorUri) -> Result<Self> { + let authority = uri.authority().ok_or_else(|| { + Error::new(ErrorKind::ConfigInvalid, "uri authority is required") + .with_context("service", Scheme::Ftp) + })?; + + let mut map = uri.options().clone(); + map.insert( + "endpoint".to_string(), + format!("{DEFAULT_SCHEME}://{authority}"), + ); + + if let Some(root) = uri.root() { + map.insert("root".to_string(), root.to_string()); + } + + Self::from_iter(map) + } + fn into_builder(self) -> Self::Builder { FtpBuilder { config: self } } @@ -355,6 +376,8 @@ impl FtpBackend { #[cfg(test)] mod build_test { use super::FtpBuilder; + use crate::services::FtpConfig; + use crate::types::OperatorUri; use crate::*; #[test] @@ -385,4 +408,34 @@ mod build_test { let e = b.unwrap_err(); assert_eq!(e.kind(), ErrorKind::ConfigInvalid); } + + #[test] + fn from_uri_sets_endpoint_and_root() { + let uri = OperatorUri::new( + "ftp://example.com/public/data".parse().unwrap(), + Vec::<(String, String)>::new(), + ) + .unwrap(); + + let cfg = FtpConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("ftp://example.com";)); + assert_eq!(cfg.root.as_deref(), Some("public/data")); + } + + #[test] + fn from_uri_applies_credentials_from_query() { + let uri = OperatorUri::new( + "ftp://example.com/data".parse().unwrap(), + vec![ + ("user".to_string(), "alice".to_string()), + ("password".to_string(), "secret".to_string()), + ], + ) + .unwrap(); + + let cfg = FtpConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("ftp://example.com";)); + assert_eq!(cfg.user.as_deref(), Some("alice")); + assert_eq!(cfg.password.as_deref(), Some("secret")); + } } diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 9aa0a32f1..99675cbf5 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -28,10 +28,30 @@ use super::core::HttpCore; use super::error::parse_error; use crate::raw::*; use crate::services::HttpConfig; +use crate::types::OperatorUri; use crate::*; impl Configurator for HttpConfig { type Builder = HttpBuilder; + fn from_uri(uri: &OperatorUri) -> Result<Self> { + let authority = uri.authority().ok_or_else(|| { + Error::new(ErrorKind::ConfigInvalid, "uri authority is required") + .with_context("service", Scheme::Http) + })?; + + let mut map = uri.options().clone(); + map.insert( + "endpoint".to_string(), + format!("{}://{}", uri.scheme(), authority), + ); + + if let Some(root) = uri.root() { + map.insert("root".to_string(), root.to_string()); + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { HttpBuilder { @@ -289,3 +309,49 @@ impl Access for HttpBackend { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_uri_sets_endpoint_and_root() { + let uri = OperatorUri::new( + "http://example.com/static/assets".parse().unwrap(), + Vec::<(String, String)>::new(), + ) + .unwrap(); + + let cfg = HttpConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("http://example.com";)); + assert_eq!(cfg.root.as_deref(), Some("static/assets")); + } + + #[test] + fn from_uri_preserves_query_options() { + let uri = OperatorUri::new( + "http://cdn.example.com/data?token=abc123".parse().unwrap(), + Vec::<(String, String)>::new(), + ) + .unwrap(); + let cfg = HttpConfig::from_uri(&uri).unwrap(); + + assert_eq!(cfg.endpoint.as_deref(), Some("http://cdn.example.com";)); + assert_eq!(cfg.token.as_deref(), Some("abc123")); + } + + #[test] + fn from_uri_ignores_endpoint_override() { + let uri = OperatorUri::new( + "http://example.com/data".parse().unwrap(), + vec![( + "endpoint".to_string(), + "https://cdn.example.com".to_string(), + )], + ) + .unwrap(); + let cfg = HttpConfig::from_uri(&uri).unwrap(); + + assert_eq!(cfg.endpoint.as_deref(), Some("http://example.com";)); + } +} diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index 15dad4262..c3f592c9a 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -38,9 +38,27 @@ use super::reader::SftpReader; use super::writer::SftpWriter; use crate::raw::*; use crate::services::SftpConfig; +use crate::types::OperatorUri; use crate::*; impl Configurator for SftpConfig { type Builder = SftpBuilder; + + fn from_uri(uri: &OperatorUri) -> Result<Self> { + let authority = uri.authority().ok_or_else(|| { + Error::new(ErrorKind::ConfigInvalid, "uri authority is required") + .with_context("service", Scheme::Sftp) + })?; + + let mut map = uri.options().clone(); + map.insert("endpoint".to_string(), authority.to_string()); + + if let Some(root) = uri.root() { + map.insert("root".to_string(), root.to_string()); + } + + Self::from_iter(map) + } + fn into_builder(self) -> Self::Builder { SftpBuilder { config: self } } @@ -395,3 +413,40 @@ impl Access for SftpBackend { Ok(RpRename::default()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_uri_sets_endpoint_and_root() { + let uri = OperatorUri::new( + "sftp://sftp.example.com/home/alice".parse().unwrap(), + Vec::<(String, String)>::new(), + ) + .unwrap(); + + let cfg = SftpConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("sftp.example.com")); + assert_eq!(cfg.root.as_deref(), Some("home/alice")); + } + + #[test] + fn from_uri_applies_connection_overrides() { + let uri = OperatorUri::new( + "sftp://host".parse().unwrap(), + vec![ + ("user".to_string(), "alice".to_string()), + ("key".to_string(), "/home/alice/.ssh/id_rsa".to_string()), + ("known_hosts_strategy".to_string(), "accept".to_string()), + ], + ) + .unwrap(); + + let cfg = SftpConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("host")); + assert_eq!(cfg.user.as_deref(), Some("alice")); + assert_eq!(cfg.key.as_deref(), Some("/home/alice/.ssh/id_rsa")); + assert_eq!(cfg.known_hosts_strategy.as_deref(), Some("accept")); + } +} diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index ef3d76e4f..c3dc48b64 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -32,10 +32,27 @@ use super::lister::WebdavLister; use super::writer::WebdavWriter; use crate::raw::*; use crate::services::WebdavConfig; +use crate::types::OperatorUri; use crate::*; impl Configurator for WebdavConfig { type Builder = WebdavBuilder; + fn from_uri(uri: &OperatorUri) -> Result<Self> { + let authority = uri.authority().ok_or_else(|| { + Error::new(ErrorKind::ConfigInvalid, "uri authority is required") + .with_context("service", Scheme::Webdav) + })?; + + let mut map = uri.options().clone(); + map.insert("endpoint".to_string(), format!("https://{authority}";)); + + if let Some(root) = uri.root() { + map.insert("root".to_string(), root.to_string()); + } + + Self::from_iter(map) + } + #[allow(deprecated)] fn into_builder(self) -> Self::Builder { WebdavBuilder { @@ -316,3 +333,50 @@ impl Access for WebdavBackend { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_uri_sets_endpoint_and_root() { + let uri = OperatorUri::new( + "webdav://webdav.example.com/remote.php/webdav" + .parse() + .unwrap(), + Vec::<(String, String)>::new(), + ) + .unwrap(); + + let cfg = WebdavConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("https://webdav.example.com";)); + assert_eq!(cfg.root.as_deref(), Some("remote.php/webdav")); + } + + #[test] + fn from_uri_ignores_endpoint_override() { + let uri = OperatorUri::new( + "webdav://dav.internal/data".parse().unwrap(), + vec![( + "endpoint".to_string(), + "http://dav.internal:8080".to_string(), + )], + ) + .unwrap(); + + let cfg = WebdavConfig::from_uri(&uri).unwrap(); + assert_eq!(cfg.endpoint.as_deref(), Some("https://dav.internal";)); + } + + #[test] + fn from_uri_propagates_disable_copy() { + let uri = OperatorUri::new( + "webdav://dav.example.com".parse().unwrap(), + vec![("disable_copy".to_string(), "true".to_string())], + ) + .unwrap(); + + let cfg = WebdavConfig::from_uri(&uri).unwrap(); + assert!(cfg.disable_copy); + } +} diff --git a/core/src/types/operator/uri.rs b/core/src/types/operator/uri.rs index 7cfc7900c..3d00d7b5d 100644 --- a/core/src/types/operator/uri.rs +++ b/core/src/types/operator/uri.rs @@ -26,6 +26,7 @@ use crate::{Error, ErrorKind, Result}; #[derive(Clone, Debug, Eq, PartialEq)] pub struct OperatorUri { scheme: String, + authority: Option<String>, name: Option<String>, root: Option<String>, options: HashMap<String, String>, @@ -64,14 +65,19 @@ impl OperatorUri { options.insert(key.to_ascii_lowercase(), value); } - let name = uri.authority().and_then(|authority| { - let host = authority.host(); - if host.is_empty() { - None - } else { - Some(host.to_string()) + let (authority, name) = match uri.authority() { + Some(authority) => { + let authority_str = authority.as_str().to_string(); + let host = authority.host(); + let name = if host.is_empty() { + None + } else { + Some(host.to_string()) + }; + (Some(authority_str), name) } - }); + None => (None, None), + }; let decoded_path = percent_decode_str(uri.path()).decode_utf8_lossy(); let trimmed = decoded_path.trim_matches('/'); @@ -83,6 +89,7 @@ impl OperatorUri { Ok(Self { scheme, + authority, name, root, options, @@ -99,6 +106,11 @@ impl OperatorUri { self.name.as_deref() } + /// Authority extracted from the URI, if present (host with optional port). + pub fn authority(&self) -> Option<&str> { + self.authority.as_deref() + } + /// Root path (without leading slash) extracted from the URI path, if present. pub fn root(&self) -> Option<&str> { self.root.as_deref() @@ -235,6 +247,7 @@ mod tests { .unwrap(); assert_eq!(uri.scheme(), "s3"); + assert_eq!(uri.authority(), Some("example-bucket")); assert_eq!(uri.name(), Some("example-bucket")); assert_eq!(uri.root(), Some("photos/2024")); assert!(uri.options().is_empty()); @@ -261,4 +274,18 @@ mod tests { Some("https://custom";) ); } + + #[test] + fn parse_uri_with_port_preserves_authority() { + let uri = OperatorUri::new( + "http://example.com:8080/root".parse().unwrap(), + Vec::<(String, String)>::new(), + ) + .unwrap(); + + assert_eq!(uri.scheme(), "http"); + assert_eq!(uri.authority(), Some("example.com:8080")); + assert_eq!(uri.name(), Some("example.com")); + assert_eq!(uri.root(), Some("root")); + } }
