This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch xuanwo/reqsign-http-send-via-accessor-http-client in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 7d6481be37d072cb4aa72fd20220abc03fd53129 Author: Xuanwo <[email protected]> AuthorDate: Mon Mar 2 19:45:11 2026 +0800 refactor(core): route reqsign send via accessor http client --- core/Cargo.lock | 30 ++-------------- core/core/Cargo.toml | 1 + core/core/src/raw/http_util/client.rs | 64 +++++++++++++++++++++++++++++++++++ core/core/src/raw/http_util/mod.rs | 1 + core/services/azblob/Cargo.toml | 1 - core/services/azblob/src/backend.rs | 10 +++--- core/services/azdls/Cargo.toml | 1 - core/services/azdls/src/backend.rs | 10 +++--- core/services/azfile/Cargo.toml | 1 - core/services/azfile/src/backend.rs | 10 +++--- core/services/cos/Cargo.toml | 4 --- core/services/cos/src/backend.rs | 13 +++---- core/services/gcs/Cargo.toml | 5 --- core/services/gcs/src/backend.rs | 13 +++---- core/services/obs/Cargo.toml | 1 - core/services/obs/src/backend.rs | 10 +++--- core/services/oss/Cargo.toml | 1 - core/services/oss/src/backend.rs | 10 +++--- core/services/s3/Cargo.toml | 5 +-- core/services/s3/src/backend.rs | 17 ++++------ 20 files changed, 111 insertions(+), 97 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 837d3f170..bcf379a61 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -6029,6 +6029,7 @@ dependencies = [ "pretty_assertions", "quick-xml", "rand 0.8.5", + "reqsign-core", "reqwest", "serde", "serde_json", @@ -6374,7 +6375,6 @@ dependencies = [ "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "serde", "serde_json", "sha2", @@ -6395,7 +6395,6 @@ dependencies = [ "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "serde", "serde_json", "tokio", @@ -6414,7 +6413,6 @@ dependencies = [ "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "serde", "tokio", ] @@ -6485,9 +6483,7 @@ dependencies = [ "quick-xml", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "reqsign-tencent-cos", - "reqwest", "serde", "tokio", ] @@ -6606,8 +6602,6 @@ dependencies = [ "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", - "reqsign-http-send-reqwest", - "reqwest", "serde", "serde_json", "tokio", @@ -6855,7 +6849,6 @@ dependencies = [ "quick-xml", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "reqsign-huaweicloud-obs", "serde", "tokio", @@ -6900,7 +6893,6 @@ dependencies = [ "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", "serde", "tokio", ] @@ -6988,11 +6980,10 @@ dependencies = [ "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", - "reqsign-http-send-reqwest", - "reqwest", "serde", "serde_json", "tokio", + "url", ] [[package]] @@ -8759,23 +8750,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "reqsign-http-send-reqwest" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "46186bce769674f9200ad01af6f2ca42de3e819ddc002fff1edae135bfb6cd9c" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "futures-channel", - "http 1.4.0", - "http-body-util", - "reqsign-core", - "reqwest", - "wasm-bindgen-futures", -] - [[package]] name = "reqsign-huaweicloud-obs" version = "2.0.2" diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml index a9298eeb1..235fea6b6 100644 --- a/core/core/Cargo.toml +++ b/core/core/Cargo.toml @@ -75,6 +75,7 @@ md-5 = "0.10" mea = { workspace = true } percent-encoding = "2" quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } +reqsign-core = { version = "2.0.2", default-features = false } reqwest = { version = "0.12.24", features = [ "stream", ], default-features = false } diff --git a/core/core/src/raw/http_util/client.rs b/core/core/src/raw/http_util/client.rs index 4c69cb890..993c614e8 100644 --- a/core/core/src/raw/http_util/client.rs +++ b/core/core/src/raw/http_util/client.rs @@ -62,6 +62,20 @@ pub struct HttpClient { fetcher: HttpFetcher, } +/// A reqsign `HttpSend` implementation that always forwards requests to the +/// current http client stored inside [`AccessorInfo`]. +#[derive(Clone)] +pub struct AccessorInfoHttpSend { + info: Arc<AccessorInfo>, +} + +impl AccessorInfoHttpSend { + /// Create a new [`AccessorInfoHttpSend`]. + pub fn new(info: Arc<AccessorInfo>) -> Self { + Self { info } + } +} + /// We don't want users to know details about our clients. impl Debug for HttpClient { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { @@ -69,6 +83,12 @@ impl Debug for HttpClient { } } +impl Debug for AccessorInfoHttpSend { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AccessorInfoHttpSend").finish() + } +} + impl Default for HttpClient { fn default() -> Self { Self { @@ -109,6 +129,50 @@ impl HttpClient { } } +impl reqsign_core::HttpSend for HttpClient { + fn http_send<'life0, 'async_trait>( + &'life0 self, + req: http::Request<Bytes>, + ) -> std::pin::Pin< + Box<dyn Future<Output = reqsign_core::Result<http::Response<Bytes>>> + Send + 'async_trait>, + > + where + 'life0: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + let req = req.map(Buffer::from); + let resp = self.send(req).await.map_err(|err| { + let retryable = err.is_temporary(); + reqsign_core::Error::unexpected("send request via OpenDAL HttpClient") + .with_source(err) + .set_retryable(retryable) + })?; + + let (parts, body) = resp.into_parts(); + Ok(http::Response::from_parts(parts, body.to_bytes())) + }) + } +} + +impl reqsign_core::HttpSend for AccessorInfoHttpSend { + fn http_send<'life0, 'async_trait>( + &'life0 self, + req: http::Request<Bytes>, + ) -> std::pin::Pin< + Box<dyn Future<Output = reqsign_core::Result<http::Response<Bytes>>> + Send + 'async_trait>, + > + where + 'life0: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + let client = self.info.http_client(); + reqsign_core::HttpSend::http_send(&client, req).await + }) + } +} + /// HttpFetch is the trait to fetch a request in async way. /// User should implement this trait to provide their own http client. pub trait HttpFetch: Send + Sync + Unpin + 'static { diff --git a/core/core/src/raw/http_util/mod.rs b/core/core/src/raw/http_util/mod.rs index ae6b1e585..93ef4d167 100644 --- a/core/core/src/raw/http_util/mod.rs +++ b/core/core/src/raw/http_util/mod.rs @@ -23,6 +23,7 @@ //! it easier to develop services and layers outside opendal. mod client; +pub use client::AccessorInfoHttpSend; /// temporary client used by several features #[allow(unused_imports)] pub use client::GLOBAL_REQWEST_CLIENT; diff --git a/core/services/azblob/Cargo.toml b/core/services/azblob/Cargo.toml index a0dffe4d0..ef40e154f 100644 --- a/core/services/azblob/Cargo.toml +++ b/core/services/azblob/Cargo.toml @@ -41,7 +41,6 @@ quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-azure-storage = { version = "2.0.2", default-features = false } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } serde = { workspace = true, features = ["derive"] } sha2 = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } diff --git a/core/services/azblob/src/backend.rs b/core/services/azblob/src/backend.rs index 69def7c3a..5b9d98936 100644 --- a/core/services/azblob/src/backend.rs +++ b/core/services/azblob/src/backend.rs @@ -32,7 +32,6 @@ use reqsign_core::OsEnv; use reqsign_core::Signer; use reqsign_core::StaticEnv; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; use sha2::Digest; use sha2::Sha256; @@ -369,9 +368,11 @@ impl Builder for AzblobBuilder { } }; + let info = Arc::new(AccessorInfo::default()); + let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(StaticEnv { home_dir: os_env.home_dir(), envs, @@ -401,8 +402,7 @@ impl Builder for AzblobBuilder { Ok(AzblobBackend { core: Arc::new(AzblobCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(AZBLOB_SCHEME) + info.set_scheme(AZBLOB_SCHEME) .set_root(&root) .set_name(container) .set_native_capability(Capability { @@ -447,7 +447,7 @@ impl Builder for AzblobBuilder { ..Default::default() }); - am.into() + info.clone() }, root, endpoint, diff --git a/core/services/azdls/Cargo.toml b/core/services/azdls/Cargo.toml index cd7af72a2..1a6a233b3 100644 --- a/core/services/azdls/Cargo.toml +++ b/core/services/azdls/Cargo.toml @@ -40,7 +40,6 @@ quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-azure-storage = { version = "2.0.2", default-features = false } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/core/services/azdls/src/backend.rs b/core/services/azdls/src/backend.rs index e162cd437..8409dcb04 100644 --- a/core/services/azdls/src/backend.rs +++ b/core/services/azdls/src/backend.rs @@ -30,7 +30,6 @@ use reqsign_core::OsEnv; use reqsign_core::Signer; use reqsign_core::StaticEnv; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; use super::AZDLS_SCHEME; use super::config::AzdlsConfig; @@ -293,9 +292,11 @@ impl Builder for AzdlsBuilder { } let os_env = OsEnv; + let info = Arc::new(AccessorInfo::default()); + let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(StaticEnv { home_dir: os_env.home_dir(), envs, @@ -319,8 +320,7 @@ impl Builder for AzdlsBuilder { Ok(AzdlsBackend { core: Arc::new(AzdlsCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(AZDLS_SCHEME) + info.set_scheme(AZDLS_SCHEME) .set_root(&root) .set_name(filesystem) .set_native_capability(Capability { @@ -348,7 +348,7 @@ impl Builder for AzdlsBuilder { ..Default::default() }); - am.into() + info.clone() }, filesystem: self.config.filesystem.clone(), root, diff --git a/core/services/azfile/Cargo.toml b/core/services/azfile/Cargo.toml index 13e1f3be3..da190655b 100644 --- a/core/services/azfile/Cargo.toml +++ b/core/services/azfile/Cargo.toml @@ -40,7 +40,6 @@ quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-azure-storage = { version = "2.0.2", default-features = false } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } serde = { workspace = true, features = ["derive"] } [dev-dependencies] diff --git a/core/services/azfile/src/backend.rs b/core/services/azfile/src/backend.rs index 6c8d7b492..ede0296c1 100644 --- a/core/services/azfile/src/backend.rs +++ b/core/services/azfile/src/backend.rs @@ -30,7 +30,6 @@ use reqsign_core::OsEnv; use reqsign_core::Signer; use reqsign_core::StaticEnv; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; use super::AZFILE_SCHEME; use super::config::AzfileConfig; @@ -209,9 +208,11 @@ impl Builder for AzfileBuilder { } let os_env = OsEnv; + let info = Arc::new(AccessorInfo::default()); + let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(StaticEnv { home_dir: os_env.home_dir(), envs, @@ -232,8 +233,7 @@ impl Builder for AzfileBuilder { Ok(AzfileBackend { core: Arc::new(AzfileCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(AZFILE_SCHEME) + info.set_scheme(AZFILE_SCHEME) .set_root(&root) .set_native_capability(Capability { stat: true, @@ -254,7 +254,7 @@ impl Builder for AzfileBuilder { ..Default::default() }); - am.into() + info.clone() }, root, endpoint, diff --git a/core/services/cos/Cargo.toml b/core/services/cos/Cargo.toml index aae8ee078..22a8c64f3 100644 --- a/core/services/cos/Cargo.toml +++ b/core/services/cos/Cargo.toml @@ -38,11 +38,7 @@ opendal-core = { path = "../../core", version = "0.55.0", default-features = fal quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } reqsign-tencent-cos = { version = "2.0.2", default-features = false } -reqwest = { version = "0.12.24", default-features = false, features = [ - "stream", -] } serde = { workspace = true, features = ["derive"] } [dev-dependencies] diff --git a/core/services/cos/src/backend.rs b/core/services/cos/src/backend.rs index 164c6f6a1..85990873b 100644 --- a/core/services/cos/src/backend.rs +++ b/core/services/cos/src/backend.rs @@ -27,7 +27,6 @@ use reqsign_core::Env as _; use reqsign_core::OsEnv; use reqsign_core::Signer; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; use reqsign_tencent_cos::DefaultCredentialProvider; use reqsign_tencent_cos::RequestSigner; use reqsign_tencent_cos::StaticCredentialProvider; @@ -43,9 +42,6 @@ use super::lister::CosObjectVersionsLister; use super::writer::CosWriter; use super::writer::CosWriters; use opendal_core::raw::*; -use std::sync::LazyLock; - -static GLOBAL_REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(reqwest::Client::new); use opendal_core::*; /// Tencent-Cloud COS services support. @@ -181,11 +177,13 @@ impl Builder for CosBuilder { let endpoint = uri.host().unwrap().replace(&format!("//{bucket}."), "//"); debug!("backend use endpoint {}", &endpoint); + let info = Arc::new(AccessorInfo::default()); + let os_env = OsEnv; let envs = os_env.vars(); let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(os_env); let mut credential = if self.config.disable_config_load { @@ -222,8 +220,7 @@ impl Builder for CosBuilder { Ok(CosBackend { core: Arc::new(CosCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(COS_SCHEME) + info.set_scheme(COS_SCHEME) .set_root(&root) .set_name(&bucket) .set_native_capability(Capability { @@ -282,7 +279,7 @@ impl Builder for CosBuilder { ..Default::default() }); - am.into() + info.clone() }, bucket: bucket.clone(), root, diff --git a/core/services/gcs/Cargo.toml b/core/services/gcs/Cargo.toml index 46c76e51e..716d7bee1 100644 --- a/core/services/gcs/Cargo.toml +++ b/core/services/gcs/Cargo.toml @@ -41,11 +41,6 @@ quick-xml = { workspace = true, features = ["serialize"] } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } reqsign-google = { version = "2.0.2", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } -reqwest = { version = "0.12.24", default-features = false, features = [ - "json", - "stream", -] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tokio = { workspace = true, features = ["rt"] } diff --git a/core/services/gcs/src/backend.rs b/core/services/gcs/src/backend.rs index 36d2ebbb6..91697cd99 100644 --- a/core/services/gcs/src/backend.rs +++ b/core/services/gcs/src/backend.rs @@ -17,7 +17,6 @@ use std::fmt::Debug; use std::sync::Arc; -use std::sync::LazyLock; use std::time::Duration; use bytes::Bytes; @@ -38,7 +37,6 @@ use reqsign_google::DefaultCredentialProvider; use reqsign_google::RequestSigner; use reqsign_google::StaticCredentialProvider; use reqsign_google::Token; -use reqsign_http_send_reqwest::ReqwestHttpSend; use serde::Deserialize; use super::GCS_SCHEME; @@ -52,8 +50,6 @@ use super::writer::GcsWriters; use opendal_core::raw::*; use opendal_core::*; -static GLOBAL_REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(reqwest::Client::new); - const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com";; const DEFAULT_GCS_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_write";; @@ -408,9 +404,11 @@ impl Builder for GcsBuilder { let mut envs = os_env.vars(); envs.insert("GOOGLE_SCOPE".to_string(), scope.clone()); + let info = Arc::new(AccessorInfo::default()); + let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(StaticEnv { home_dir: os_env.home_dir(), envs, @@ -473,8 +471,7 @@ impl Builder for GcsBuilder { let backend = GcsBackend { core: Arc::new(GcsCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(GCS_SCHEME) + info.set_scheme(GCS_SCHEME) .set_root(&root) .set_name(bucket) .set_native_capability(Capability { @@ -528,7 +525,7 @@ impl Builder for GcsBuilder { ..Default::default() }); - am.into() + info.clone() }, endpoint, bucket: bucket.to_string(), diff --git a/core/services/obs/Cargo.toml b/core/services/obs/Cargo.toml index 3d828df60..9968ac1d4 100644 --- a/core/services/obs/Cargo.toml +++ b/core/services/obs/Cargo.toml @@ -38,7 +38,6 @@ opendal-core = { path = "../../core", version = "0.55.0", default-features = fal quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-core = "2.0.2" reqsign-file-read-tokio = "2.0.2" -reqsign-http-send-reqwest = "2.0.1" reqsign-huaweicloud-obs = "2.0.2" serde = { workspace = true, features = ["derive"] } diff --git a/core/services/obs/src/backend.rs b/core/services/obs/src/backend.rs index 42bcbe2ba..99c5fe2ca 100644 --- a/core/services/obs/src/backend.rs +++ b/core/services/obs/src/backend.rs @@ -30,7 +30,6 @@ use reqsign_core::OsEnv; use reqsign_core::ProvideCredentialChain; use reqsign_core::Signer; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; use reqsign_huaweicloud_obs::EnvCredentialProvider; use reqsign_huaweicloud_obs::RequestSigner; use reqsign_huaweicloud_obs::StaticCredentialProvider; @@ -175,9 +174,11 @@ impl Builder for ObsBuilder { }; debug!("backend use endpoint {}", &endpoint); + let info = Arc::new(AccessorInfo::default()); + let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(OsEnv); let mut provider = ProvideCredentialChain::new().push(EnvCredentialProvider::new()); @@ -201,8 +202,7 @@ impl Builder for ObsBuilder { Ok(ObsBackend { core: Arc::new(ObsCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(OBS_SCHEME) + info.set_scheme(OBS_SCHEME) .set_root(&root) .set_name(&bucket) .set_native_capability(Capability { @@ -251,7 +251,7 @@ impl Builder for ObsBuilder { ..Default::default() }); - am.into() + info.clone() }, bucket, root, diff --git a/core/services/oss/Cargo.toml b/core/services/oss/Cargo.toml index 22ebd0192..4569cd8c7 100644 --- a/core/services/oss/Cargo.toml +++ b/core/services/oss/Cargo.toml @@ -39,7 +39,6 @@ quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-aliyun-oss = { version = "2.0.2", default-features = false } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } serde = { workspace = true, features = ["derive"] } [dev-dependencies] diff --git a/core/services/oss/src/backend.rs b/core/services/oss/src/backend.rs index c152872d7..82b10d5e3 100644 --- a/core/services/oss/src/backend.rs +++ b/core/services/oss/src/backend.rs @@ -33,7 +33,6 @@ use reqsign_core::ProvideCredentialChain; use reqsign_core::Signer; use reqsign_core::StaticEnv; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; use super::OSS_SCHEME; use super::config::OssConfig; @@ -474,9 +473,11 @@ impl Builder for OssBuilder { assume_role = assume_role.with_role_session_name(role_session_name.clone()); } + let info = Arc::new(AccessorInfo::default()); + let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(StaticEnv { home_dir: os_env.home_dir(), envs, @@ -506,8 +507,7 @@ impl Builder for OssBuilder { Ok(OssBackend { core: Arc::new(OssCore { info: { - let am = AccessorInfo::default(); - am.set_scheme(OSS_SCHEME) + info.set_scheme(OSS_SCHEME) .set_root(&root) .set_name(bucket) .set_native_capability(Capability { @@ -571,7 +571,7 @@ impl Builder for OssBuilder { ..Default::default() }); - am.into() + info.clone() }, root, bucket: bucket.to_owned(), diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml index 87bae334a..adcc136a8 100644 --- a/core/services/s3/Cargo.toml +++ b/core/services/s3/Cargo.toml @@ -42,11 +42,8 @@ quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] } reqsign-aws-v4 = { version = "2.0.1", default-features = false } reqsign-core = { version = "2.0.1", default-features = false } reqsign-file-read-tokio = { version = "2.0.1", default-features = false } -reqsign-http-send-reqwest = { version = "2.0.1", default-features = false } -reqwest = { version = "0.12.24", default-features = false, features = [ - "stream", -] } serde = { workspace = true, features = ["derive"] } +url = { workspace = true } [dev-dependencies] pretty_assertions = "1" diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index d3ae6aa76..a828d0cdc 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -18,7 +18,6 @@ use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Write; -use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; @@ -42,8 +41,7 @@ use reqsign_core::OsEnv; use reqsign_core::ProvideCredentialChain; use reqsign_core::Signer; use reqsign_file_read_tokio::TokioFileRead; -use reqsign_http_send_reqwest::ReqwestHttpSend; -use reqwest::Url; +use url::Url; use crate::S3_SCHEME; use crate::config::S3Config; @@ -59,8 +57,6 @@ use crate::writer::S3Writers; use opendal_core::raw::*; use opendal_core::*; -static GLOBAL_REQWEST_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(reqwest::Client::new); - /// Allow constructing correct region endpoint if user gives a global endpoint. static ENDPOINT_TEMPLATES: LazyLock<HashMap<&'static str, &'static str>> = LazyLock::new(|| { let mut m = HashMap::new(); @@ -499,7 +495,7 @@ impl S3Builder { endpoint = endpoint.replace(&format!("//{bucket}."), "//"); // Omit default ports if specified. - if let Ok(url) = Url::from_str(&endpoint) { + if let Ok(url) = Url::parse(&endpoint) { // Remove the trailing `/` of root path. endpoint = url.to_string().trim_end_matches('/').to_string(); } @@ -800,10 +796,12 @@ impl Builder for S3Builder { let endpoint = Self::build_endpoint(&config, ®ion); debug!("backend use endpoint: {endpoint}"); + let info = Arc::new(AccessorInfo::default()); + // Create the context for reqsign-core let ctx = Context::new() .with_file_read(TokioFileRead) - .with_http_send(ReqwestHttpSend::new(GLOBAL_REQWEST_CLIENT.clone())) + .with_http_send(AccessorInfoHttpSend::new(info.clone())) .with_env(OsEnv); let mut provider = { @@ -870,8 +868,7 @@ impl Builder for S3Builder { Ok(S3Backend { core: Arc::new(S3Core { info: { - let am = AccessorInfo::default(); - am.set_scheme(S3_SCHEME) + info.set_scheme(S3_SCHEME) .set_root(&root) .set_name(bucket) .set_native_capability(Capability { @@ -944,7 +941,7 @@ impl Builder for S3Builder { ..Default::default() }); - am.into() + info.clone() }, bucket: bucket.to_string(), endpoint,
