This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch xuanwo/azure-common-crate in repository https://gitbox.apache.org/repos/asf/opendal.git
commit f601c0bd73dd2697e9a91f55cb9ae6cf932a1666 Author: Xuanwo <[email protected]> AuthorDate: Mon Dec 8 02:23:11 2025 +0800 refactor: Split all azure storage services and ghac Signed-off-by: Xuanwo <[email protected]> --- core/Cargo.lock | 83 +++++++++++++++++++++- core/Cargo.toml | 12 ++-- core/core/Cargo.toml | 22 ------ core/core/src/raw/mod.rs | 13 ---- core/core/src/services/mod.rs | 20 ------ core/core/src/types/list.rs | 41 ----------- core/services/azblob/Cargo.toml | 45 ++++++++++++ .../azblob => services/azblob/src}/backend.rs | 16 +++-- .../azblob => services/azblob/src}/config.rs | 8 +-- .../azblob => services/azblob/src}/core.rs | 4 +- .../azblob => services/azblob/src}/deleter.rs | 6 +- .../azblob => services/azblob/src}/docs.md | 0 .../azblob => services/azblob/src}/error.rs | 26 ++++--- .../azblob/mod.rs => services/azblob/src/lib.rs} | 6 +- .../azblob => services/azblob/src}/lister.rs | 4 +- .../azblob => services/azblob/src}/writer.rs | 4 +- core/services/azdls/Cargo.toml | 39 ++++++++++ .../azdls => services/azdls/src}/backend.rs | 12 ++-- .../azdls => services/azdls/src}/config.rs | 30 ++++---- .../services/azdls => services/azdls/src}/core.rs | 4 +- .../azdls => services/azdls/src}/deleter.rs | 4 +- .../services/azdls => services/azdls/src}/docs.md | 0 .../services/azdls => services/azdls/src}/error.rs | 26 ++++--- .../azdls/mod.rs => services/azdls/src/lib.rs} | 2 +- .../azdls => services/azdls/src}/lister.rs | 4 +- .../azdls => services/azdls/src}/writer.rs | 4 +- core/services/azfile/Cargo.toml | 38 ++++++++++ .../azfile => services/azfile/src}/backend.rs | 12 ++-- .../azfile => services/azfile/src}/config.rs | 30 ++++---- .../azfile => services/azfile/src}/core.rs | 4 +- .../azfile => services/azfile/src}/deleter.rs | 4 +- .../azfile => services/azfile/src}/docs.md | 0 .../azfile => services/azfile/src}/error.rs | 26 ++++--- .../azfile/mod.rs => services/azfile/src/lib.rs} | 2 +- .../azfile => services/azfile/src}/lister.rs | 4 +- .../azfile => services/azfile/src}/writer.rs | 4 +- core/services/azure-common/Cargo.toml | 32 +++++++++ .../azure.rs => services/azure-common/src/lib.rs} | 23 +++--- core/services/ghac/Cargo.toml | 41 +++++++++++ .../services/ghac => services/ghac/src}/backend.rs | 4 +- .../services/ghac => services/ghac/src}/config.rs | 8 +-- .../services/ghac => services/ghac/src}/core.rs | 4 +- .../services/ghac => services/ghac/src}/docs.md | 0 .../services/ghac => services/ghac/src}/error.rs | 4 +- .../ghac/mod.rs => services/ghac/src/lib.rs} | 2 +- .../services/ghac => services/ghac/src}/writer.rs | 33 ++++++--- core/src/lib.rs | 8 +++ 47 files changed, 463 insertions(+), 255 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 6ed89eba4..ba0554e55 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5407,6 +5407,10 @@ dependencies = [ "log", "opendal-core", "opendal-layer-async-backtrace", + "opendal-service-azblob", + "opendal-service-azdls", + "opendal-service-azfile", + "opendal-service-ghac", "opendal-service-moka", "opendal-service-s3", "opentelemetry", @@ -5475,7 +5479,6 @@ dependencies = [ "futures", "futures-rustls", "getrandom 0.2.16", - "ghac", "governor", "hdfs-native", "hdrs", @@ -5585,6 +5588,84 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-azblob" +version = "0.55.0" +dependencies = [ + "base64 0.22.1", + "bytes", + "ctor", + "http 1.3.1", + "log", + "opendal-core", + "opendal-service-azure-common", + "pretty_assertions", + "quick-xml", + "reqsign", + "serde", + "serde_json", + "sha2", + "uuid", +] + +[[package]] +name = "opendal-service-azdls" +version = "0.55.0" +dependencies = [ + "bytes", + "ctor", + "http 1.3.1", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml", + "reqsign", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azfile" +version = "0.55.0" +dependencies = [ + "bytes", + "ctor", + "http 1.3.1", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml", + "reqsign", + "serde", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.55.0" +dependencies = [ + "http 1.3.1", + "opendal-core", + "reqsign", +] + +[[package]] +name = "opendal-service-ghac" +version = "0.55.0" +dependencies = [ + "bytes", + "ctor", + "ghac", + "http 1.3.1", + "log", + "opendal-core", + "opendal-service-azblob", + "prost 0.13.5", + "reqsign", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "opendal-service-moka" version = "0.55.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index 329ef377d..b40233e4d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -69,9 +69,9 @@ layers-tracing = ["opendal-core/layers-tracing"] reqwest-rustls-tls = ["opendal-core/reqwest-rustls-tls"] services-aliyun-drive = ["opendal-core/services-aliyun-drive"] services-alluxio = ["opendal-core/services-alluxio"] -services-azblob = ["opendal-core/services-azblob"] -services-azdls = ["opendal-core/services-azdls"] -services-azfile = ["opendal-core/services-azfile"] +services-azblob = ["dep:opendal-service-azblob"] +services-azdls = ["dep:opendal-service-azdls"] +services-azfile = ["dep:opendal-service-azfile"] services-b2 = ["opendal-core/services-b2"] services-cacache = ["opendal-core/services-cacache"] services-cloudflare-kv = ["opendal-core/services-cloudflare-kv"] @@ -87,7 +87,7 @@ services-fs = ["opendal-core/services-fs"] services-ftp = ["opendal-core/services-ftp"] services-gcs = ["opendal-core/services-gcs"] services-gdrive = ["opendal-core/services-gdrive"] -services-ghac = ["opendal-core/services-ghac"] +services-ghac = ["dep:opendal-service-ghac"] services-github = ["opendal-core/services-github"] services-gridfs = ["opendal-core/services-gridfs"] services-hdfs = ["opendal-core/services-hdfs"] @@ -155,6 +155,10 @@ opendal-core = { path = "core", version = "0.55.0", default-features = false } opendal-layer-async-backtrace = { path = "layers/async-backtrace", version = "0.55.0", optional = true, default-features = false } opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional = true, default-features = false } opendal-service-moka = { path = "services/moka", version = "0.55.0", optional = true, default-features = false } +opendal-service-azblob = { path = "services/azblob", version = "0.55.0", optional = true, default-features = false } +opendal-service-azdls = { path = "services/azdls", version = "0.55.0", optional = true, default-features = false } +opendal-service-azfile = { path = "services/azfile", version = "0.55.0", optional = true, default-features = false } +opendal-service-ghac = { path = "services/ghac", version = "0.55.0", optional = true, default-features = false } [dev-dependencies] anyhow = { version = "1.0.100", features = ["std"] } diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml index 768d7f292..a590ca022 100644 --- a/core/core/Cargo.toml +++ b/core/core/Cargo.toml @@ -55,9 +55,6 @@ tests = [ "dep:rand", "dep:sha2", "dep:dotenvy", - "services-azblob", - "services-azdls", - "services-azfile", "services-fs", "services-http", "services-memory", @@ -102,22 +99,6 @@ layers-dtrace = ["dep:probe"] services-aliyun-drive = [] services-alluxio = [] -services-azblob = [ - "dep:sha2", - "dep:reqsign", - "reqsign?/services-azblob", - "reqsign?/reqwest_request", -] -services-azdls = [ - "dep:reqsign", - "reqsign?/services-azblob", - "reqsign?/reqwest_request", -] -services-azfile = [ - "dep:reqsign", - "reqsign?/services-azblob", - "reqsign?/reqwest_request", -] services-b2 = [] services-cacache = ["dep:cacache"] services-cloudflare-kv = [] @@ -146,7 +127,6 @@ services-gcs = [ "reqsign?/reqwest_request", ] services-gdrive = ["internal-path-cache"] -services-ghac = ["dep:ghac", "dep:prost", "services-azblob"] services-github = [] services-gridfs = ["dep:mongodb", "dep:mongodb-internal-macros"] services-hdfs = ["dep:hdrs"] @@ -269,8 +249,6 @@ foundationdb = { version = "0.9.0", features = [ "embedded-fdb-include", "fdb-7_3", ], optional = true } -# for services-ghac -ghac = { version = "0.2.0", optional = true } # for services-hdfs hdrs = { version = "0.3.2", optional = true, features = ["async_file"] } # for services-upyun diff --git a/core/core/src/raw/mod.rs b/core/core/src/raw/mod.rs index 446cdb8eb..f85f4bf86 100644 --- a/core/core/src/raw/mod.rs +++ b/core/core/src/raw/mod.rs @@ -29,19 +29,6 @@ mod accessor; pub use accessor::*; -#[cfg(any( - feature = "services-azblob", - feature = "services-azdls", - feature = "services-azfile" -))] -mod azure; -#[cfg(any( - feature = "services-azblob", - feature = "services-azdls", - feature = "services-azfile" -))] -pub(crate) use azure::*; - mod layer; pub use layer::*; diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs index 23db1ae87..549ff6e79 100644 --- a/core/core/src/services/mod.rs +++ b/core/core/src/services/mod.rs @@ -29,21 +29,6 @@ mod alluxio; #[cfg(feature = "services-alluxio")] pub use alluxio::*; -#[cfg(feature = "services-azblob")] -mod azblob; -#[cfg(feature = "services-azblob")] -pub use azblob::*; - -#[cfg(feature = "services-azdls")] -mod azdls; -#[cfg(feature = "services-azdls")] -pub use azdls::*; - -#[cfg(feature = "services-azfile")] -mod azfile; -#[cfg(feature = "services-azfile")] -pub use azfile::*; - #[cfg(feature = "services-b2")] mod b2; #[cfg(feature = "services-b2")] @@ -119,11 +104,6 @@ mod gdrive; #[cfg(feature = "services-gdrive")] pub use gdrive::*; -#[cfg(feature = "services-ghac")] -mod ghac; -#[cfg(feature = "services-ghac")] -pub use ghac::*; - #[cfg(feature = "services-github")] mod github; #[cfg(feature = "services-github")] diff --git a/core/core/src/types/list.rs b/core/core/src/types/list.rs index f1c0dccb5..ab48a7d50 100644 --- a/core/core/src/types/list.rs +++ b/core/core/src/types/list.rs @@ -94,44 +94,3 @@ impl Stream for Lister { Poll::Ready(None) } } - -#[cfg(test)] -#[cfg(feature = "services-azblob")] -mod tests { - use futures::StreamExt; - use futures::future; - - use super::*; - use crate::services::Azblob; - - /// Inspired by <https://gist.github.com/kyle-mccarthy/1e6ae89cc34495d731b91ebf5eb5a3d9> - /// - /// Invalid lister should not panic nor endless loop. - #[tokio::test] - async fn test_invalid_lister() -> Result<()> { - let _ = tracing_subscriber::fmt().try_init(); - - let builder = Azblob::default() - .container("container") - .account_name("account_name") - .account_key("YWNjb3VudF9rZXk=") // Valid base64 encoding of "account_key" - .endpoint("https://account_name.blob.core.windows.net";); - - let operator = Operator::new(builder)?.finish(); - - let lister = operator.lister("/").await?; - - lister - .filter_map(|entry| { - dbg!(&entry); - future::ready(entry.ok()) - }) - .for_each(|entry| { - println!("{entry:?}"); - future::ready(()) - }) - .await; - - Ok(()) - } -} diff --git a/core/services/azblob/Cargo.toml b/core/services/azblob/Cargo.toml new file mode 100644 index 000000000..ddc122b50 --- /dev/null +++ b/core/services/azblob/Cargo.toml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-service-azblob" +version = "0.55.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/apache/opendal"; +description = "Apache OpenDAL Azure Blob Storage service implementation" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +opendal-service-azure-common = { path = "../azure-common", version = "0.55.0" } +base64 = "0.22" +bytes = "1.6" +ctor = "0.6" +http = "1.1" +log = "0.4" +quick-xml = { version = "0.38", features = ["serialize", "overlapped-lists"] } +reqsign = { version = "0.16.5", default-features = false, features = ["reqwest_request", "services-azblob"] } +serde = { version = "1", features = ["derive"] } +sha2 = "0.10" +uuid = { version = "1", features = ["v4", "serde"] } + +[dev-dependencies] +pretty_assertions = "1" +serde_json = "1" diff --git a/core/core/src/services/azblob/backend.rs b/core/services/azblob/src/backend.rs similarity index 97% rename from core/core/src/services/azblob/backend.rs rename to core/services/azblob/src/backend.rs index 0aedc029c..b6e507ca3 100644 --- a/core/core/src/services/azblob/backend.rs +++ b/core/services/azblob/src/backend.rs @@ -39,8 +39,11 @@ use super::error::parse_error; use super::lister::AzblobLister; use super::writer::AzblobWriter; use super::writer::AzblobWriters; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; +use opendal_service_azure_common::{ + AzureStorageService, azure_account_name_from_endpoint, azure_config_from_connection_string, +}; const AZBLOB_BATCH_LIMIT: usize = 256; @@ -278,8 +281,7 @@ impl AzblobBuilder { /// Connection strings can only configure the endpoint, account name and /// authentication information. Users still need to configure container name. pub fn from_connection_string(conn: &str) -> Result<Self> { - let config = - raw::azure_config_from_connection_string(conn, raw::AzureStorageService::Blob)?; + let config = azure_config_from_connection_string(conn, AzureStorageService::Blob)?; Ok(AzblobConfig::from(config).into_builder()) } @@ -320,7 +322,7 @@ impl Builder for AzblobBuilder { .config .account_name .clone() - .or_else(|| raw::azure_account_name_from_endpoint(endpoint.as_str())) + .or_else(|| azure_account_name_from_endpoint(endpoint.as_str())) { config_loader.account_name = Some(v); } @@ -565,6 +567,10 @@ impl Access for AzblobBackend { ErrorKind::Unsupported, "operation is not supported", )), + _ => Err(Error::new( + ErrorKind::Unsupported, + "presign operation is not supported", + )), }; let mut req = req?; diff --git a/core/core/src/services/azblob/config.rs b/core/services/azblob/src/config.rs similarity index 97% rename from core/core/src/services/azblob/config.rs rename to core/services/azblob/src/config.rs index 36345b42e..4fcfbdf81 100644 --- a/core/core/src/services/azblob/config.rs +++ b/core/services/azblob/src/config.rs @@ -88,10 +88,10 @@ impl Debug for AzblobConfig { } } -impl crate::Configurator for AzblobConfig { +impl opendal_core::Configurator for AzblobConfig { type Builder = AzblobBuilder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> { + fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> { let mut map = uri.options().clone(); if let Some(container) = uri.name() { @@ -118,8 +118,8 @@ impl crate::Configurator for AzblobConfig { #[cfg(test)] mod tests { use super::*; - use crate::Configurator; - use crate::types::OperatorUri; + use opendal_core::Configurator; + use opendal_core::OperatorUri; #[test] fn test_container_name_aliases() { diff --git a/core/core/src/services/azblob/core.rs b/core/services/azblob/src/core.rs similarity index 99% rename from core/core/src/services/azblob/core.rs rename to core/services/azblob/src/core.rs index b2d615ff3..792e1ee02 100644 --- a/core/core/src/services/azblob/core.rs +++ b/core/services/azblob/src/core.rs @@ -39,8 +39,8 @@ use serde::Deserialize; use serde::Serialize; use uuid::Uuid; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub mod constants { // Indicates the Blob Storage version that was used to execute the request diff --git a/core/core/src/services/azblob/deleter.rs b/core/services/azblob/src/deleter.rs similarity index 97% rename from core/core/src/services/azblob/deleter.rs rename to core/services/azblob/src/deleter.rs index 4ecca5a4e..a2aba9fd9 100644 --- a/core/core/src/services/azblob/deleter.rs +++ b/core/services/azblob/src/deleter.rs @@ -21,9 +21,9 @@ use http::StatusCode; use super::core::*; use super::error::parse_error; -use crate::raw::oio::BatchDeleteResult; -use crate::raw::*; -use crate::*; +use opendal_core::raw::oio::BatchDeleteResult; +use opendal_core::raw::*; +use opendal_core::*; pub struct AzblobDeleter { core: Arc<AzblobCore>, diff --git a/core/core/src/services/azblob/docs.md b/core/services/azblob/src/docs.md similarity index 100% rename from core/core/src/services/azblob/docs.md rename to core/services/azblob/src/docs.md diff --git a/core/core/src/services/azblob/error.rs b/core/services/azblob/src/error.rs similarity index 92% rename from core/core/src/services/azblob/error.rs rename to core/services/azblob/src/error.rs index 0d26b8cd5..965dfb0cb 100644 --- a/core/core/src/services/azblob/error.rs +++ b/core/services/azblob/src/error.rs @@ -20,12 +20,11 @@ use std::fmt::Debug; use bytes::Buf; use http::Response; use http::StatusCode; +use opendal_core::*; +use opendal_service_azure_common::with_azure_error_response_context; use quick_xml::de; use serde::Deserialize; -use crate::raw::*; -use crate::*; - /// AzblobError is the error returned by azure blob service. #[derive(Default, Deserialize)] #[serde(default, rename_all = "PascalCase")] @@ -83,18 +82,17 @@ pub(super) fn parse_error(resp: Response<Buffer>) -> Error { }; // If there is no body here, fill with error code. - if message.is_empty() { - if let Some(v) = parts.headers.get("x-ms-error-code") { - if let Ok(code) = v.to_str() { - message = format!( - "{:?}", - AzblobError { - code: code.to_string(), - ..Default::default() - } - ) + if message.is_empty() + && let Some(v) = parts.headers.get("x-ms-error-code") + && let Ok(code) = v.to_str() + { + message = format!( + "{:?}", + AzblobError { + code: code.to_string(), + ..Default::default() } - } + ) } let mut err = Error::new(kind, &message); diff --git a/core/core/src/services/azblob/mod.rs b/core/services/azblob/src/lib.rs similarity index 92% rename from core/core/src/services/azblob/mod.rs rename to core/services/azblob/src/lib.rs index 75a1d7211..69fdf9d88 100644 --- a/core/core/src/services/azblob/mod.rs +++ b/core/services/azblob/src/lib.rs @@ -18,15 +18,15 @@ /// Default scheme for azblob service. pub const AZBLOB_SCHEME: &str = "azblob"; -use crate::types::DEFAULT_OPERATOR_REGISTRY; +use opendal_core::DEFAULT_OPERATOR_REGISTRY; mod backend; mod config; -pub(crate) mod core; +pub mod core; mod deleter; mod error; mod lister; -pub(crate) mod writer; +pub mod writer; pub use backend::AzblobBuilder as Azblob; pub use config::AzblobConfig; diff --git a/core/core/src/services/azblob/lister.rs b/core/services/azblob/src/lister.rs similarity index 98% rename from core/core/src/services/azblob/lister.rs rename to core/services/azblob/src/lister.rs index 2d2747125..ffd84c87f 100644 --- a/core/core/src/services/azblob/lister.rs +++ b/core/services/azblob/src/lister.rs @@ -23,8 +23,8 @@ use quick_xml::de; use super::core::AzblobCore; use super::core::ListBlobsOutput; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub struct AzblobLister { core: Arc<AzblobCore>, diff --git a/core/core/src/services/azblob/writer.rs b/core/services/azblob/src/writer.rs similarity index 99% rename from core/core/src/services/azblob/writer.rs rename to core/services/azblob/src/writer.rs index 1559383e1..6d9fc2145 100644 --- a/core/core/src/services/azblob/writer.rs +++ b/core/services/azblob/src/writer.rs @@ -23,8 +23,8 @@ use uuid::Uuid; use super::core::AzblobCore; use super::core::constants::X_MS_VERSION_ID; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; const X_MS_BLOB_TYPE: &str = "x-ms-blob-type"; diff --git a/core/services/azdls/Cargo.toml b/core/services/azdls/Cargo.toml new file mode 100644 index 000000000..c9d8f445a --- /dev/null +++ b/core/services/azdls/Cargo.toml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-service-azdls" +version = "0.55.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/apache/opendal"; +description = "Apache OpenDAL Azure Data Lake Storage Gen2 service implementation" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +opendal-service-azure-common = { path = "../azure-common", version = "0.55.0" } +bytes = "1.6" +ctor = "0.6" +http = "1.1" +log = "0.4" +quick-xml = { version = "0.38", features = ["serialize", "overlapped-lists"] } +reqsign = { version = "0.16.5", default-features = false, features = ["reqwest_request", "services-azblob"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/core/core/src/services/azdls/backend.rs b/core/services/azdls/src/backend.rs similarity index 97% rename from core/core/src/services/azdls/backend.rs rename to core/services/azdls/src/backend.rs index 292345b23..c6e52ce43 100644 --- a/core/core/src/services/azdls/backend.rs +++ b/core/services/azdls/src/backend.rs @@ -34,8 +34,11 @@ use super::error::parse_error; use super::lister::AzdlsLister; use super::writer::AzdlsWriter; use super::writer::AzdlsWriters; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; +use opendal_service_azure_common::{ + AzureStorageService, azure_account_name_from_endpoint, azure_config_from_connection_string, +}; impl From<AzureStorageConfig> for AzdlsConfig { fn from(config: AzureStorageConfig) -> Self { @@ -227,8 +230,7 @@ impl AzdlsBuilder { /// .unwrap(); /// ``` pub fn from_connection_string(conn_str: &str) -> Result<Self> { - let config = - raw::azure_config_from_connection_string(conn_str, raw::AzureStorageService::Adls)?; + let config = azure_config_from_connection_string(conn_str, AzureStorageService::Adls)?; Ok(AzdlsConfig::from(config).into_builder()) } @@ -265,7 +267,7 @@ impl Builder for AzdlsBuilder { .config .account_name .clone() - .or_else(|| raw::azure_account_name_from_endpoint(endpoint.as_str())), + .or_else(|| azure_account_name_from_endpoint(endpoint.as_str())), account_key: self.config.account_key.clone(), sas_token: self.config.sas_token, client_id: self.config.client_id.clone(), diff --git a/core/core/src/services/azdls/config.rs b/core/services/azdls/src/config.rs similarity index 87% rename from core/core/src/services/azdls/config.rs rename to core/services/azdls/src/config.rs index b4db8c059..28efd83b6 100644 --- a/core/core/src/services/azdls/config.rs +++ b/core/services/azdls/src/config.rs @@ -70,29 +70,29 @@ impl Debug for AzdlsConfig { } } -impl crate::Configurator for AzdlsConfig { +impl opendal_core::Configurator for AzdlsConfig { type Builder = AzdlsBuilder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> { + fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> { let mut map = uri.options().clone(); if let Some(authority) = uri.authority() { map.insert("endpoint".to_string(), format!("https://{authority}";)); } - if let Some(host) = uri.name() { - if let Some(account) = host.split('.').next() { - if !account.is_empty() { - map.entry("account_name".to_string()) - .or_insert_with(|| account.to_string()); - } - } + if let Some(account) = uri + .name() + .and_then(|host| host.split('.').next()) + .filter(|account| !account.is_empty()) + { + map.entry("account_name".to_string()) + .or_insert_with(|| account.to_string()); } if let Some(root) = uri.root() { if let Some((filesystem, rest)) = root.split_once('/') { if filesystem.is_empty() { - return Err(crate::Error::new( - crate::ErrorKind::ConfigInvalid, + return Err(opendal_core::Error::new( + opendal_core::ErrorKind::ConfigInvalid, "filesystem is required in uri path", ) .with_context("service", AZDLS_SCHEME)); @@ -107,8 +107,8 @@ impl crate::Configurator for AzdlsConfig { } if !map.contains_key("filesystem") { - return Err(crate::Error::new( - crate::ErrorKind::ConfigInvalid, + return Err(opendal_core::Error::new( + opendal_core::ErrorKind::ConfigInvalid, "filesystem is required", ) .with_context("service", AZDLS_SCHEME)); @@ -129,8 +129,8 @@ impl crate::Configurator for AzdlsConfig { #[cfg(test)] mod tests { use super::*; - use crate::Configurator; - use crate::types::OperatorUri; + use opendal_core::Configurator; + use opendal_core::OperatorUri; #[test] fn from_uri_sets_endpoint_filesystem_root_and_account() { diff --git a/core/core/src/services/azdls/core.rs b/core/services/azdls/src/core.rs similarity index 99% rename from core/core/src/services/azdls/core.rs rename to core/services/azdls/src/core.rs index e88b757cb..f40cfc636 100644 --- a/core/core/src/services/azdls/core.rs +++ b/core/services/azdls/src/core.rs @@ -32,8 +32,8 @@ use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; const X_MS_RENAME_SOURCE: &str = "x-ms-rename-source"; const X_MS_VERSION: &str = "x-ms-version"; diff --git a/core/core/src/services/azdls/deleter.rs b/core/services/azdls/src/deleter.rs similarity index 96% rename from core/core/src/services/azdls/deleter.rs rename to core/services/azdls/src/deleter.rs index 583b61d64..f90c87c8a 100644 --- a/core/core/src/services/azdls/deleter.rs +++ b/core/services/azdls/src/deleter.rs @@ -21,8 +21,8 @@ use http::StatusCode; use super::core::*; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub struct AzdlsDeleter { core: Arc<AzdlsCore>, diff --git a/core/core/src/services/azdls/docs.md b/core/services/azdls/src/docs.md similarity index 100% rename from core/core/src/services/azdls/docs.md rename to core/services/azdls/src/docs.md diff --git a/core/core/src/services/azdls/error.rs b/core/services/azdls/src/error.rs similarity index 88% rename from core/core/src/services/azdls/error.rs rename to core/services/azdls/src/error.rs index 83770195e..11ea17b50 100644 --- a/core/core/src/services/azdls/error.rs +++ b/core/services/azdls/src/error.rs @@ -20,12 +20,11 @@ use std::fmt::Debug; use bytes::Buf; use http::Response; use http::StatusCode; +use opendal_core::*; +use opendal_service_azure_common::with_azure_error_response_context; use quick_xml::de; use serde::Deserialize; -use crate::raw::*; -use crate::*; - /// AzdlsError is the error returned by azure dfs service. #[derive(Default, Deserialize)] #[serde(default, rename_all = "PascalCase")] @@ -81,18 +80,17 @@ pub(super) fn parse_error(resp: Response<Buffer>) -> Error { Err(_) => String::from_utf8_lossy(&bs).into_owned(), }; // If there is no body here, fill with error code. - if message.is_empty() { - if let Some(v) = parts.headers.get("x-ms-error-code") { - if let Ok(code) = v.to_str() { - message = format!( - "{:?}", - AzdlsError { - code: code.to_string(), - ..Default::default() - } - ) + if message.is_empty() + && let Some(v) = parts.headers.get("x-ms-error-code") + && let Ok(code) = v.to_str() + { + message = format!( + "{:?}", + AzdlsError { + code: code.to_string(), + ..Default::default() } - } + ) } let mut err = Error::new(kind, &message); diff --git a/core/core/src/services/azdls/mod.rs b/core/services/azdls/src/lib.rs similarity index 96% rename from core/core/src/services/azdls/mod.rs rename to core/services/azdls/src/lib.rs index 7a75dae09..9c139e977 100644 --- a/core/core/src/services/azdls/mod.rs +++ b/core/services/azdls/src/lib.rs @@ -18,7 +18,7 @@ /// Default scheme for azdls service. pub const AZDLS_SCHEME: &str = "azdls"; -use crate::types::DEFAULT_OPERATOR_REGISTRY; +use opendal_core::DEFAULT_OPERATOR_REGISTRY; mod backend; mod config; diff --git a/core/core/src/services/azdls/lister.rs b/core/services/azdls/src/lister.rs similarity index 99% rename from core/core/src/services/azdls/lister.rs rename to core/services/azdls/src/lister.rs index 2e5744a81..80d5f38e6 100644 --- a/core/core/src/services/azdls/lister.rs +++ b/core/services/azdls/src/lister.rs @@ -23,8 +23,8 @@ use serde_json::de; use super::core::AzdlsCore; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub struct AzdlsLister { core: Arc<AzdlsCore>, diff --git a/core/core/src/services/azdls/writer.rs b/core/services/azdls/src/writer.rs similarity index 99% rename from core/core/src/services/azdls/writer.rs rename to core/services/azdls/src/writer.rs index d46e8e792..e8e4db6b9 100644 --- a/core/core/src/services/azdls/writer.rs +++ b/core/services/azdls/src/writer.rs @@ -23,8 +23,8 @@ use super::core::AzdlsCore; use super::core::FILE; use super::core::X_MS_VERSION_ID; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; /// Writer type for azdls: non-append uses PositionWriter, append uses AppendWriter. pub type AzdlsWriters = TwoWays<oio::PositionWriter<AzdlsWriter>, oio::AppendWriter<AzdlsWriter>>; diff --git a/core/services/azfile/Cargo.toml b/core/services/azfile/Cargo.toml new file mode 100644 index 000000000..20ed5177f --- /dev/null +++ b/core/services/azfile/Cargo.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-service-azfile" +version = "0.55.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/apache/opendal"; +description = "Apache OpenDAL Azure File Storage service implementation" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +opendal-service-azure-common = { path = "../azure-common", version = "0.55.0" } +bytes = "1.6" +ctor = "0.6" +http = "1.1" +log = "0.4" +quick-xml = { version = "0.38", features = ["serialize", "overlapped-lists"] } +reqsign = { version = "0.16.5", default-features = false, features = ["reqwest_request", "services-azblob"] } +serde = { version = "1", features = ["derive"] } diff --git a/core/core/src/services/azfile/backend.rs b/core/services/azfile/src/backend.rs similarity index 97% rename from core/core/src/services/azfile/backend.rs rename to core/services/azfile/src/backend.rs index ebdf6739d..e995e66e7 100644 --- a/core/core/src/services/azfile/backend.rs +++ b/core/services/azfile/src/backend.rs @@ -34,8 +34,11 @@ use super::error::parse_error; use super::lister::AzfileLister; use super::writer::AzfileWriter; use super::writer::AzfileWriters; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; +use opendal_service_azure_common::{ + AzureStorageService, azure_account_name_from_endpoint, azure_config_from_connection_string, +}; impl From<AzureStorageConfig> for AzfileConfig { fn from(config: AzureStorageConfig) -> Self { @@ -160,8 +163,7 @@ impl AzfileBuilder { /// .unwrap(); /// ``` pub fn from_connection_string(conn_str: &str) -> Result<Self> { - let config = - raw::azure_config_from_connection_string(conn_str, raw::AzureStorageService::File)?; + let config = azure_config_from_connection_string(conn_str, AzureStorageService::File)?; Ok(AzfileConfig::from(config).into_builder()) } @@ -188,7 +190,7 @@ impl Builder for AzfileBuilder { .config .account_name .clone() - .or_else(|| raw::azure_account_name_from_endpoint(endpoint.as_str())); + .or_else(|| azure_account_name_from_endpoint(endpoint.as_str())); let account_name = match account_name_option { Some(account_name) => Ok(account_name), diff --git a/core/core/src/services/azfile/config.rs b/core/services/azfile/src/config.rs similarity index 85% rename from core/core/src/services/azfile/config.rs rename to core/services/azfile/src/config.rs index e6624c717..572c375c9 100644 --- a/core/core/src/services/azfile/config.rs +++ b/core/services/azfile/src/config.rs @@ -50,29 +50,29 @@ impl Debug for AzfileConfig { } } -impl crate::Configurator for AzfileConfig { +impl opendal_core::Configurator for AzfileConfig { type Builder = AzfileBuilder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> { + fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> { let mut map = uri.options().clone(); if let Some(authority) = uri.authority() { map.insert("endpoint".to_string(), format!("https://{authority}";)); } - if let Some(host) = uri.name() { - if let Some(account) = host.split('.').next() { - if !account.is_empty() { - map.entry("account_name".to_string()) - .or_insert_with(|| account.to_string()); - } - } + if let Some(account) = uri + .name() + .and_then(|host| host.split('.').next()) + .filter(|account| !account.is_empty()) + { + map.entry("account_name".to_string()) + .or_insert_with(|| account.to_string()); } if let Some(root) = uri.root() { if let Some((share, rest)) = root.split_once('/') { if share.is_empty() { - return Err(crate::Error::new( - crate::ErrorKind::ConfigInvalid, + return Err(opendal_core::Error::new( + opendal_core::ErrorKind::ConfigInvalid, "share name is required in uri path", ) .with_context("service", AZFILE_SCHEME)); @@ -87,8 +87,8 @@ impl crate::Configurator for AzfileConfig { } if !map.contains_key("share_name") { - return Err(crate::Error::new( - crate::ErrorKind::ConfigInvalid, + return Err(opendal_core::Error::new( + opendal_core::ErrorKind::ConfigInvalid, "share name is required", ) .with_context("service", AZFILE_SCHEME)); @@ -109,8 +109,8 @@ impl crate::Configurator for AzfileConfig { #[cfg(test)] mod tests { use super::*; - use crate::Configurator; - use crate::types::OperatorUri; + use opendal_core::Configurator; + use opendal_core::OperatorUri; #[test] fn from_uri_sets_endpoint_share_root_and_account() { diff --git a/core/core/src/services/azfile/core.rs b/core/services/azfile/src/core.rs similarity index 99% rename from core/core/src/services/azfile/core.rs rename to core/services/azfile/src/core.rs index 5cd8205fc..da4257089 100644 --- a/core/core/src/services/azfile/core.rs +++ b/core/services/azfile/src/core.rs @@ -33,8 +33,8 @@ use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; const X_MS_VERSION: &str = "x-ms-version"; const X_MS_WRITE: &str = "x-ms-write"; diff --git a/core/core/src/services/azfile/deleter.rs b/core/services/azfile/src/deleter.rs similarity index 97% rename from core/core/src/services/azfile/deleter.rs rename to core/services/azfile/src/deleter.rs index eb881e9dd..6400cebf8 100644 --- a/core/core/src/services/azfile/deleter.rs +++ b/core/services/azfile/src/deleter.rs @@ -21,8 +21,8 @@ use http::StatusCode; use super::core::*; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub struct AzfileDeleter { core: Arc<AzfileCore>, diff --git a/core/core/src/services/azfile/docs.md b/core/services/azfile/src/docs.md similarity index 100% rename from core/core/src/services/azfile/docs.md rename to core/services/azfile/src/docs.md diff --git a/core/core/src/services/azfile/error.rs b/core/services/azfile/src/error.rs similarity index 88% rename from core/core/src/services/azfile/error.rs rename to core/services/azfile/src/error.rs index ec0bd6970..9c62c281a 100644 --- a/core/core/src/services/azfile/error.rs +++ b/core/services/azfile/src/error.rs @@ -20,12 +20,11 @@ use std::fmt::Debug; use bytes::Buf; use http::Response; use http::StatusCode; +use opendal_core::*; +use opendal_service_azure_common::with_azure_error_response_context; use quick_xml::de; use serde::Deserialize; -use crate::raw::*; -use crate::*; - /// AzfileError is the error returned by azure file service. #[derive(Default, Deserialize)] #[serde(default, rename_all = "PascalCase")] @@ -82,18 +81,17 @@ pub(super) fn parse_error(resp: Response<Buffer>) -> Error { }; // If there is no body here, fill with error code. - if message.is_empty() { - if let Some(v) = parts.headers.get("x-ms-error-code") { - if let Ok(code) = v.to_str() { - message = format!( - "{:?}", - AzfileError { - code: code.to_string(), - ..Default::default() - } - ) + if message.is_empty() + && let Some(v) = parts.headers.get("x-ms-error-code") + && let Ok(code) = v.to_str() + { + message = format!( + "{:?}", + AzfileError { + code: code.to_string(), + ..Default::default() } - } + ) } let mut err = Error::new(kind, &message); diff --git a/core/core/src/services/azfile/mod.rs b/core/services/azfile/src/lib.rs similarity index 96% rename from core/core/src/services/azfile/mod.rs rename to core/services/azfile/src/lib.rs index c659719f8..ea3f41abf 100644 --- a/core/core/src/services/azfile/mod.rs +++ b/core/services/azfile/src/lib.rs @@ -18,7 +18,7 @@ /// Default scheme for azfile service. pub const AZFILE_SCHEME: &str = "azfile"; -use crate::types::DEFAULT_OPERATOR_REGISTRY; +use opendal_core::DEFAULT_OPERATOR_REGISTRY; mod backend; mod config; diff --git a/core/core/src/services/azfile/lister.rs b/core/services/azfile/src/lister.rs similarity index 99% rename from core/core/src/services/azfile/lister.rs rename to core/services/azfile/src/lister.rs index e52cbd0d6..289db82b9 100644 --- a/core/core/src/services/azfile/lister.rs +++ b/core/services/azfile/src/lister.rs @@ -24,8 +24,8 @@ use serde::Deserialize; use super::core::AzfileCore; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub struct AzfileLister { core: Arc<AzfileCore>, diff --git a/core/core/src/services/azfile/writer.rs b/core/services/azfile/src/writer.rs similarity index 98% rename from core/core/src/services/azfile/writer.rs rename to core/services/azfile/src/writer.rs index b8686c625..cce523218 100644 --- a/core/core/src/services/azfile/writer.rs +++ b/core/services/azfile/src/writer.rs @@ -21,8 +21,8 @@ use http::StatusCode; use super::core::AzfileCore; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; pub type AzfileWriters = TwoWays<oio::OneShotWriter<AzfileWriter>, oio::AppendWriter<AzfileWriter>>; diff --git a/core/services/azure-common/Cargo.toml b/core/services/azure-common/Cargo.toml new file mode 100644 index 000000000..7dba50f67 --- /dev/null +++ b/core/services/azure-common/Cargo.toml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-service-azure-common" +version = "0.55.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/apache/opendal"; +description = "Shared Azure helpers for Apache OpenDAL services" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +http = "1.1" +reqsign = { version = "0.16.5", default-features = false } diff --git a/core/core/src/raw/azure.rs b/core/services/azure-common/src/lib.rs similarity index 97% rename from core/core/src/raw/azure.rs rename to core/services/azure-common/src/lib.rs index c9ef55bbc..e380539df 100644 --- a/core/core/src/raw/azure.rs +++ b/core/services/azure-common/src/lib.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(missing_docs)] //! Azure Storage helpers. //! //! This module provides utilities and shared abstractions for services built @@ -25,10 +27,9 @@ use std::collections::HashMap; use http::Uri; use http::response::Parts; +use opendal_core::{Error, ErrorKind, Result}; use reqsign::{AzureStorageConfig, AzureStorageCredential}; -use crate::{Error, ErrorKind, Result}; - /// Parses an [Azure connection string][1] into a configuration object. /// /// The connection string doesn't have to specify all required parameters @@ -38,7 +39,7 @@ use crate::{Error, ErrorKind, Result}; /// the fields used to parse the endpoint. /// /// [1]: https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string -pub(crate) fn azure_config_from_connection_string( +pub fn azure_config_from_connection_string( conn_str: &str, storage: AzureStorageService, ) -> Result<AzureStorageConfig> { @@ -72,21 +73,22 @@ pub(crate) fn azure_config_from_connection_string( /// The service that a connection string refers to. The type influences /// interpretation of endpoint-related fields during parsing. #[derive(PartialEq)] -pub(crate) enum AzureStorageService { +pub enum AzureStorageService { /// Azure Blob Storage. Blob, /// Azure File Storage. - #[cfg(feature = "services-azfile")] File, /// Azure Data Lake Storage Gen2. /// Backed by Blob Storage but exposed through a different endpoint (`dfs`). - #[cfg(feature = "services-azdls")] Adls, } -pub(crate) fn azure_account_name_from_endpoint(endpoint: &str) -> Option<String> { +/// Try to extract the storage account name from an Azure endpoint. +/// +/// Returns `None` if the endpoint doesn't match known Azure Storage suffixes. +pub fn azure_account_name_from_endpoint(endpoint: &str) -> Option<String> { /// Known Azure Storage endpoint suffixes. const KNOWN_ENDPOINT_SUFFIXES: &[&str] = &[ "core.windows.net", // Azure public cloud @@ -184,9 +186,7 @@ fn collect_endpoint( ) -> Result<Option<String>> { match storage { AzureStorageService::Blob => collect_or_build_endpoint(key_values, "BlobEndpoint", "blob"), - #[cfg(feature = "services-azfile")] AzureStorageService::File => collect_or_build_endpoint(key_values, "FileEndpoint", "file"), - #[cfg(feature = "services-azdls")] AzureStorageService::Adls => { // ADLS doesn't have a dedicated endpoint field and we can only // build it from parts. @@ -329,7 +329,7 @@ mod tests { use http::Uri; use reqsign::AzureStorageConfig; - use crate::raw::azure::censor_sas_uri; + use super::censor_sas_uri; use super::{ AzureStorageService, azure_account_name_from_endpoint, azure_config_from_connection_string, @@ -451,7 +451,6 @@ mod tests { ), ]; - #[cfg(feature = "services-azdls")] test_cases.push( ("adls endpoint from parts", (AzureStorageService::Adls, "AccountName=testaccount;EndpointSuffix=core.windows.net;DefaultEndpointsProtocol=https"), @@ -463,7 +462,6 @@ mod tests { ) ); - #[cfg(feature = "services-azfile")] test_cases.extend(vec![ ( "file endpoint from field", @@ -490,7 +488,6 @@ mod tests { ), ]); - #[cfg(feature = "services-azdls")] test_cases.push(( "azdls development storage", (AzureStorageService::Adls, "UseDevelopmentStorage=true"), diff --git a/core/services/ghac/Cargo.toml b/core/services/ghac/Cargo.toml new file mode 100644 index 000000000..d6f6384c0 --- /dev/null +++ b/core/services/ghac/Cargo.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-service-ghac" +version = "0.55.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/apache/opendal"; +description = "Apache OpenDAL GitHub Actions Cache service implementation" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +opendal-service-azblob = { path = "../azblob", version = "0.55.0", default-features = false } +bytes = "1.6" +ghac = { version = "0.2.0", default-features = false } +http = "1.1" +log = "0.4" +prost = { version = "0.13", default-features = false } +reqsign = { version = "0.16.5", default-features = false } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +ctor = "0.6" diff --git a/core/core/src/services/ghac/backend.rs b/core/services/ghac/src/backend.rs similarity index 99% rename from core/core/src/services/ghac/backend.rs rename to core/services/ghac/src/backend.rs index b2c896e15..7063dc601 100644 --- a/core/core/src/services/ghac/backend.rs +++ b/core/services/ghac/src/backend.rs @@ -30,8 +30,8 @@ use super::core::GhacCore; use super::core::*; use super::error::parse_error; use super::writer::GhacWriter; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; fn value_or_env( explicit_value: Option<String>, diff --git a/core/core/src/services/ghac/config.rs b/core/services/ghac/src/config.rs similarity index 94% rename from core/core/src/services/ghac/config.rs rename to core/services/ghac/src/config.rs index 07469326e..4acda69ac 100644 --- a/core/core/src/services/ghac/config.rs +++ b/core/services/ghac/src/config.rs @@ -47,10 +47,10 @@ impl Debug for GhacConfig { } } -impl crate::Configurator for GhacConfig { +impl opendal_core::Configurator for GhacConfig { type Builder = GhacBuilder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> { + fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> { let mut map = uri.options().clone(); if let Some(authority) = uri.authority() { @@ -89,8 +89,8 @@ impl crate::Configurator for GhacConfig { #[cfg(test)] mod tests { use super::*; - use crate::Configurator; - use crate::types::OperatorUri; + use opendal_core::Configurator; + use opendal_core::OperatorUri; #[test] fn from_uri_sets_endpoint_version_and_root() { diff --git a/core/core/src/services/ghac/core.rs b/core/services/ghac/src/core.rs similarity index 99% rename from core/core/src/services/ghac/core.rs rename to core/services/ghac/src/core.rs index d4518bb7f..5effb1bea 100644 --- a/core/core/src/services/ghac/core.rs +++ b/core/services/ghac/src/core.rs @@ -38,8 +38,8 @@ use serde::Deserialize; use serde::Serialize; use super::error::parse_error; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; /// The base url for cache url. pub const CACHE_URL_BASE: &str = "_apis/artifactcache"; diff --git a/core/core/src/services/ghac/docs.md b/core/services/ghac/src/docs.md similarity index 100% rename from core/core/src/services/ghac/docs.md rename to core/services/ghac/src/docs.md diff --git a/core/core/src/services/ghac/error.rs b/core/services/ghac/src/error.rs similarity index 97% rename from core/core/src/services/ghac/error.rs rename to core/services/ghac/src/error.rs index 7efafe1f6..903f1d829 100644 --- a/core/core/src/services/ghac/error.rs +++ b/core/services/ghac/src/error.rs @@ -18,8 +18,8 @@ use http::Response; use http::StatusCode; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; /// Parse error response into Error. pub(super) fn parse_error(resp: Response<Buffer>) -> Error { diff --git a/core/core/src/services/ghac/mod.rs b/core/services/ghac/src/lib.rs similarity index 96% rename from core/core/src/services/ghac/mod.rs rename to core/services/ghac/src/lib.rs index c35782850..c1981b075 100644 --- a/core/core/src/services/ghac/mod.rs +++ b/core/services/ghac/src/lib.rs @@ -18,7 +18,7 @@ /// Default scheme for ghac service. pub const GHAC_SCHEME: &str = "ghac"; -use crate::types::DEFAULT_OPERATOR_REGISTRY; +use opendal_core::DEFAULT_OPERATOR_REGISTRY; mod backend; mod config; diff --git a/core/core/src/services/ghac/writer.rs b/core/services/ghac/src/writer.rs similarity index 89% rename from core/core/src/services/ghac/writer.rs rename to core/services/ghac/src/writer.rs index e0d550753..d6b4f4692 100644 --- a/core/core/src/services/ghac/writer.rs +++ b/core/services/ghac/src/writer.rs @@ -15,28 +15,29 @@ // specific language governing permissions and limitations // under the License. +use std::future::Future; use std::str::FromStr; use std::sync::Arc; use super::core::*; use super::error::parse_error; -use crate::raw::*; -use crate::services::core::AzblobCore; -use crate::services::writer::AzblobWriter; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; +use opendal_service_azblob::core::AzblobCore; +use opendal_service_azblob::writer::AzblobWriter; -pub type GhacWriter = TwoWays<GhacWriterV1, GhacWriterV2>; +pub struct GhacWriter(pub TwoWays<GhacWriterV1, GhacWriterV2>); impl GhacWriter { /// TODO: maybe we can move the signed url logic to azblob service instead. pub fn new(core: Arc<GhacCore>, write_path: String, url: String) -> Result<Self> { match core.service_version { - GhacVersion::V1 => Ok(TwoWays::One(GhacWriterV1 { + GhacVersion::V1 => Ok(GhacWriter(TwoWays::One(GhacWriterV1 { core, path: write_path, url, size: 0, - })), + }))), GhacVersion::V2 => { let uri = http::Uri::from_str(&url) .map_err(new_http_uri_invalid_error)? @@ -123,18 +124,32 @@ impl GhacWriter { }); let w = AzblobWriter::new(azure_core, OpWrite::default(), path.to_string()); let writer = oio::BlockWriter::new(core.info.clone(), w, 4); - Ok(TwoWays::Two(GhacWriterV2 { + Ok(GhacWriter(TwoWays::Two(GhacWriterV2 { core, writer, path: write_path, url, size: 0, - })) + }))) } } } } +impl oio::Write for GhacWriter { + fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend { + self.0.write(bs) + } + + fn abort(&mut self) -> impl Future<Output = Result<()>> + MaybeSend { + self.0.abort() + } + + fn close(&mut self) -> impl Future<Output = Result<Metadata>> + MaybeSend { + self.0.close() + } +} + pub struct GhacWriterV1 { core: Arc<GhacCore>, diff --git a/core/src/lib.rs b/core/src/lib.rs index 8349138de..e6746cef7 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -27,6 +27,14 @@ pub use opendal_core::*; /// Re-export of service implementations. pub mod services { pub use opendal_core::services::*; + #[cfg(feature = "services-azblob")] + pub use opendal_service_azblob::*; + #[cfg(feature = "services-azdls")] + pub use opendal_service_azdls::*; + #[cfg(feature = "services-azfile")] + pub use opendal_service_azfile::*; + #[cfg(feature = "services-ghac")] + pub use opendal_service_ghac::*; #[cfg(feature = "services-moka")] pub use opendal_service_moka::*; #[cfg(feature = "services-s3")]
