This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 5dfc3c29f refactor(services/azblob): add AzblobConfig (#3553)
5dfc3c29f is described below
commit 5dfc3c29fb3975d9ab5377412698b82aa23836c4
Author: Au <[email protected]>
AuthorDate: Sat Nov 11 15:24:10 2023 +0800
refactor(services/azblob): add AzblobConfig (#3553)
* refactor(services/azblob): add AzblobConfig
* docs: add comments to Azblob Config
* feat(services/azblob): impl from_map based on AzblobConfig
---
core/src/services/azblob/backend.rs | 188 ++++++++++++++++++++++--------------
core/src/services/azblob/mod.rs | 1 +
core/src/services/mod.rs | 2 +
3 files changed, 116 insertions(+), 75 deletions(-)
diff --git a/core/src/services/azblob/backend.rs
b/core/src/services/azblob/backend.rs
index 6d6f5612d..acd29e74d 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -29,6 +29,7 @@ use log::debug;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
+use serde::Deserialize;
use sha2::Digest;
use sha2::Sha256;
@@ -53,25 +54,49 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[
const AZBLOB_BATCH_LIMIT: usize = 256;
/// Azure Storage Blob services support.
-#[doc = include_str!("docs.md")]
-#[derive(Default, Clone)]
-pub struct AzblobBuilder {
- root: Option<String>,
- container: String,
- endpoint: Option<String>,
- account_name: Option<String>,
- account_key: Option<String>,
- encryption_key: Option<String>,
- encryption_key_sha256: Option<String>,
- encryption_algorithm: Option<String>,
- sas_token: Option<String>,
- http_client: Option<HttpClient>,
- batch_max_operations: Option<usize>,
+#[derive(Default, Deserialize, Clone)]
+pub struct AzblobConfig {
+ /// The root of Azblob service backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+
+ /// The container name of Azblob service backend.
+ pub container: String,
+
+ /// The endpoint of Azblob service backend.
+ ///
+ /// Endpoint must be full uri, e.g.
+ ///
+ /// - Azblob: `https://accountname.blob.core.windows.net`
+ /// - Azurite: `http://127.0.0.1:10000/devstoreaccount1`
+ pub endpoint: Option<String>,
+
+ /// The account name of Azblob service backend.
+ pub account_name: Option<String>,
+
+ /// The account key of Azblob service backend.
+ pub account_key: Option<String>,
+
+ /// The encryption key of Azblob service backend.
+ pub encryption_key: Option<String>,
+
+ /// The encryption key sha256 of Azblob service backend.
+ pub encryption_key_sha256: Option<String>,
+
+ /// The encryption algorithm of Azblob service backend.
+ pub encryption_algorithm: Option<String>,
+
+ /// The sas token of Azblob service backend.
+ pub sas_token: Option<String>,
+
+ /// The maximum batch operations of Azblob service backend.
+ pub batch_max_operations: Option<usize>,
}
-impl Debug for AzblobBuilder {
+impl Debug for AzblobConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- let mut ds = f.debug_struct("Builder");
+ let mut ds = f.debug_struct("AzblobConfig");
ds.field("root", &self.root);
ds.field("container", &self.container);
@@ -91,13 +116,30 @@ impl Debug for AzblobBuilder {
}
}
+#[doc = include_str!("docs.md")]
+#[derive(Default, Clone)]
+pub struct AzblobBuilder {
+ config: AzblobConfig,
+ http_client: Option<HttpClient>,
+}
+
+impl Debug for AzblobBuilder {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut ds = f.debug_struct("Builder");
+
+ ds.field("config", &self.config);
+
+ ds.finish()
+ }
+}
+
impl AzblobBuilder {
/// Set root of this backend.
///
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
if !root.is_empty() {
- self.root = Some(root.to_string())
+ self.config.root = Some(root.to_string())
}
self
@@ -105,12 +147,12 @@ impl AzblobBuilder {
/// Set container name of this backend.
pub fn container(&mut self, container: &str) -> &mut Self {
- self.container = container.to_string();
+ self.config.container = container.to_string();
self
}
- /// Set endpoint of this backend.
+ /// Set endpoint of this backend
///
/// Endpoint must be full uri, e.g.
///
@@ -119,7 +161,7 @@ impl AzblobBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
- self.endpoint = Some(endpoint.trim_end_matches('/').to_string());
+ self.config.endpoint =
Some(endpoint.trim_end_matches('/').to_string());
}
self
@@ -131,7 +173,7 @@ impl AzblobBuilder {
/// - If not, we will try to load it from environment.
pub fn account_name(&mut self, account_name: &str) -> &mut Self {
if !account_name.is_empty() {
- self.account_name = Some(account_name.to_string());
+ self.config.account_name = Some(account_name.to_string());
}
self
@@ -143,7 +185,7 @@ impl AzblobBuilder {
/// - If not, we will try to load it from environment.
pub fn account_key(&mut self, account_key: &str) -> &mut Self {
if !account_key.is_empty() {
- self.account_key = Some(account_key.to_string());
+ self.config.account_key = Some(account_key.to_string());
}
self
@@ -163,7 +205,7 @@ impl AzblobBuilder {
/// Please use `server_side_encryption_with_*` helpers if even possible.
pub fn encryption_key(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.encryption_key = Some(v.to_string());
+ self.config.encryption_key = Some(v.to_string());
}
self
@@ -183,7 +225,7 @@ impl AzblobBuilder {
/// Please use `server_side_encryption_with_*` helpers if even possible.
pub fn encryption_key_sha256(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.encryption_key_sha256 = Some(v.to_string());
+ self.config.encryption_key_sha256 = Some(v.to_string());
}
self
@@ -203,7 +245,7 @@ impl AzblobBuilder {
/// Please use `server_side_encryption_with_*` helpers if even possible.
pub fn encryption_algorithm(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.encryption_algorithm = Some(v.to_string());
+ self.config.encryption_algorithm = Some(v.to_string());
}
self
@@ -224,9 +266,10 @@ impl AzblobBuilder {
/// for more info.
pub fn server_side_encryption_with_customer_key(&mut self, key: &[u8]) ->
&mut Self {
// Only AES256 is supported for now
- self.encryption_algorithm = Some("AES256".to_string());
- self.encryption_key = Some(BASE64_STANDARD.encode(key));
- self.encryption_key_sha256 =
Some(BASE64_STANDARD.encode(Sha256::digest(key).as_slice()));
+ self.config.encryption_algorithm = Some("AES256".to_string());
+ self.config.encryption_key = Some(BASE64_STANDARD.encode(key));
+ self.config.encryption_key_sha256 =
+ Some(BASE64_STANDARD.encode(Sha256::digest(key).as_slice()));
self
}
@@ -239,7 +282,7 @@ impl AzblobBuilder {
/// for more info.
pub fn sas_token(&mut self, sas_token: &str) -> &mut Self {
if !sas_token.is_empty() {
- self.sas_token = Some(sas_token.to_string());
+ self.config.sas_token = Some(sas_token.to_string());
}
self
@@ -258,7 +301,7 @@ impl AzblobBuilder {
/// Set maximum batch operations of this backend.
pub fn batch_max_operations(&mut self, batch_max_operations: usize) ->
&mut Self {
- self.batch_max_operations = Some(batch_max_operations);
+ self.config.batch_max_operations = Some(batch_max_operations);
self
}
@@ -331,6 +374,7 @@ impl AzblobBuilder {
} else if let Some(v) = conn_map.get("EndpointSuffix") {
let protocol =
conn_map.get("DefaultEndpointsProtocol").unwrap_or(&"https");
let account_name = builder
+ .config
.account_name
.as_ref()
.ok_or_else(|| {
@@ -353,41 +397,31 @@ impl Builder for AzblobBuilder {
type Accessor = AzblobBackend;
fn from_map(map: HashMap<String, String>) -> Self {
- let mut builder = AzblobBuilder::default();
+ let config = AzblobConfig::deserialize(ConfigDeserializer::new(map))
+ .expect("config deserialize must succeed");
- map.get("root").map(|v| builder.root(v));
- map.get("container").map(|v| builder.container(v));
- map.get("endpoint").map(|v| builder.endpoint(v));
- map.get("account_name").map(|v| builder.account_name(v));
- map.get("account_key").map(|v| builder.account_key(v));
- map.get("encryption_key").map(|v| builder.encryption_key(v));
- map.get("encryption_key_sha256")
- .map(|v| builder.encryption_key_sha256(v));
- map.get("encryption_algorithm")
- .map(|v| builder.encryption_algorithm(v));
- map.get("sas_token").map(|v| builder.sas_token(v));
- map.get("batch_max_operations")
- .map(|v|
builder.batch_max_operations(v.parse::<usize>().unwrap()));
-
- builder
+ AzblobBuilder {
+ config,
+ http_client: None,
+ }
}
fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);
- let root = normalize_root(&self.root.take().unwrap_or_default());
+ let root =
normalize_root(&self.config.root.take().unwrap_or_default());
debug!("backend use root {}", root);
// Handle endpoint, region and container name.
- let container = match self.container.is_empty() {
- false => Ok(&self.container),
+ let container = match self.config.container.is_empty() {
+ false => Ok(&self.config.container),
true => Err(Error::new(ErrorKind::ConfigInvalid, "container is
empty")
.with_operation("Builder::build")
.with_context("service", Scheme::Azblob)),
}?;
debug!("backend use container {}", &container);
- let endpoint = match &self.endpoint {
+ let endpoint = match &self.config.endpoint {
Some(endpoint) => Ok(endpoint.clone()),
None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is
empty")
.with_operation("Builder::build")
@@ -406,30 +440,31 @@ impl Builder for AzblobBuilder {
let config_loader = AzureStorageConfig {
account_name: self
+ .config
.account_name
.clone()
.or_else(||
infer_storage_name_from_endpoint(endpoint.as_str())),
- account_key: self.account_key.clone(),
- sas_token: self.sas_token.clone(),
+ account_key: self.config.account_key.clone(),
+ sas_token: self.config.sas_token.clone(),
..Default::default()
};
let encryption_key =
- match &self.encryption_key {
+ match &self.config.encryption_key {
None => None,
Some(v) => Some(build_header_value(v).map_err(|err| {
err.with_context("key",
"server_side_encryption_customer_key")
})?),
};
- let encryption_key_sha256 = match &self.encryption_key_sha256 {
+ let encryption_key_sha256 = match &self.config.encryption_key_sha256 {
None => None,
Some(v) => Some(build_header_value(v).map_err(|err| {
err.with_context("key",
"server_side_encryption_customer_key_sha256")
})?),
};
- let encryption_algorithm = match &self.encryption_algorithm {
+ let encryption_algorithm = match &self.config.encryption_algorithm {
None => None,
Some(v) => {
if v == "AES256" {
@@ -449,7 +484,10 @@ impl Builder for AzblobBuilder {
let signer = AzureStorageSigner::new();
- let batch_max_operations =
self.batch_max_operations.unwrap_or(AZBLOB_BATCH_LIMIT);
+ let batch_max_operations = self
+ .config
+ .batch_max_operations
+ .unwrap_or(AZBLOB_BATCH_LIMIT);
debug!("backend build finished: {:?}", &self);
Ok(AzblobBackend {
@@ -459,14 +497,14 @@ impl Builder for AzblobBuilder {
encryption_key,
encryption_key_sha256,
encryption_algorithm,
- container: self.container.clone(),
+ container: self.config.container.clone(),
client,
loader: cred_loader,
signer,
batch_max_operations,
}),
- has_sas_token: self.sas_token.is_some(),
+ has_sas_token: self.config.sas_token.is_some(),
})
}
}
@@ -793,7 +831,7 @@ mod tests {
assert_eq!(azblob.core.container, "container".to_string());
assert_eq!(
- azblob_builder.account_key.unwrap(),
+ azblob_builder.config.account_key.unwrap(),
"account-key".to_string()
);
}
@@ -814,7 +852,7 @@ mod tests {
assert_eq!(azblob.core.container, "container".to_string());
- assert_eq!(azblob_builder.account_key, None);
+ assert_eq!(azblob_builder.config.account_key, None);
}
#[test]
@@ -837,11 +875,11 @@ mod tests {
assert_eq!(azblob.core.container, "container".to_string());
assert_eq!(
- azblob_builder.account_key.unwrap(),
+ azblob_builder.config.account_key.unwrap(),
"account-key".to_string()
);
- assert_eq!(azblob_builder.sas_token.unwrap(), "sas".to_string());
+ assert_eq!(azblob_builder.config.sas_token.unwrap(),
"sas".to_string());
}
#[test]
@@ -858,11 +896,11 @@ TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
.expect("from connection string must succeed");
assert_eq!(
- builder.endpoint.unwrap(),
+ builder.config.endpoint.unwrap(),
"http://127.0.0.1:10000/devstoreaccount1";
);
- assert_eq!(builder.account_name.unwrap(), "devstoreaccount1");
- assert_eq!(builder.account_key.unwrap(),
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==");
+ assert_eq!(builder.config.account_name.unwrap(), "devstoreaccount1");
+ assert_eq!(builder.config.account_key.unwrap(),
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==");
let builder = AzblobBuilder::from_connection_string(
r#"
@@ -875,11 +913,11 @@ EndpointSuffix=core.chinacloudapi.cn;
.expect("from connection string must succeed");
assert_eq!(
- builder.endpoint.unwrap(),
+ builder.config.endpoint.unwrap(),
"https://storagesample.blob.core.chinacloudapi.cn";
);
- assert_eq!(builder.account_name.unwrap(), "storagesample");
- assert_eq!(builder.account_key.unwrap(), "account-key")
+ assert_eq!(builder.config.account_name.unwrap(), "storagesample");
+ assert_eq!(builder.config.account_key.unwrap(), "account-key")
}
#[test]
@@ -896,12 +934,12 @@
SharedAccessSignature=sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:0
.expect("from connection string must succeed");
assert_eq!(
- builder.endpoint.unwrap(),
+ builder.config.endpoint.unwrap(),
"http://127.0.0.1:10000/devstoreaccount1";
);
- assert_eq!(builder.sas_token.unwrap(),
"sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
- assert_eq!(builder.account_name, None);
- assert_eq!(builder.account_key, None);
+ assert_eq!(builder.config.sas_token.unwrap(),
"sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
+ assert_eq!(builder.config.account_name, None);
+ assert_eq!(builder.config.account_key, None);
}
#[test]
@@ -917,8 +955,8 @@
SharedAccessSignature=sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:0
.expect("from connection string must succeed");
// SAS should be preferred over shared key
- assert_eq!(builder.sas_token.unwrap(),
"sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
- assert_eq!(builder.account_name, None);
- assert_eq!(builder.account_key, None);
+ assert_eq!(builder.config.sas_token.unwrap(),
"sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
+ assert_eq!(builder.config.account_name, None);
+ assert_eq!(builder.config.account_key, None);
}
}
diff --git a/core/src/services/azblob/mod.rs b/core/src/services/azblob/mod.rs
index 8170b176b..efc6e42f0 100644
--- a/core/src/services/azblob/mod.rs
+++ b/core/src/services/azblob/mod.rs
@@ -17,6 +17,7 @@
mod backend;
pub use backend::AzblobBuilder as Azblob;
+pub use backend::AzblobConfig;
mod core;
mod error;
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index d4298e9dc..8ed196f87 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -23,6 +23,8 @@
mod azblob;
#[cfg(feature = "services-azblob")]
pub use azblob::Azblob;
+#[cfg(feature = "services-azblob")]
+pub use azblob::AzblobConfig;
#[cfg(feature = "services-azdls")]
mod azdls;
