This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new ec9edd1a feat(services/gcs): Allow setting default storage_class
(#1996)
ec9edd1a is described below
commit ec9edd1aa10043c532635ee9ad66270b2aa3b0b8
Author: silver-ymz <[email protected]>
AuthorDate: Fri Apr 14 20:35:45 2023 +0800
feat(services/gcs): Allow setting default storage_class (#1996)
* feat(services/gcs): Allow setting default storage_class
* fix(services/gcs): Change impl of default storage_class
* fix(services/gcs): Change impl of default storage_class
---
core/src/services/gcs/backend.rs | 22 +++++++++++++++++++
core/src/services/gcs/core.rs | 47 +++++++++++++++++++++++++++++++++-------
2 files changed, 61 insertions(+), 8 deletions(-)
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 619d3c34..cbd5cb5f 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -62,6 +62,7 @@ const DEFAULT_GCS_SCOPE: &str =
"https://www.googleapis.com/auth/devstorage.read
/// - `endpoint`: Customizable endpoint setting
/// - `credentials`: Credential string for GCS OAuth2
/// - `predefined_acl`: Predefined ACL for GCS
+/// - `default_storage_class`: Default storage class for GCS
///
/// You can refer to [`GcsBuilder`]'s docs for more information
///
@@ -88,6 +89,8 @@ const DEFAULT_GCS_SCOPE: &str =
"https://www.googleapis.com/auth/devstorage.read
/// builder.credential("authentication token");
/// // set the predefined ACL for GCS
/// builder.predefined_acl("publicRead");
+/// // set the default storage class for GCS
+/// builder.default_storage_class("STANDARD");
///
/// let op: Operator = Operator::new(builder)?.finish();
/// Ok(())
@@ -115,6 +118,7 @@ pub struct GcsBuilder {
http_client: Option<HttpClient>,
customed_token_loader: Option<Box<dyn GoogleTokenLoad>>,
predefined_acl: Option<String>,
+ default_storage_class: Option<String>,
}
impl GcsBuilder {
@@ -218,6 +222,20 @@ impl GcsBuilder {
};
self
}
+
+ /// Set the default storage class for GCS.
+ ///
+ /// Available values are:
+ /// - `STANDARD`
+ /// - `NEARLINE`
+ /// - `COLDLINE`
+ /// - `ARCHIVE`
+ pub fn default_storage_class(&mut self, class: &str) -> &mut Self {
+ if !class.is_empty() {
+ self.default_storage_class = Some(class.to_string())
+ };
+ self
+ }
}
impl Debug for GcsBuilder {
@@ -233,6 +251,7 @@ impl Debug for GcsBuilder {
if self.predefined_acl.is_some() {
ds.field("predefined_acl", &self.predefined_acl);
}
+ ds.field("default_storage_class", &self.default_storage_class);
ds.finish()
}
}
@@ -250,6 +269,8 @@ impl Builder for GcsBuilder {
map.get("credential").map(|v| builder.credential(v));
map.get("scope").map(|v| builder.scope(v));
map.get("predefined_acl").map(|v| builder.predefined_acl(v));
+ map.get("default_storage_class")
+ .map(|v| builder.default_storage_class(v));
builder
}
@@ -324,6 +345,7 @@ impl Builder for GcsBuilder {
token_loader,
credential_loader: cred_loader,
predefined_acl: self.predefined_acl.clone(),
+ default_storage_class: self.default_storage_class.clone(),
}),
};
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index 780999d5..780b178d 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -21,6 +21,7 @@ use std::fmt::Write;
use backon::ExponentialBuilder;
use backon::Retryable;
+use bytes::BytesMut;
use http::header::CONTENT_LENGTH;
use http::header::CONTENT_TYPE;
use http::Request;
@@ -46,6 +47,7 @@ pub struct GcsCore {
pub credential_loader: GoogleCredentialLoader,
pub predefined_acl: Option<String>,
+ pub default_storage_class: Option<String>,
}
impl Debug for GcsCore {
@@ -139,9 +141,14 @@ impl GcsCore {
let p = build_abs_path(&self.root, path);
let mut url = format!(
- "{}/upload/storage/v1/b/{}/o?uploadType=media&name={}",
+ "{}/upload/storage/v1/b/{}/o?uploadType={}&name={}",
self.endpoint,
self.bucket,
+ if self.default_storage_class.is_some() {
+ "multipart"
+ } else {
+ "media"
+ },
percent_encode_path(&p)
);
@@ -155,14 +162,38 @@ impl GcsCore {
req = req.header(CONTENT_LENGTH, size)
}
- if let Some(mime) = content_type {
- req = req.header(CONTENT_TYPE, mime)
- }
-
- // Set body
- let req = req.body(body).map_err(new_request_build_error)?;
+ if let Some(storage_class) = &self.default_storage_class {
+ req = req.header(CONTENT_TYPE, "multipart/related;
boundary=my-boundary");
+
+ let mut req_body = BytesMut::with_capacity(100);
+ write!(
+ &mut req_body,
+ "--my-boundary\nContent-Type: application/json;
charset=UTF-8\n\n{{\"storageClass\": \"{}\"}}\n\n--my-boundary\n",
+ storage_class
+ ).unwrap();
+
+ if let Some(mime) = content_type {
+ write!(&mut req_body, "Content-Type: {}\n\n", mime).unwrap();
+ } else {
+ write!(&mut req_body, "Content-Type:
application/octet-stream\n\n").unwrap();
+ }
+
+ if let AsyncBody::Bytes(bytes) = body {
+ req_body.extend_from_slice(&bytes);
+ }
+ write!(&mut req_body, "\n--my-boundary").unwrap();
+
+ let req_body = AsyncBody::Bytes(req_body.freeze());
+ let req = req.body(req_body).map_err(new_request_build_error)?;
+ Ok(req)
+ } else {
+ if let Some(content_type) = content_type {
+ req = req.header(CONTENT_TYPE, content_type);
+ }
- Ok(req)
+ let req = req.body(body).map_err(new_request_build_error)?;
+ Ok(req)
+ }
}
pub async fn gcs_get_object_metadata(&self, path: &str) ->
Result<Response<IncomingAsyncBody>> {
