This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 0c1cdb59c fix(services/gcs): migrate to new multipart impl for
gcs_insert_object_request (#2838)
0c1cdb59c is described below
commit 0c1cdb59c70cada10a448501aeb15b1b639c9c60
Author: Mingzhuo Yin <[email protected]>
AuthorDate: Fri Aug 25 10:40:08 2023 +0800
fix(services/gcs): migrate to new multipart impl for
gcs_insert_object_request (#2838)
* fix(services/gcs): migrate to new multipart impl for
gcs_insert_object_request
Signed-off-by: silver-ymz <[email protected]>
* add new ci to test gcs with default storage class
Signed-off-by: silver-ymz <[email protected]>
* add support for stream
Signed-off-by: silver-ymz <[email protected]>
* use json!() to build metadata
Signed-off-by: silver-ymz <[email protected]>
---------
Signed-off-by: silver-ymz <[email protected]>
---
.github/workflows/service_test_gcs.yml | 20 +++++++++++++++
core/src/services/gcs/core.rs | 46 +++++++++++++++++++++-------------
2 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/.github/workflows/service_test_gcs.yml
b/.github/workflows/service_test_gcs.yml
index 53c324644..ca3bde624 100644
--- a/.github/workflows/service_test_gcs.yml
+++ b/.github/workflows/service_test_gcs.yml
@@ -55,3 +55,23 @@ jobs:
OPENDAL_GCS_ROOT: ${{ secrets.OPENDAL_GCS_ROOT }}
OPENDAL_GCS_BUCKET: ${{ secrets.OPENDAL_GCS_BUCKET }}
OPENDAL_GCS_CREDENTIAL: ${{ secrets.OPENDAL_GCS_CREDENTIAL }}
+
+ gcs-with-default-storage-class:
+ runs-on: ubuntu-latest
+ if: github.event_name == 'push' ||
!github.event.pull_request.head.repo.fork
+ steps:
+ - uses: actions/checkout@v3
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+ with:
+ need-nextest: true
+ - name: Test
+ shell: bash
+ working-directory: core
+ run: cargo nextest run gcs
+ env:
+ OPENDAL_GCS_TEST: ${{ secrets.OPENDAL_GCS_TEST }}
+ OPENDAL_GCS_ROOT: ${{ secrets.OPENDAL_GCS_ROOT }}
+ OPENDAL_GCS_BUCKET: ${{ secrets.OPENDAL_GCS_BUCKET }}
+ OPENDAL_GCS_CREDENTIAL: ${{ secrets.OPENDAL_GCS_CREDENTIAL }}
+ OPENDAL_GCS_DEFAULT_STORAGE_CLASS: STANDARD
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index 35cf401ee..3608b445c 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -23,7 +23,6 @@ use std::time::Duration;
use backon::ExponentialBuilder;
use backon::Retryable;
use bytes::Bytes;
-use bytes::BytesMut;
use http::header::CONTENT_LENGTH;
use http::header::CONTENT_RANGE;
use http::header::CONTENT_TYPE;
@@ -37,6 +36,7 @@ use reqsign::GoogleCredentialLoader;
use reqsign::GoogleSigner;
use reqsign::GoogleToken;
use reqsign::GoogleTokenLoader;
+use serde_json::json;
use super::uri::percent_encode_path;
use crate::raw::*;
@@ -234,28 +234,38 @@ impl GcsCore {
req = req.header(CONTENT_LENGTH, size.unwrap_or_default());
if let Some(storage_class) = &self.default_storage_class {
- req = req.header(CONTENT_TYPE, "multipart/related;
boundary=my-boundary");
+ let mut multipart = Multipart::new();
- let mut req_body = BytesMut::with_capacity(100);
- write!(
- &mut req_body,
- "--my-boundary\nContent-Type: application/json;
charset=UTF-8\n\n{{\"storageClass\": \"{}\"}}\n\n--my-boundary\n",
- storage_class
- ).unwrap();
+ multipart = multipart.part(
+ FormDataPart::new("metadata")
+ .header(
+ CONTENT_TYPE,
+ "application/json; charset=UTF-8".parse().unwrap(),
+ )
+ .content(json!({"storageClass":
storage_class}).to_string()),
+ );
- if let Some(mime) = content_type {
- write!(&mut req_body, "Content-Type: {}\n\n", mime).unwrap();
- } else {
- write!(&mut req_body, "Content-Type:
application/octet-stream\n\n").unwrap();
- }
+ let mut media_part = FormDataPart::new("media").header(
+ CONTENT_TYPE,
+ content_type
+ .unwrap_or("application/octet-stream")
+ .parse()
+ .unwrap(),
+ );
- if let AsyncBody::Bytes(bytes) = body {
- req_body.extend_from_slice(&bytes);
+ match body {
+ AsyncBody::Empty => {}
+ AsyncBody::Bytes(bytes) => {
+ media_part = media_part.content(bytes);
+ }
+ AsyncBody::Stream(stream) => {
+ media_part = media_part.stream(size.unwrap(), stream);
+ }
}
- write!(&mut req_body, "\n--my-boundary").unwrap();
- let req_body = AsyncBody::Bytes(req_body.freeze());
- let req = req.body(req_body).map_err(new_request_build_error)?;
+ multipart = multipart.part(media_part);
+
+ let req = multipart.apply(Request::post(url))?;
Ok(req)
} else {
if let Some(content_type) = content_type {
