This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 91544102 feat(services/vercel): Add vercel remote cache support (#2193)
91544102 is described below
commit 91544102c42bc38904c180fcd4a64ce15819ac96
Author: Retrospection <[email protected]>
AuthorDate: Wed May 3 01:02:49 2023 +0800
feat(services/vercel): Add vercel remote cache support (#2193)
* feat(services/vercel): Add vercel remote cache support
Signed-off-by: Retros <[email protected]>
* chore
Signed-off-by: Xuanwo <[email protected]>
* Don't enable not ready services as default
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Retros <[email protected]>
Signed-off-by: Xuanwo <[email protected]>
Co-authored-by: Xuanwo <[email protected]>
---
core/Cargo.toml | 3 +-
core/src/services/mod.rs | 9 +-
core/src/services/onedrive/builder.rs | 2 +-
core/src/services/vercel_artifacts/backend.rs | 133 +++++++++++++++++++++
.../{onedrive => vercel_artifacts}/builder.rs | 68 ++++-------
core/src/services/vercel_artifacts/error.rs | 49 ++++++++
core/src/services/vercel_artifacts/mod.rs | 23 ++++
core/src/services/vercel_artifacts/writer.rs | 71 +++++++++++
core/src/types/scheme.rs | 3 +
9 files changed, 309 insertions(+), 52 deletions(-)
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 7c10ac0b..16679510 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -49,8 +49,6 @@ default = [
"services-s3",
"services-webdav",
"services-webhdfs",
- "services-onedrive",
- "services-gdrive",
]
# Build docs or not.
@@ -145,6 +143,7 @@ services-sftp = [
]
services-sled = ["dep:sled"]
services-supabase = []
+services-vercel-artifacts = []
services-wasabi = [
"dep:reqsign",
"reqsign?/services-aws",
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index a2371940..0ce43b03 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -124,9 +124,9 @@ mod sled;
#[cfg(feature = "services-sled")]
pub use self::sled::Sled;
-// #[cfg(feature = "services-supabase")]
+#[cfg(feature = "services-supabase")]
mod supabase;
-// #[cfg(feature = "services-supabase")]
+#[cfg(feature = "services-supabase")]
pub use supabase::Supabase;
#[cfg(feature = "services-wasabi")]
@@ -154,3 +154,8 @@ pub use gdrive::Gdrive;
#[cfg(feature = "services-webhdfs")]
pub use webhdfs::Webhdfs;
+
+#[cfg(feature = "services-vercel-artifacts")]
+mod vercel_artifacts;
+#[cfg(feature = "services-vercel-artifacts")]
+pub use vercel_artifacts::VercelArtifacts;
diff --git a/core/src/services/onedrive/builder.rs
b/core/src/services/onedrive/builder.rs
index 4dc3da91..beab47a0 100644
--- a/core/src/services/onedrive/builder.rs
+++ b/core/src/services/onedrive/builder.rs
@@ -52,7 +52,7 @@ use crate::*;
/// - `access_token`: set the access_token for Graph API
/// - `root`: Set the work directory for backend
///
-/// You can refer to [`OneDriveBuilder`]'s docs for more information
+/// You can refer to [`OnedriveBuilder`]'s docs for more information
///
/// # Example
///
diff --git a/core/src/services/vercel_artifacts/backend.rs
b/core/src/services/vercel_artifacts/backend.rs
new file mode 100644
index 00000000..c77c810d
--- /dev/null
+++ b/core/src/services/vercel_artifacts/backend.rs
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use async_trait::async_trait;
+use http::{header, Request, Response, StatusCode};
+use std::fmt::Debug;
+
+use crate::{
+ ops::{OpRead, OpWrite},
+ raw::{
+ new_request_build_error, parse_into_metadata, Accessor, AccessorInfo,
AsyncBody,
+ HttpClient, IncomingAsyncBody, RpRead, RpWrite,
+ },
+ types::Result,
+ Capability, Error, ErrorKind,
+};
+
+use super::{error::parse_error, writer::VercelArtifactsWriter};
+
+#[derive(Clone)]
+pub struct VercelArtifactsBackend {
+ pub(crate) access_token: String,
+ pub(crate) client: HttpClient,
+}
+
+impl Debug for VercelArtifactsBackend {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let mut de = f.debug_struct("VercelArtifactsBackend");
+ de.field("access_token", &self.access_token);
+ de.finish()
+ }
+}
+
+#[async_trait]
+impl Accessor for VercelArtifactsBackend {
+ type Reader = IncomingAsyncBody;
+ type BlockingReader = ();
+ type Writer = VercelArtifactsWriter;
+ type BlockingWriter = ();
+ type Pager = ();
+ type BlockingPager = ();
+
+ fn info(&self) -> AccessorInfo {
+ let mut ma = AccessorInfo::default();
+ ma.set_scheme(crate::Scheme::VercelArtifacts)
+ .set_capability(Capability {
+ read: true,
+ write: true,
+ ..Default::default()
+ });
+
+ ma
+ }
+
+ async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead,
Self::Reader)> {
+ let resp = self.vercel_artifacts_get(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ let meta = parse_into_metadata(path, resp.headers())?;
+ Ok((RpRead::with_metadata(meta), resp.into_body()))
+ }
+
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite,
Self::Writer)> {
+ if args.content_length().is_none() {
+ return Err(Error::new(
+ ErrorKind::Unsupported,
+ "write without content length is not supported",
+ ));
+ }
+
+ Ok((
+ RpWrite::default(),
+ VercelArtifactsWriter::new(self.clone(), args, path.to_string()),
+ ))
+ }
+}
+
+impl VercelArtifactsBackend {
+ async fn vercel_artifacts_get(&self, hash: &str) ->
Result<Response<IncomingAsyncBody>> {
+ let url: String = format!("https://api.vercel.com/v8/artifacts/{}";,
hash);
+
+ let mut req = Request::get(&url);
+
+ let auth_header_content = format!("Bearer {}", self.access_token);
+ req = req.header(header::AUTHORIZATION, auth_header_content);
+
+ let req = req
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ self.client.send(req).await
+ }
+
+ pub async fn vercel_artifacts_put(
+ &self,
+ hash: &str,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<Response<IncomingAsyncBody>> {
+ let url = format!("https://api.vercel.com/v8/artifacts/{}";, hash);
+
+ let mut req = Request::put(&url);
+
+ let auth_header_content = format!("Bearer {}", self.access_token);
+ req = req.header(header::AUTHORIZATION, auth_header_content);
+ req = req.header(header::CONTENT_LENGTH, size);
+
+ let req = req.body(body).map_err(new_request_build_error)?;
+
+ self.client.send(req).await
+ }
+}
diff --git a/core/src/services/onedrive/builder.rs
b/core/src/services/vercel_artifacts/builder.rs
similarity index 60%
copy from core/src/services/onedrive/builder.rs
copy to core/src/services/vercel_artifacts/builder.rs
index 4dc3da91..7e760ecd 100644
--- a/core/src/services/onedrive/builder.rs
+++ b/core/src/services/vercel_artifacts/builder.rs
@@ -16,18 +16,13 @@
// under the License.
use std::collections::HashMap;
-use std::fmt::Debug;
-use std::fmt::Formatter;
-use log::debug;
-
-use super::backend::OnedriveBackend;
-use crate::raw::normalize_root;
+use super::backend::VercelArtifactsBackend;
use crate::raw::HttpClient;
use crate::Scheme;
use crate::*;
-/// [OneDrive](https://onedrive.com) backend support.
+/// [Vercel Cache](https://vercel.com/docs/concepts/monorepos/remote-caching)
backend support.
///
/// # Capabilities
///
@@ -35,24 +30,19 @@ use crate::*;
///
/// - [x] read
/// - [x] write
-/// - [ ] copy
-/// - [ ] rename
-/// - [ ] list
+/// - [ ] ~~copy~~
+/// - [ ] ~~rename~~
+/// - [ ] ~~list~~
/// - [ ] ~~scan~~
/// - [ ] ~~presign~~
/// - [ ] blocking
///
/// # Notes
-///
-/// Currently, only OneDrive Personal is supported.
-/// For uploading, only files under 4MB are supported via the Simple Upload
API
(<https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online>).
-///
/// # Configuration
///
-/// - `access_token`: set the access_token for Graph API
-/// - `root`: Set the work directory for backend
+/// - `access_token`: set the access_token for Rest API
///
-/// You can refer to [`OneDriveBuilder`]'s docs for more information
+/// You can refer to [`VercelArtifactsBuilder`]'s docs for more information
///
/// # Example
///
@@ -60,35 +50,28 @@ use crate::*;
///
/// ```no_run
/// use anyhow::Result;
-/// use opendal::services::Onedrive;
+/// use opendal::services::VercelArtifacts;
/// use opendal::Operator;
///
/// #[tokio::main]
/// async fn main() -> Result<()> {
/// // create backend builder
-/// let mut builder = Onedrive::default();
+/// let mut builder = VercelArtifacts::default();
///
-/// builder.access_token("xxx").root("/path/to/root");
+/// builder.access_token("xxx");
///
/// let op: Operator = Operator::new(builder)?.finish();
/// Ok(())
/// }
/// ```
#[derive(Default)]
-pub struct OnedriveBuilder {
+pub struct VercelArtifactsBuilder {
access_token: Option<String>,
- root: Option<String>,
http_client: Option<HttpClient>,
}
-impl Debug for OnedriveBuilder {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("Backend").field("root", &self.root).finish()
- }
-}
-
-impl OnedriveBuilder {
- /// set the bearer access token for OneDrive
+impl VercelArtifactsBuilder {
+ /// set the bearer access token for Vercel
///
/// default: no access token, which leads to failure
pub fn access_token(&mut self, access_token: &str) -> &mut Self {
@@ -96,12 +79,6 @@ impl OnedriveBuilder {
self
}
- /// Set root path of OneDrive folder.
- pub fn root(&mut self, root: &str) -> &mut Self {
- self.root = Some(root.to_string());
- self
- }
-
/// Specify the http client that used by this service.
///
/// # Notes
@@ -114,35 +91,32 @@ impl OnedriveBuilder {
}
}
-impl Builder for OnedriveBuilder {
- const SCHEME: Scheme = Scheme::Onedrive;
+impl Builder for VercelArtifactsBuilder {
+ const SCHEME: Scheme = Scheme::VercelArtifacts;
- type Accessor = OnedriveBackend;
+ type Accessor = VercelArtifactsBackend;
fn from_map(map: HashMap<String, String>) -> Self {
let mut builder = Self::default();
-
- map.get("root").map(|v| builder.root(v));
map.get("access_token").map(|v| builder.access_token(v));
-
builder
}
fn build(&mut self) -> Result<Self::Accessor> {
- let root = normalize_root(&self.root.take().unwrap_or_default());
- debug!("backend use root {}", root);
-
let client = if let Some(client) = self.http_client.take() {
client
} else {
HttpClient::new().map_err(|err| {
err.with_operation("Builder::build")
- .with_context("service", Scheme::Onedrive)
+ .with_context("service", Scheme::VercelArtifacts)
})?
};
match self.access_token.clone() {
- Some(access_token) => Ok(OnedriveBackend::new(root, access_token,
client)),
+ Some(access_token) => Ok(VercelArtifactsBackend {
+ access_token,
+ client,
+ }),
None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not
set")),
}
}
diff --git a/core/src/services/vercel_artifacts/error.rs
b/core/src/services/vercel_artifacts/error.rs
new file mode 100644
index 00000000..6fe6b57d
--- /dev/null
+++ b/core/src/services/vercel_artifacts/error.rs
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use http::Response;
+use http::StatusCode;
+
+use crate::raw::*;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Result;
+
+/// Parse error response into Error.
+pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
+ let (parts, body) = resp.into_parts();
+ let bs = body.bytes().await?;
+
+ let (kind, retryable) = match parts.status {
+ StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+ StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+ StatusCode::INTERNAL_SERVER_ERROR
+ | StatusCode::BAD_GATEWAY
+ | StatusCode::SERVICE_UNAVAILABLE
+ | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+ _ => (ErrorKind::Unexpected, false),
+ };
+
+ let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
+ .with_context("response", format!("{parts:?}"));
+
+ if retryable {
+ err = err.set_temporary();
+ }
+
+ Ok(err)
+}
diff --git a/core/src/services/vercel_artifacts/mod.rs
b/core/src/services/vercel_artifacts/mod.rs
new file mode 100644
index 00000000..3bfddabc
--- /dev/null
+++ b/core/src/services/vercel_artifacts/mod.rs
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod backend;
+mod builder;
+mod error;
+mod writer;
+
+pub use builder::VercelArtifactsBuilder as VercelArtifacts;
diff --git a/core/src/services/vercel_artifacts/writer.rs
b/core/src/services/vercel_artifacts/writer.rs
new file mode 100644
index 00000000..a44ad71a
--- /dev/null
+++ b/core/src/services/vercel_artifacts/writer.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use async_trait::async_trait;
+use bytes::Bytes;
+use http::StatusCode;
+
+use super::backend::VercelArtifactsBackend;
+use super::error::parse_error;
+use crate::ops::OpWrite;
+use crate::raw::*;
+use crate::*;
+
+pub struct VercelArtifactsWriter {
+ backend: VercelArtifactsBackend,
+ op: OpWrite,
+
+ path: String,
+}
+
+impl VercelArtifactsWriter {
+ pub fn new(backend: VercelArtifactsBackend, op: OpWrite, path: String) ->
Self {
+ VercelArtifactsWriter { backend, op, path }
+ }
+}
+
+#[async_trait]
+impl oio::Write for VercelArtifactsWriter {
+ async fn write(&mut self, bs: Bytes) -> Result<()> {
+ let resp = self
+ .backend
+ .vercel_artifacts_put(
+ self.path.as_str(),
+ self.op.content_length().unwrap(),
+ AsyncBody::Bytes(bs),
+ )
+ .await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ resp.into_body().consume().await?;
+ Ok(())
+ }
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ Ok(())
+ }
+
+ async fn close(&mut self) -> Result<()> {
+ Ok(())
+ }
+}
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index 4f7c2020..8fac5113 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -79,6 +79,8 @@ pub enum Scheme {
Sled,
/// [Supabase][crate::services::Supabase]: Supabase storage service
Supabase,
+ /// [Vercel Artifacts][crate::services::vercel_artifacts]: Vercel
Artifacts service, as known as Vercel Remote Caching.
+ VercelArtifacts,
/// [wasabi][crate::services::Wasabi]: Wasabi service
Wasabi,
/// [webdav][crate::services::Webdav]: WebDAV support.
@@ -174,6 +176,7 @@ impl From<Scheme> for &'static str {
Scheme::Sftp => "sftp",
Scheme::Sled => "sled",
Scheme::Supabase => "supabase",
+ Scheme::VercelArtifacts => "vercel_artifacts",
Scheme::Oss => "oss",
Scheme::Wasabi => "wasabi",
Scheme::Webdav => "webdav",
