Xuanwo commented on code in PR #4281: URL: https://github.com/apache/opendal/pull/4281#discussion_r1505580167
########## core/src/services/github/core.rs: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde::Serialize; + +use crate::raw::*; +use crate::*; + +use self::constants::USER_AGENT; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "opendal-"; Review Comment: This constant only be used once, we don't need to extract them. ########## core/src/services/github/core.rs: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde::Serialize; + +use crate::raw::*; +use crate::*; + +use self::constants::USER_AGENT; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "opendal-"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, format!("{}{}", USER_AGENT, VERSION)); Review Comment: Please add api version header ########## core/src/services/github/core.rs: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde::Serialize; + +use crate::raw::*; +use crate::*; + +use self::constants::USER_AGENT; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "opendal-"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, format!("{}{}", USER_AGENT, VERSION)); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubCore { + pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(Some(sha.trim_matches('"').to_string())) + } + StatusCode::NOT_FOUND => Ok(None), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await?; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = RequestBody { Review Comment: `RequestBody` is too general, how about `CreateOrUpdateContentsRequest`? Aligning with github's API name make it more easy to understand. Please ensure you use distinct types for the create and delete APIs, even though most are similar. ########## core/src/types/scheme.rs: ########## @@ -422,6 +425,7 @@ impl From<Scheme> for &'static str { Scheme::Postgresql => "postgresql", Scheme::Mysql => "mysql", Scheme::Gdrive => "gdrive", + Scheme::Github => "github_contents", Review Comment: not updated. ########## core/src/types/scheme.rs: ########## @@ -338,6 +340,7 @@ impl FromStr for Scheme { "gdrive" => Ok(Scheme::Gdrive), "ghac" => Ok(Scheme::Ghac), "gridfs" => Ok(Scheme::Gridfs), + "github_contents" => Ok(Scheme::Github), Review Comment: the same ########## core/src/services/github/core.rs: ########## @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde::Serialize; + +use crate::raw::*; +use crate::*; + +use self::constants::USER_AGENT; + +use super::error::parse_error; + +pub(super) mod constants { + pub const USER_AGENT: &str = "opendal-"; +} + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubCore { + #[inline] + pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result<request::Builder> { + let req = req.header(header::USER_AGENT, format!("{}{}", USER_AGENT, VERSION)); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubCore { + pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(Some(sha.trim_matches('"').to_string())) + } + StatusCode::NOT_FOUND => Ok(None), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result<Response<IncomingAsyncBody>> { + let sha = self.get_file_sha(path).await?; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = RequestBody { + message: format!("Write {} at {} via opendal", path, chrono::Local::now()), + content: Some(base64::engine::general_purpose::STANDARD.encode(&bs)), + sha: None, + }; + + if let Some(sha) = sha { + req_body.sha = Some(sha); + } + + let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?; + + let req = req + .header("Accept", "application/vnd.github+json") + .body(AsyncBody::Bytes(Bytes::from(req_body))) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn delete(&self, path: &str) -> Result<()> { + let sha = self.get_file_sha(path).await?; + + match sha { + Some(sha) => { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}";, + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::delete(url); + + let req = self.sign(req)?; + + let req_body = RequestBody { + message: format!("Delete {} at {} via opendal", path, chrono::Local::now()), + sha: Some(sha), + content: None, + }; + + let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?; + + let req = req + .header("Accept", "application/vnd.github.object+json") + .body(AsyncBody::Bytes(Bytes::from(req_body))) + .map_err(new_request_build_error)?; + + let resp = self.send(req).await?; + + match resp.status() { + StatusCode::OK => Ok(()), + _ => Err(parse_error(resp).await?), + } + } + None => Ok(()), Review Comment: How about perform early return for better reading? ```rust let Some(sha) = self.get_file_sha(path).await? else { return Ok(()) } // handle sha ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
