This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 9b9230c68 refactor(services/gdrive): Extract folder search logic
(#3234)
9b9230c68 is described below
commit 9b9230c689ba138424b7a66bce6cb4aa4478eaec
Author: Xuanwo <[email protected]>
AuthorDate: Mon Oct 16 16:52:06 2023 +0800
refactor(services/gdrive): Extract folder search logic (#3234)
* Fix build
Signed-off-by: Xuanwo <[email protected]>
* Save work
Signed-off-by: Xuanwo <[email protected]>
* Save
Signed-off-by: Xuanwo <[email protected]>
* Fix test
Signed-off-by: Xuanwo <[email protected]>
* Fix test
Signed-off-by: Xuanwo <[email protected]>
* Polish parent
Signed-off-by: Xuanwo <[email protected]>
* Fix build
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
core/src/services/gdrive/backend.rs | 53 ++------
core/src/services/gdrive/core.rs | 246 +++++++++++++++++-------------------
2 files changed, 125 insertions(+), 174 deletions(-)
diff --git a/core/src/services/gdrive/backend.rs
b/core/src/services/gdrive/backend.rs
index e7bc2a2bd..879eab7cc 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -31,7 +31,6 @@ use super::pager::GdrivePager;
use super::writer::GdriveWriter;
use crate::raw::*;
use crate::services::gdrive::core::GdriveFile;
-use crate::services::gdrive::core::GdriveFileList;
use crate::types::Result;
use crate::*;
@@ -100,50 +99,22 @@ impl Accessor for GdriveBackend {
async fn create_dir(&self, path: &str, _args: OpCreateDir) ->
Result<RpCreateDir> {
let parent = self.core.ensure_parent_path(path).await?;
- let path = path.split('/').filter(|&x| !x.is_empty()).last().unwrap();
+ // Make sure `/` has been trimmed.
+ let path = get_basename(path).trim_end_matches('/');
// As Google Drive allows files have the same name, we need to check
if the folder exists.
- let resp = self.core.gdrive_search_folder(path, &parent).await?;
- let status = resp.status();
-
- match status {
- StatusCode::OK => {
- let body = resp.into_body().bytes().await?;
- let meta = serde_json::from_slice::<GdriveFileList>(&body)
- .map_err(new_json_deserialize_error)?;
-
- if !meta.files.is_empty() {
- let mut cache = self.core.path_cache.lock().await;
-
- cache.insert(
- build_abs_path(&self.core.root, path),
- meta.files[0].id.clone(),
- );
-
- return Ok(RpCreateDir::default());
- }
- }
- _ => return Err(parse_error(resp).await?),
- }
-
- let resp = self.core.gdrive_create_folder(path, Some(parent)).await?;
+ let folder_id = self.core.gdrive_search_folder(&parent, path).await?;
- let status = resp.status();
-
- match status {
- StatusCode::OK => {
- let body = resp.into_body().bytes().await?;
- let meta = serde_json::from_slice::<GdriveFile>(&body)
- .map_err(new_json_deserialize_error)?;
-
- let mut cache = self.core.path_cache.lock().await;
+ let id = if let Some(id) = folder_id {
+ id
+ } else {
+ self.core.gdrive_create_folder(&parent, path).await?
+ };
- cache.insert(build_abs_path(&self.core.root, path),
meta.id.clone());
+ let mut cache = self.core.path_cache.lock().await;
+ cache.insert(build_abs_path(&self.core.root, path), id);
- Ok(RpCreateDir::default())
- }
- _ => Err(parse_error(resp).await?),
- }
+ Ok(RpCreateDir::default())
}
async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead,
Self::Reader)> {
@@ -233,7 +204,7 @@ impl Accessor for GdriveBackend {
let status = resp.status();
match status {
- StatusCode::NO_CONTENT => {
+ StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => {
let mut cache = self.core.path_cache.lock().await;
cache.remove(&build_abs_path(&self.core.root, path));
diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs
index 4e7216184..3239b281b 100644
--- a/core/src/services/gdrive/core.rs
+++ b/core/src/services/gdrive/core.rs
@@ -96,52 +96,24 @@ impl GdriveCore {
continue;
}
- let mut query = format!(
- "name = \"{}\" and \"{}\" in parents and trashed = false",
- item, parent_id
- );
- if i != file_path_items.len() - 1 || path.ends_with('/') {
- query += " and mimeType =
'application/vnd.google-apps.folder'";
- }
-
- let mut req = Request::get(format!(
- "https://www.googleapis.com/drive/v3/files?q={}";,
- percent_encode_path(&query)
- ))
- .body(AsyncBody::default())
- .map_err(new_request_build_error)?;
-
- self.sign(&mut req).await?;
-
- let resp = self.client.send(req).await?;
- let status = resp.status();
-
- match status {
- StatusCode::OK => {
- let resp_body = &resp.into_body().bytes().await?;
-
- let gdrive_file_list: GdriveFileList =
-
serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
-
- if gdrive_file_list.files.is_empty() {
- return Err(Error::new(
- ErrorKind::NotFound,
- &format!("path not found: {}", item),
- ));
- }
-
- if gdrive_file_list.files.len() > 1 {
- return Err(Error::new(ErrorKind::Unexpected,
&format!("please ensure that the file corresponding to the path exists and is
unique. the response body is {}", String::from_utf8_lossy(resp_body))));
- }
-
- parent_id = gdrive_file_list.files[0].id.clone();
-
- cache.insert(path_part, parent_id.clone());
- }
- _ => {
- return Err(parse_error(resp).await?);
- }
- }
+ let id = if i != file_path_items.len() - 1 || path.ends_with('/') {
+ self.gdrive_search_folder(&parent_id, item).await?
+ } else {
+ self.gdrive_search_file(&parent_id, item)
+ .await?
+ .map(|v| v.id)
+ };
+
+ if let Some(id) = id {
+ parent_id = id;
+ cache.insert(path_part, parent_id.clone());
+ } else {
+ // TODO: return None instead of error.
+ return Err(Error::new(
+ ErrorKind::NotFound,
+ &format!("path not found: {}", item),
+ ));
+ };
}
Ok(parent_id)
@@ -170,84 +142,72 @@ impl GdriveCore {
continue;
}
- let query = format!(
- "name = \"{}\" and \"{}\" in parents and trashed = false and
mimeType = 'application/vnd.google-apps.folder'",
- item, parent
- );
+ let folder_id = self.gdrive_search_folder(&parent, item).await?;
+ let folder_id = if let Some(id) = folder_id {
+ id
+ } else {
+ self.gdrive_create_folder(&parent, item).await?
+ };
+
+ parent = folder_id;
+ cache.insert(path_part, parent.clone());
+ }
+
+ Ok(parent.to_owned())
+ }
- let mut req = Request::get(format!(
- "https://www.googleapis.com/drive/v3/files?q={}";,
- percent_encode_path(&query)
- ))
+ /// Search a folder by name
+ ///
+ /// returns it's file id if exists, otherwise returns `None`.
+ pub async fn gdrive_search_file(
+ &self,
+ parent: &str,
+ basename: &str,
+ ) -> Result<Option<GdriveFile>> {
+ let query =
+ format!("name = \"{basename}\" and \"{parent}\" in parents and
trashed = false");
+ let url = format!(
+ "https://www.googleapis.com/drive/v3/files?q={}";,
+ percent_encode_path(&query)
+ );
+
+ let mut req = Request::get(&url)
.body(AsyncBody::Empty)
.map_err(new_request_build_error)?;
- self.sign(&mut req).await?;
- let resp = self.client.send(req).await?;
- let status = resp.status();
+ self.sign(&mut req).await?;
- match status {
- StatusCode::OK => {
- let resp_body = &resp.into_body().bytes().await?;
+ let resp = self.client.send(req).await?;
+ let status = resp.status();
+ if !status.is_success() {
+ return Err(parse_error(resp).await?);
+ }
- let gdrive_file_list: GdriveFileList =
-
serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
-
- if gdrive_file_list.files.len() != 1 {
- let parent_name = file_path_items[i];
- let resp_body = self
- .gdrive_create_folder(parent_name,
Some(parent.to_owned()))
- .await?
- .into_body()
- .bytes()
- .await?;
- let parent_meta: GdriveFile =
serde_json::from_slice(&resp_body)
- .map_err(new_json_deserialize_error)?;
-
- parent = parent_meta.id;
- } else {
- parent = gdrive_file_list.files[0].id.clone();
- }
-
- cache.insert(path_part, parent.clone());
- }
- StatusCode::NOT_FOUND => {
- let parent_name = file_path_items[i];
- let res = self
- .gdrive_create_folder(parent_name,
Some(parent.to_owned()))
- .await?;
-
- let status = res.status();
-
- match status {
- StatusCode::OK => {
- let parent_id = res.into_body().bytes().await?;
- parent =
String::from_utf8_lossy(&parent_id).to_string();
-
- cache.insert(path_part, parent.clone());
- }
- _ => {
- return Err(parse_error(res).await?);
- }
- }
- }
- _ => {
- return Err(parse_error(resp).await?);
- }
- }
+ let body = resp.into_body().bytes().await?;
+ let mut file_list: GdriveFileList =
+ serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
+
+ if file_list.files.len() > 1 {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "please ensure that the file corresponding to the path is
unique.",
+ ));
}
- Ok(parent.to_owned())
+ Ok(file_list.files.pop())
}
+ /// Search a folder by name
+ ///
+ /// returns it's file id if exists, otherwise returns `None`.
pub async fn gdrive_search_folder(
&self,
- target: &str,
parent: &str,
- ) -> Result<Response<IncomingAsyncBody>> {
+ basename: &str,
+ ) -> Result<Option<String>> {
let query = format!(
- "name = '{}' and '{}' in parents and trashed = false and mimeType
= 'application/vnd.google-apps.folder'",
- target, parent
+ "name = \"{}\" and \"{}\" in parents and trashed = false and
mimeType = 'application/vnd.google-apps.folder'",
+ basename, parent
);
let url = format!(
"https://www.googleapis.com/drive/v3/files?q={}";,
@@ -260,40 +220,60 @@ impl GdriveCore {
self.sign(&mut req).await?;
- self.client.send(req).await
+ let resp = self.client.send(req).await?;
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let body = resp.into_body().bytes().await?;
+ let meta: GdriveFileList =
+
serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
+
+ if let Some(f) = meta.files.first() {
+ Ok(Some(f.id.clone()))
+ } else {
+ Ok(None)
+ }
+ }
+ _ => Err(parse_error(resp).await?),
+ }
}
/// Create a folder.
- /// Should provide the parent folder id.
- /// Or will create the folder in the root folder.
- pub async fn gdrive_create_folder(
- &self,
- name: &str,
- parent: Option<String>,
- ) -> Result<Response<IncomingAsyncBody>> {
+ ///
+ /// # Input
+ ///
+ /// `parent_id` is the parent folder id.
+ ///
+ /// # Output
+ ///
+ /// Returns created folder's id while success, otherwise returns an error.
+ pub async fn gdrive_create_folder(&self, parent_id: &str, name: &str) ->
Result<String> {
let url = "https://www.googleapis.com/drive/v3/files";;
+ let content = serde_json::to_vec(&json!({
+ "name": name,
+ "mimeType": "application/vnd.google-apps.folder",
+ // If the parent is not provided, the folder will be created in
the root folder.
+ "parents": [parent_id],
+ }))
+ .map_err(new_json_serialize_error)?;
+
let mut req = Request::post(url)
.header(header::CONTENT_TYPE, "application/json")
- .body(AsyncBody::Bytes(bytes::Bytes::from(
- serde_json::to_vec(&json!({
- "name": name,
- "mimeType": "application/vnd.google-apps.folder",
- // If the parent is not provided, the folder will be
created in the root folder.
- "parents": [parent.unwrap_or("root".to_owned())],
- }))
- .map_err(|e| {
- Error::new(
- ErrorKind::Unexpected,
- &format!("failed to serialize json(create folder
result): {}", e),
- )
- })?,
- )))
+ .body(AsyncBody::Bytes(Bytes::from(content)))
.map_err(new_request_build_error)?;
self.sign(&mut req).await?;
- self.client.send(req).await
+ let resp = self.client.send(req).await?;
+ if !resp.status().is_success() {
+ return Err(parse_error(resp).await?);
+ }
+ let body = resp.into_body().bytes().await?;
+ let file: GdriveFile =
serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
+
+ Ok(file.id)
}
pub async fn gdrive_stat(&self, path: &str) ->
Result<Response<IncomingAsyncBody>> {
