This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch polish-gdrive in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit c891652f2e37a6b717bc11e7e5d0d9679f9cf85a Author: Xuanwo <[email protected]> AuthorDate: Sat Oct 7 16:14:15 2023 +0800 Use moka to replace the path cache Signed-off-by: Xuanwo <[email protected]> --- core/Cargo.toml | 2 +- core/src/services/gdrive/backend.rs | 35 +++++++++++++---------------------- core/src/services/gdrive/builder.rs | 3 ++- core/src/services/gdrive/core.rs | 10 +++++----- core/src/services/gdrive/mod.rs | 3 ++- 5 files changed, 23 insertions(+), 30 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index f1c568c51..796fa91aa 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -131,7 +131,7 @@ services-gcs = [ "reqsign?/services-google", "reqsign?/reqwest_request", ] -services-gdrive = [] +services-gdrive = ["dep:moka"] services-ghac = [] services-hdfs = ["dep:hdrs"] services-http = [] diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index e7bc2a2bd..1cccca607 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -55,23 +55,14 @@ impl Accessor for GdriveBackend { .set_root(&self.core.root) .set_native_capability(Capability { stat: true, - read: true, - list: true, - list_with_delimiter_slash: true, - write: true, - create_dir: true, - rename: true, - delete: true, - copy: true, - ..Default::default() }); @@ -113,9 +104,7 @@ impl Accessor for GdriveBackend { .map_err(new_json_deserialize_error)?; if !meta.files.is_empty() { - let mut cache = self.core.path_cache.lock().await; - - cache.insert( + self.core.path_cache.insert( build_abs_path(&self.core.root, path), meta.files[0].id.clone(), ); @@ -136,9 +125,9 @@ impl Accessor for GdriveBackend { let meta = serde_json::from_slice::<GdriveFile>(&body) .map_err(new_json_deserialize_error)?; - let mut cache = self.core.path_cache.lock().await; - - cache.insert(build_abs_path(&self.core.root, path), meta.id.clone()); + self.core + .path_cache + .insert(build_abs_path(&self.core.root, path), meta.id.clone()); Ok(RpCreateDir::default()) } @@ -216,10 +205,12 @@ impl Accessor for GdriveBackend { let meta = serde_json::from_slice::<GdriveFile>(&body) .map_err(new_json_deserialize_error)?; - let mut cache = self.core.path_cache.lock().await; - - cache.remove(&build_abs_path(&self.core.root, from)); - cache.insert(build_abs_path(&self.core.root, to), meta.id.clone()); + self.core + .path_cache + .invalidate(&build_abs_path(&self.core.root, from)); + self.core + .path_cache + .insert(build_abs_path(&self.core.root, to), meta.id.clone()); Ok(RpRename::default()) } @@ -234,9 +225,9 @@ impl Accessor for GdriveBackend { match status { StatusCode::NO_CONTENT => { - let mut cache = self.core.path_cache.lock().await; - - cache.remove(&build_abs_path(&self.core.root, path)); + self.core + .path_cache + .invalidate(&build_abs_path(&self.core.root, path)); return Ok(RpDelete::default()); } diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs index 984c27909..28966ec17 100644 --- a/core/src/services/gdrive/builder.rs +++ b/core/src/services/gdrive/builder.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; use log::debug; +use moka::sync::Cache; use tokio::sync::Mutex; use super::backend::GdriveBackend; @@ -195,7 +196,7 @@ impl Builder for GdriveBuilder { root, signer: Arc::new(Mutex::new(signer)), client, - path_cache: Arc::default(), + path_cache: Cache::builder().max_capacity(10_000).build(), }), }) } diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 4e7216184..785895a41 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -29,6 +28,7 @@ use http::header; use http::Request; use http::Response; use http::StatusCode; +use moka::sync::Cache; use serde::Deserialize; use serde_json::json; use tokio::sync::Mutex; @@ -55,7 +55,7 @@ pub struct GdriveCore { /// /// - The path is rooted at the root of the Google Drive. /// - The path is absolute path, like `foo/bar`. - pub path_cache: Arc<Mutex<HashMap<String, String>>>, + pub path_cache: Cache<String, String>, } impl Debug for GdriveCore { @@ -80,7 +80,7 @@ impl GdriveCore { pub(crate) async fn get_file_id_by_path(&self, file_path: &str) -> Result<String> { let path = build_abs_path(&self.root, file_path); - let mut cache = self.path_cache.lock().await; + let cache = &self.path_cache; if let Some(id) = cache.get(&path) { return Ok(id.to_owned()); @@ -161,7 +161,7 @@ impl GdriveCore { let mut file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); file_path_items.pop(); - let mut cache = self.path_cache.lock().await; + let cache = &self.path_cache; for (i, item) in file_path_items.iter().enumerate() { let path_part = file_path_items[0..=i].join("/"); @@ -406,7 +406,7 @@ impl GdriveCore { .collect(); source_parent.pop(); - let cache = self.path_cache.lock().await; + let cache = &self.path_cache; let file_name = build_abs_path(&self.root, target) .split('/') diff --git a/core/src/services/gdrive/mod.rs b/core/src/services/gdrive/mod.rs index 2ef0caaec..ec6274b28 100644 --- a/core/src/services/gdrive/mod.rs +++ b/core/src/services/gdrive/mod.rs @@ -17,9 +17,10 @@ mod backend; mod builder; +pub use builder::GdriveBuilder as Gdrive; + mod core; mod error; -pub use builder::GdriveBuilder as Gdrive; mod pager; mod writer;
