This is an automated email from the ASF dual-hosted git repository. suyanhanx pushed a commit to branch gdrive-cache in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 5bb7f6d6a9ad22d95667349d370a845f0dc3cb11 Author: suyanhanx <[email protected]> AuthorDate: Mon Aug 28 15:37:15 2023 +0800 enable CI Signed-off-by: suyanhanx <[email protected]> --- .github/workflows/service_test_gdrive.yml | 64 +++++++++++++++++++++++++++++++ core/src/services/gdrive/backend.rs | 24 +++++++++++- core/src/services/gdrive/core.rs | 26 +++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/.github/workflows/service_test_gdrive.yml b/.github/workflows/service_test_gdrive.yml new file mode 100644 index 000000000..63b33f739 --- /dev/null +++ b/.github/workflows/service_test_gdrive.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Service Test Google Drive + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "core/tests/**" + - "!core/src/docs/**" + - "!core/src/services/**" + - "core/src/services/gdrive/**" + - ".github/workflows/service_test_gdrive.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + gdrive: + runs-on: ubuntu-latest + if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork + steps: + - uses: actions/checkout@v3 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + + - name: Load secret + id: op-load-secret + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + OPENDAL_GDRIVE_TEST: op://services/gdrive/test + OPENDAL_GDRIVE_ROOT: op://services/gdrive/root + OPENDAL_GDRIVE_REFRESH_TOKEN: op://services/gdrive/refresh_token + OPENDAL_GDRIVE_CLIENT_ID: op://services/gdrive/client_id + OPENDAL_GDRIVE_CLIENT_SECRET: op://services/gdrive/client_secret + + - name: Test + shell: bash + working-directory: core + run: cargo test gdrive --features=services-gdrive -- --test-threads=1 diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index 246e32f33..7886b7daa 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -101,6 +101,10 @@ impl Accessor for GdriveBackend { .map_err(new_json_deserialize_error)?; if !meta.files.is_empty() { + let mut cache = self.core.path_cache.lock().await; + + cache.insert(path.to_string(), meta.files[0].id.clone()); + return Ok(RpCreateDir::default()); } } @@ -112,7 +116,17 @@ impl Accessor for GdriveBackend { let status = resp.status(); match status { - StatusCode::OK => Ok(RpCreateDir::default()), + StatusCode::OK => { + let body = resp.into_body().bytes().await?; + let meta = serde_json::from_slice::<GdriveFile>(&body) + .map_err(new_json_deserialize_error)?; + + let mut cache = self.core.path_cache.lock().await; + + cache.insert(path.to_string(), meta.id.clone()); + + Ok(RpCreateDir::default()) + } _ => Err(parse_error(resp).await?), } } @@ -185,7 +199,13 @@ impl Accessor for GdriveBackend { let status = resp.status(); match status { - StatusCode::NO_CONTENT => return Ok(RpDelete::default()), + StatusCode::NO_CONTENT => { + let mut cache = self.core.path_cache.lock().await; + + cache.remove(path); + + return Ok(RpDelete::default()); + } _ => return Err(parse_error(resp).await?), } }; diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 389d46569..24ff1cf89 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -73,10 +73,22 @@ impl GdriveCore { pub(crate) async fn get_file_id_by_path(&self, file_path: &str) -> Result<String> { let path = build_rooted_abs_path(&self.root, file_path); + let mut cache = self.path_cache.lock().await; + + if let Some(id) = cache.get(&path) { + return Ok(id.to_owned()); + } + let mut parent_id = "root".to_owned(); let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); for (i, item) in file_path_items.iter().enumerate() { + let path_part = file_path_items[0..=i].join("/"); + if let Some(id) = cache.get(&path_part) { + parent_id = id.to_owned(); + continue; + } + let mut query = format!( "name = \"{}\" and \"{}\" in parents and trashed = false", item, parent_id @@ -116,6 +128,8 @@ impl GdriveCore { } parent_id = gdrive_file_list.files[0].id.clone(); + + cache.insert(path_part, parent_id.clone()); } _ => { return Err(parse_error(resp).await?); @@ -140,7 +154,15 @@ impl GdriveCore { let mut file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); file_path_items.pop(); + let mut cache = self.path_cache.lock().await; + for (i, item) in file_path_items.iter().enumerate() { + let path_part = file_path_items[0..=i].join("/"); + if let Some(id) = cache.get(&path_part) { + parent = id.to_owned(); + continue; + } + let query = format!( "name = \"{}\" and \"{}\" in parents and trashed = false and mimeType = 'application/vnd.google-apps.folder'", item, parent @@ -179,6 +201,8 @@ impl GdriveCore { } else { parent = gdrive_file_list.files[0].id.clone(); } + + cache.insert(path_part, parent.clone()); } StatusCode::NOT_FOUND => { let parent_name = file_path_items[i]; @@ -192,6 +216,8 @@ impl GdriveCore { StatusCode::OK => { let parent_id = res.into_body().bytes().await?; parent = String::from_utf8_lossy(&parent_id).to_string(); + + cache.insert(path_part, parent.clone()); } _ => { return Err(parse_error(res).await?);
