This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 46028e2a6 CI(services/gdrive): enable CI testing (#2955)
46028e2a6 is described below
commit 46028e2a618c7218cec41cb1537cf19a4b3bef28
Author: Suyan <[email protected]>
AuthorDate: Mon Aug 28 16:00:43 2023 +0800
CI(services/gdrive): enable CI testing (#2955)
enable CI
Signed-off-by: suyanhanx <[email protected]>
---
.github/workflows/service_test_gdrive.yml | 64 +++++++++++++++++++++++++++++++
core/src/services/gdrive/backend.rs | 24 +++++++++++-
core/src/services/gdrive/core.rs | 26 +++++++++++++
3 files changed, 112 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/service_test_gdrive.yml
b/.github/workflows/service_test_gdrive.yml
new file mode 100644
index 000000000..0d000c384
--- /dev/null
+++ b/.github/workflows/service_test_gdrive.yml
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Service Test Google Drive
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+ paths:
+ - "core/src/**"
+ - "core/tests/**"
+ - "!core/src/docs/**"
+ - "!core/src/services/**"
+ - "core/src/services/gdrive/**"
+ - ".github/workflows/service_test_gdrive.yml"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
+jobs:
+ gdrive:
+ runs-on: ubuntu-latest
+ if: github.event_name == 'push' ||
!github.event.pull_request.head.repo.fork
+ steps:
+ - uses: actions/checkout@v3
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+
+ - name: Load secret
+ id: op-load-secret
+ uses: 1password/load-secrets-action@v1
+ with:
+ export-env: true
+ env:
+ OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+ OPENDAL_GDRIVE_TEST: op://services/gdrive/test
+ OPENDAL_GDRIVE_ROOT: op://services/gdrive/root
+ OPENDAL_GDRIVE_REFRESH_TOKEN: op://services/gdrive/refresh_token
+ OPENDAL_GDRIVE_CLIENT_ID: op://services/gdrive/client_id
+ OPENDAL_GDRIVE_CLIENT_SECRET: op://services/gdrive/client_secret
+
+ - name: Test
+ shell: bash
+ working-directory: core
+ run: cargo test gdrive --features=services-gdrive
diff --git a/core/src/services/gdrive/backend.rs
b/core/src/services/gdrive/backend.rs
index 246e32f33..7886b7daa 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -101,6 +101,10 @@ impl Accessor for GdriveBackend {
.map_err(new_json_deserialize_error)?;
if !meta.files.is_empty() {
+ let mut cache = self.core.path_cache.lock().await;
+
+ cache.insert(path.to_string(), meta.files[0].id.clone());
+
return Ok(RpCreateDir::default());
}
}
@@ -112,7 +116,17 @@ impl Accessor for GdriveBackend {
let status = resp.status();
match status {
- StatusCode::OK => Ok(RpCreateDir::default()),
+ StatusCode::OK => {
+ let body = resp.into_body().bytes().await?;
+ let meta = serde_json::from_slice::<GdriveFile>(&body)
+ .map_err(new_json_deserialize_error)?;
+
+ let mut cache = self.core.path_cache.lock().await;
+
+ cache.insert(path.to_string(), meta.id.clone());
+
+ Ok(RpCreateDir::default())
+ }
_ => Err(parse_error(resp).await?),
}
}
@@ -185,7 +199,13 @@ impl Accessor for GdriveBackend {
let status = resp.status();
match status {
- StatusCode::NO_CONTENT => return Ok(RpDelete::default()),
+ StatusCode::NO_CONTENT => {
+ let mut cache = self.core.path_cache.lock().await;
+
+ cache.remove(path);
+
+ return Ok(RpDelete::default());
+ }
_ => return Err(parse_error(resp).await?),
}
};
diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs
index 389d46569..24ff1cf89 100644
--- a/core/src/services/gdrive/core.rs
+++ b/core/src/services/gdrive/core.rs
@@ -73,10 +73,22 @@ impl GdriveCore {
pub(crate) async fn get_file_id_by_path(&self, file_path: &str) ->
Result<String> {
let path = build_rooted_abs_path(&self.root, file_path);
+ let mut cache = self.path_cache.lock().await;
+
+ if let Some(id) = cache.get(&path) {
+ return Ok(id.to_owned());
+ }
+
let mut parent_id = "root".to_owned();
let file_path_items: Vec<&str> = path.split('/').filter(|&x|
!x.is_empty()).collect();
for (i, item) in file_path_items.iter().enumerate() {
+ let path_part = file_path_items[0..=i].join("/");
+ if let Some(id) = cache.get(&path_part) {
+ parent_id = id.to_owned();
+ continue;
+ }
+
let mut query = format!(
"name = \"{}\" and \"{}\" in parents and trashed = false",
item, parent_id
@@ -116,6 +128,8 @@ impl GdriveCore {
}
parent_id = gdrive_file_list.files[0].id.clone();
+
+ cache.insert(path_part, parent_id.clone());
}
_ => {
return Err(parse_error(resp).await?);
@@ -140,7 +154,15 @@ impl GdriveCore {
let mut file_path_items: Vec<&str> = path.split('/').filter(|&x|
!x.is_empty()).collect();
file_path_items.pop();
+ let mut cache = self.path_cache.lock().await;
+
for (i, item) in file_path_items.iter().enumerate() {
+ let path_part = file_path_items[0..=i].join("/");
+ if let Some(id) = cache.get(&path_part) {
+ parent = id.to_owned();
+ continue;
+ }
+
let query = format!(
"name = \"{}\" and \"{}\" in parents and trashed = false and
mimeType = 'application/vnd.google-apps.folder'",
item, parent
@@ -179,6 +201,8 @@ impl GdriveCore {
} else {
parent = gdrive_file_list.files[0].id.clone();
}
+
+ cache.insert(path_part, parent.clone());
}
StatusCode::NOT_FOUND => {
let parent_name = file_path_items[i];
@@ -192,6 +216,8 @@ impl GdriveCore {
StatusCode::OK => {
let parent_id = res.into_body().bytes().await?;
parent =
String::from_utf8_lossy(&parent_id).to_string();
+
+ cache.insert(path_part, parent.clone());
}
_ => {
return Err(parse_error(res).await?);
