This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch fix-stat in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit eb953e63e52f47935d25494e86ced18edb78819d Author: Xuanwo <[email protected]> AuthorDate: Wed Nov 22 19:09:42 2023 +0800 Fix gcs Signed-off-by: Xuanwo <[email protected]> --- core/src/services/gcs/backend.rs | 32 ++++--- core/src/services/gcs/core.rs | 175 ++++++++++++++++++++++++++++++++++++++ core/src/services/gcs/lister.rs | 177 +-------------------------------------- 3 files changed, 198 insertions(+), 186 deletions(-) diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index d528479dc..b3cc960a3 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -30,7 +30,7 @@ use reqsign::GoogleTokenLoader; use serde::Deserialize; use serde_json; -use super::core::GcsCore; +use super::core::*; use super::error::parse_error; use super::lister::GcsLister; use super::writer::GcsWriter; @@ -422,21 +422,35 @@ impl Accessor for GcsBackend { return Ok(RpStat::new(Metadata::new(EntryMode::DIR))); } + if path.ends_with('/') { + let resp = self + .core + .gcs_list_objects(path, "", "", Some(1), None) + .await?; + + let bs = resp.into_body().bytes().await?; + let output: ListResponse = + serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?; + + return if !output.items.is_empty() { + Ok(RpStat::new(Metadata::new(EntryMode::DIR))) + } else { + Err( + Error::new(ErrorKind::NotFound, "The directory is not found") + .with_context("path", path), + ) + }; + } + let resp = self.core.gcs_get_object_metadata(path, &args).await?; if resp.status().is_success() { - // read http response body let slc = resp.into_body().bytes().await?; let meta: GetObjectJsonResponse = serde_json::from_slice(&slc).map_err(new_json_deserialize_error)?; - let mode = if path.ends_with('/') { - EntryMode::DIR - } else { - EntryMode::FILE - }; - let mut m = Metadata::new(mode); + let mut m = Metadata::new(EntryMode::FILE); m.set_etag(&meta.etag); m.set_content_md5(&meta.md5_hash); @@ -453,8 +467,6 @@ impl Accessor for GcsBackend { m.set_last_modified(parse_datetime_from_rfc3339(&meta.updated)?); Ok(RpStat::new(m)) - } else if resp.status() == StatusCode::NOT_FOUND && path.ends_with('/') { - Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { Err(parse_error(resp).await?) } diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs index 5720db2cb..df112e504 100644 --- a/core/src/services/gcs/core.rs +++ b/core/src/services/gcs/core.rs @@ -37,6 +37,7 @@ use reqsign::GoogleCredentialLoader; use reqsign::GoogleSigner; use reqsign::GoogleToken; use reqsign::GoogleTokenLoader; +use serde::Deserialize; use serde_json::json; use super::uri::percent_encode_path; @@ -597,3 +598,177 @@ impl GcsCore { self.send(req).await } } + +/// Response JSON from GCS list objects API. +/// +/// refer to https://cloud.google.com/storage/docs/json_api/v1/objects/list for details +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ListResponse { + /// The continuation token. + /// + /// If this is the last page of results, then no continuation token is returned. + pub next_page_token: Option<String>, + /// Object name prefixes for objects that matched the listing request + /// but were excluded from [items] because of a delimiter. + pub prefixes: Vec<String>, + /// The list of objects, ordered lexicographically by name. + pub items: Vec<ListResponseItem>, +} + +#[derive(Default, Debug, Eq, PartialEq, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ListResponseItem { + pub name: String, + pub size: String, + // metadata + pub etag: String, + pub md5_hash: String, + pub updated: String, + pub content_type: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deserialize_list_response() { + let content = r#" + { + "kind": "storage#objects", + "prefixes": [ + "dir/", + "test/" + ], + "items": [ + { + "kind": "storage#object", + "id": "example/1.png/1660563214863653", + "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png";, + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media";, + "name": "1.png", + "bucket": "example", + "generation": "1660563214863653", + "metageneration": "1", + "contentType": "image/png", + "storageClass": "STANDARD", + "size": "56535", + "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==", + "crc32c": "j/un9g==", + "etag": "CKWasoTgyPkCEAE=", + "timeCreated": "2022-08-15T11:33:34.866Z", + "updated": "2022-08-15T11:33:34.866Z", + "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z" + }, + { + "kind": "storage#object", + "id": "example/2.png/1660563214883337", + "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png";, + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media";, + "name": "2.png", + "bucket": "example", + "generation": "1660563214883337", + "metageneration": "1", + "contentType": "image/png", + "storageClass": "STANDARD", + "size": "45506", + "md5Hash": "e6LsGusU7pFJZk+114NV1g==", + "crc32c": "L00QAg==", + "etag": "CIm0s4TgyPkCEAE=", + "timeCreated": "2022-08-15T11:33:34.886Z", + "updated": "2022-08-15T11:33:34.886Z", + "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z" + } + ] +} + "#; + + let output: ListResponse = + serde_json::from_str(content).expect("JSON deserialize must succeed"); + assert!(output.next_page_token.is_none()); + assert_eq!(output.items.len(), 2); + assert_eq!(output.items[0].name, "1.png"); + assert_eq!(output.items[0].size, "56535"); + assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg=="); + assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE="); + assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z"); + assert_eq!(output.items[1].name, "2.png"); + assert_eq!(output.items[1].size, "45506"); + assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g=="); + assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE="); + assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z"); + assert_eq!(output.items[1].content_type, "image/png"); + assert_eq!(output.prefixes, vec!["dir/", "test/"]) + } + + #[test] + fn test_deserialize_list_response_with_next_page_token() { + let content = r#" + { + "kind": "storage#objects", + "prefixes": [ + "dir/", + "test/" + ], + "nextPageToken": "CgYxMC5wbmc=", + "items": [ + { + "kind": "storage#object", + "id": "example/1.png/1660563214863653", + "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png";, + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media";, + "name": "1.png", + "bucket": "example", + "generation": "1660563214863653", + "metageneration": "1", + "contentType": "image/png", + "storageClass": "STANDARD", + "size": "56535", + "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==", + "crc32c": "j/un9g==", + "etag": "CKWasoTgyPkCEAE=", + "timeCreated": "2022-08-15T11:33:34.866Z", + "updated": "2022-08-15T11:33:34.866Z", + "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z" + }, + { + "kind": "storage#object", + "id": "example/2.png/1660563214883337", + "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png";, + "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media";, + "name": "2.png", + "bucket": "example", + "generation": "1660563214883337", + "metageneration": "1", + "contentType": "image/png", + "storageClass": "STANDARD", + "size": "45506", + "md5Hash": "e6LsGusU7pFJZk+114NV1g==", + "crc32c": "L00QAg==", + "etag": "CIm0s4TgyPkCEAE=", + "timeCreated": "2022-08-15T11:33:34.886Z", + "updated": "2022-08-15T11:33:34.886Z", + "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z" + } + ] +} + "#; + + let output: ListResponse = + serde_json::from_str(content).expect("JSON deserialize must succeed"); + assert_eq!(output.next_page_token, Some("CgYxMC5wbmc=".to_string())); + assert_eq!(output.items.len(), 2); + assert_eq!(output.items[0].name, "1.png"); + assert_eq!(output.items[0].size, "56535"); + assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg=="); + assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE="); + assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z"); + assert_eq!(output.items[1].name, "2.png"); + assert_eq!(output.items[1].size, "45506"); + assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g=="); + assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE="); + assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z"); + assert_eq!(output.prefixes, vec!["dir/", "test/"]) + } +} diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs index 244ca2a93..b9b61f542 100644 --- a/core/src/services/gcs/lister.rs +++ b/core/src/services/gcs/lister.rs @@ -18,10 +18,9 @@ use std::sync::Arc; use async_trait::async_trait; -use serde::Deserialize; use serde_json; -use super::core::GcsCore; +use super::core::*; use super::error::parse_error; use crate::raw::*; use crate::*; @@ -137,177 +136,3 @@ impl oio::PageList for GcsLister { Ok(()) } } - -/// Response JSON from GCS list objects API. -/// -/// refer to https://cloud.google.com/storage/docs/json_api/v1/objects/list for details -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "camelCase")] -struct ListResponse { - /// The continuation token. - /// - /// If this is the last page of results, then no continuation token is returned. - next_page_token: Option<String>, - /// Object name prefixes for objects that matched the listing request - /// but were excluded from [items] because of a delimiter. - prefixes: Vec<String>, - /// The list of objects, ordered lexicographically by name. - items: Vec<ListResponseItem>, -} - -#[derive(Default, Debug, Eq, PartialEq, Deserialize)] -#[serde(default, rename_all = "camelCase")] -struct ListResponseItem { - name: String, - size: String, - // metadata - etag: String, - md5_hash: String, - updated: String, - content_type: String, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_deserialize_list_response() { - let content = r#" - { - "kind": "storage#objects", - "prefixes": [ - "dir/", - "test/" - ], - "items": [ - { - "kind": "storage#object", - "id": "example/1.png/1660563214863653", - "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png";, - "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media";, - "name": "1.png", - "bucket": "example", - "generation": "1660563214863653", - "metageneration": "1", - "contentType": "image/png", - "storageClass": "STANDARD", - "size": "56535", - "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==", - "crc32c": "j/un9g==", - "etag": "CKWasoTgyPkCEAE=", - "timeCreated": "2022-08-15T11:33:34.866Z", - "updated": "2022-08-15T11:33:34.866Z", - "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z" - }, - { - "kind": "storage#object", - "id": "example/2.png/1660563214883337", - "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png";, - "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media";, - "name": "2.png", - "bucket": "example", - "generation": "1660563214883337", - "metageneration": "1", - "contentType": "image/png", - "storageClass": "STANDARD", - "size": "45506", - "md5Hash": "e6LsGusU7pFJZk+114NV1g==", - "crc32c": "L00QAg==", - "etag": "CIm0s4TgyPkCEAE=", - "timeCreated": "2022-08-15T11:33:34.886Z", - "updated": "2022-08-15T11:33:34.886Z", - "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z" - } - ] -} - "#; - - let output: ListResponse = - serde_json::from_str(content).expect("JSON deserialize must succeed"); - assert!(output.next_page_token.is_none()); - assert_eq!(output.items.len(), 2); - assert_eq!(output.items[0].name, "1.png"); - assert_eq!(output.items[0].size, "56535"); - assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg=="); - assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE="); - assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z"); - assert_eq!(output.items[1].name, "2.png"); - assert_eq!(output.items[1].size, "45506"); - assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g=="); - assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE="); - assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z"); - assert_eq!(output.items[1].content_type, "image/png"); - assert_eq!(output.prefixes, vec!["dir/", "test/"]) - } - - #[test] - fn test_deserialize_list_response_with_next_page_token() { - let content = r#" - { - "kind": "storage#objects", - "prefixes": [ - "dir/", - "test/" - ], - "nextPageToken": "CgYxMC5wbmc=", - "items": [ - { - "kind": "storage#object", - "id": "example/1.png/1660563214863653", - "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png";, - "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media";, - "name": "1.png", - "bucket": "example", - "generation": "1660563214863653", - "metageneration": "1", - "contentType": "image/png", - "storageClass": "STANDARD", - "size": "56535", - "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==", - "crc32c": "j/un9g==", - "etag": "CKWasoTgyPkCEAE=", - "timeCreated": "2022-08-15T11:33:34.866Z", - "updated": "2022-08-15T11:33:34.866Z", - "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z" - }, - { - "kind": "storage#object", - "id": "example/2.png/1660563214883337", - "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png";, - "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media";, - "name": "2.png", - "bucket": "example", - "generation": "1660563214883337", - "metageneration": "1", - "contentType": "image/png", - "storageClass": "STANDARD", - "size": "45506", - "md5Hash": "e6LsGusU7pFJZk+114NV1g==", - "crc32c": "L00QAg==", - "etag": "CIm0s4TgyPkCEAE=", - "timeCreated": "2022-08-15T11:33:34.886Z", - "updated": "2022-08-15T11:33:34.886Z", - "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z" - } - ] -} - "#; - - let output: ListResponse = - serde_json::from_str(content).expect("JSON deserialize must succeed"); - assert_eq!(output.next_page_token, Some("CgYxMC5wbmc=".to_string())); - assert_eq!(output.items.len(), 2); - assert_eq!(output.items[0].name, "1.png"); - assert_eq!(output.items[0].size, "56535"); - assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg=="); - assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE="); - assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z"); - assert_eq!(output.items[1].name, "2.png"); - assert_eq!(output.items[1].size, "45506"); - assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g=="); - assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE="); - assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z"); - assert_eq!(output.prefixes, vec!["dir/", "test/"]) - } -}
![https://www.googleapis.com/storage/v1/b/example/o/1.png"](https://www.googleapis.com/storage/v1/b/example/o/1.png"){kind=link}
![https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media"](https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media"){kind=link}
![https://www.googleapis.com/storage/v1/b/example/o/2.png"](https://www.googleapis.com/storage/v1/b/example/o/2.png"){kind=link}
![https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media"](https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media"){kind=link}