This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch fix-stat in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit e39939b7530d132ad9c06f27305b8a873761e68e Author: Xuanwo <[email protected]> AuthorDate: Wed Nov 22 19:27:06 2023 +0800 FIx oss Signed-off-by: Xuanwo <[email protected]> --- core/src/services/oss/backend.rs | 30 ++++++++-- core/src/services/oss/core.rs | 110 ++++++++++++++++++++++++++++++++++ core/src/services/oss/lister.rs | 124 +-------------------------------------- 3 files changed, 137 insertions(+), 127 deletions(-) diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index a9e512429..c9d1bd0ad 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -514,6 +514,31 @@ impl Accessor for OssBackend { return Ok(RpStat::new(m)); } + if path.ends_with('/') { + let resp = self + .core + .oss_list_object(path, "", "", Some(1), None) + .await?; + + if resp.status() != StatusCode::OK { + return Err(parse_error(resp).await?); + } + + let bs = resp.into_body().bytes().await?; + + let output: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()) + .map_err(|e| Error::new(ErrorKind::Unexpected, "deserialize xml").set_source(e))?; + + return if !output.contents.is_empty() { + Ok(RpStat::new(Metadata::new(EntryMode::DIR))) + } else { + Err( + Error::new(ErrorKind::NotFound, "The directory is not found") + .with_context("path", path), + ) + }; + } + let resp = self .core .oss_head_object(path, args.if_match(), args.if_none_match()) @@ -523,11 +548,6 @@ impl Accessor for OssBackend { match status { StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new), - StatusCode::NOT_FOUND if path.ends_with('/') => { - let m = Metadata::new(EntryMode::DIR); - Ok(RpStat::new(m)) - } - _ => Err(parse_error(resp).await?), } } diff --git a/core/src/services/oss/core.rs b/core/src/services/oss/core.rs index fad417c4c..66d4654b6 100644 --- a/core/src/services/oss/core.rs +++ b/core/src/services/oss/core.rs @@ -687,6 +687,36 @@ pub struct CompleteMultipartUploadResult { pub etag: String, } +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListObjectsOutput { + pub prefix: String, + pub max_keys: u64, + pub encoding_type: String, + pub is_truncated: bool, + pub common_prefixes: Vec<CommonPrefix>, + pub contents: Vec<ListObjectsOutputContent>, + pub key_count: u64, + + pub next_continuation_token: Option<String>, +} + +#[derive(Default, Debug, Deserialize, PartialEq, Eq)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListObjectsOutputContent { + pub key: String, + pub last_modified: String, + #[serde(rename = "ETag")] + pub etag: String, + pub size: u64, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct CommonPrefix { + pub prefix: String, +} + #[cfg(test)] mod tests { use bytes::Buf; @@ -843,4 +873,84 @@ mod tests { assert_eq!("oss-example", result.bucket); assert_eq!("multipart.data", result.key); } + + #[test] + fn test_parse_list_output() { + let bs = bytes::Bytes::from( + r#"<?xml version="1.0" encoding="UTF-8"?> +<ListBucketResult xmlns="https://doc.oss-cn-hangzhou.aliyuncs.com";> + <Name>examplebucket</Name> + <Prefix></Prefix> + <StartAfter>b</StartAfter> + <MaxKeys>3</MaxKeys> + <EncodingType>url</EncodingType> + <IsTruncated>true</IsTruncated> + <NextContinuationToken>CgJiYw--</NextContinuationToken> + <Contents> + <Key>b/c</Key> + <LastModified>2020-05-18T05:45:54.000Z</LastModified> + <ETag>"35A27C2B9EAEEB6F48FD7FB5861D****"</ETag> + <Size>25</Size> + <StorageClass>STANDARD</StorageClass> + <Owner> + <ID>1686240967192623</ID> + <DisplayName>1686240967192623</DisplayName> + </Owner> + </Contents> + <Contents> + <Key>ba</Key> + <LastModified>2020-05-18T11:17:58.000Z</LastModified> + <ETag>"35A27C2B9EAEEB6F48FD7FB5861D****"</ETag> + <Size>25</Size> + <StorageClass>STANDARD</StorageClass> + <Owner> + <ID>1686240967192623</ID> + <DisplayName>1686240967192623</DisplayName> + </Owner> + </Contents> + <Contents> + <Key>bc</Key> + <LastModified>2020-05-18T05:45:59.000Z</LastModified> + <ETag>"35A27C2B9EAEEB6F48FD7FB5861D****"</ETag> + <Size>25</Size> + <StorageClass>STANDARD</StorageClass> + <Owner> + <ID>1686240967192623</ID> + <DisplayName>1686240967192623</DisplayName> + </Owner> + </Contents> + <KeyCount>3</KeyCount> +</ListBucketResult>"#, + ); + + let out: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()).expect("must_success"); + + assert!(out.is_truncated); + assert_eq!(out.next_continuation_token, Some("CgJiYw--".to_string())); + assert!(out.common_prefixes.is_empty()); + + assert_eq!( + out.contents, + vec![ + ListObjectsOutputContent { + key: "b/c".to_string(), + last_modified: "2020-05-18T05:45:54.000Z".to_string(), + etag: "\"35A27C2B9EAEEB6F48FD7FB5861D****\"".to_string(), + size: 25, + }, + ListObjectsOutputContent { + key: "ba".to_string(), + last_modified: "2020-05-18T11:17:58.000Z".to_string(), + etag: "\"35A27C2B9EAEEB6F48FD7FB5861D****\"".to_string(), + size: 25, + }, + ListObjectsOutputContent { + key: "bc".to_string(), + last_modified: "2020-05-18T05:45:59.000Z".to_string(), + etag: "\"35A27C2B9EAEEB6F48FD7FB5861D****\"".to_string(), + size: 25, + } + ] + ) + } } diff --git a/core/src/services/oss/lister.rs b/core/src/services/oss/lister.rs index 6412cf8bf..9e6eb4652 100644 --- a/core/src/services/oss/lister.rs +++ b/core/src/services/oss/lister.rs @@ -21,16 +21,11 @@ use async_trait::async_trait; use bytes::Buf; use quick_xml::de; use quick_xml::escape::unescape; -use serde::Deserialize; use super::core::*; use super::error::parse_error; use crate::raw::*; -use crate::EntryMode; -use crate::Error; -use crate::ErrorKind; -use crate::Metadata; -use crate::Result; +use crate::*; pub struct OssLister { core: Arc<OssCore>, @@ -86,7 +81,7 @@ impl oio::PageList for OssLister { let bs = resp.into_body().bytes().await?; - let output: ListBucketOutput = de::from_reader(bs.reader()) + let output: ListObjectsOutput = de::from_reader(bs.reader()) .map_err(|e| Error::new(ErrorKind::Unexpected, "deserialize xml").set_source(e))?; ctx.done = !output.is_truncated; @@ -127,118 +122,3 @@ impl oio::PageList for OssLister { Ok(()) } } - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct ListBucketOutput { - prefix: String, - max_keys: u64, - encoding_type: String, - is_truncated: bool, - common_prefixes: Vec<CommonPrefix>, - contents: Vec<Content>, - key_count: u64, - - next_continuation_token: Option<String>, -} - -#[derive(Default, Debug, Deserialize, PartialEq, Eq)] -#[serde(default, rename_all = "PascalCase")] -struct Content { - key: String, - last_modified: String, - #[serde(rename = "ETag")] - etag: String, - size: u64, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct CommonPrefix { - prefix: String, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_list_output() { - let bs = bytes::Bytes::from( - r#"<?xml version="1.0" encoding="UTF-8"?> -<ListBucketResult xmlns="https://doc.oss-cn-hangzhou.aliyuncs.com";> - <Name>examplebucket</Name> - <Prefix></Prefix> - <StartAfter>b</StartAfter> - <MaxKeys>3</MaxKeys> - <EncodingType>url</EncodingType> - <IsTruncated>true</IsTruncated> - <NextContinuationToken>CgJiYw--</NextContinuationToken> - <Contents> - <Key>b/c</Key> - <LastModified>2020-05-18T05:45:54.000Z</LastModified> - <ETag>"35A27C2B9EAEEB6F48FD7FB5861D****"</ETag> - <Size>25</Size> - <StorageClass>STANDARD</StorageClass> - <Owner> - <ID>1686240967192623</ID> - <DisplayName>1686240967192623</DisplayName> - </Owner> - </Contents> - <Contents> - <Key>ba</Key> - <LastModified>2020-05-18T11:17:58.000Z</LastModified> - <ETag>"35A27C2B9EAEEB6F48FD7FB5861D****"</ETag> - <Size>25</Size> - <StorageClass>STANDARD</StorageClass> - <Owner> - <ID>1686240967192623</ID> - <DisplayName>1686240967192623</DisplayName> - </Owner> - </Contents> - <Contents> - <Key>bc</Key> - <LastModified>2020-05-18T05:45:59.000Z</LastModified> - <ETag>"35A27C2B9EAEEB6F48FD7FB5861D****"</ETag> - <Size>25</Size> - <StorageClass>STANDARD</StorageClass> - <Owner> - <ID>1686240967192623</ID> - <DisplayName>1686240967192623</DisplayName> - </Owner> - </Contents> - <KeyCount>3</KeyCount> -</ListBucketResult>"#, - ); - - let out: ListBucketOutput = de::from_reader(bs.reader()).expect("must_success"); - - assert!(out.is_truncated); - assert_eq!(out.next_continuation_token, Some("CgJiYw--".to_string())); - assert!(out.common_prefixes.is_empty()); - - assert_eq!( - out.contents, - vec![ - Content { - key: "b/c".to_string(), - last_modified: "2020-05-18T05:45:54.000Z".to_string(), - etag: "\"35A27C2B9EAEEB6F48FD7FB5861D****\"".to_string(), - size: 25, - }, - Content { - key: "ba".to_string(), - last_modified: "2020-05-18T11:17:58.000Z".to_string(), - etag: "\"35A27C2B9EAEEB6F48FD7FB5861D****\"".to_string(), - size: 25, - }, - Content { - key: "bc".to_string(), - last_modified: "2020-05-18T05:45:59.000Z".to_string(), - etag: "\"35A27C2B9EAEEB6F48FD7FB5861D****\"".to_string(), - size: 25, - } - ] - ) - } -}
