This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch fix-stat in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit b66c7cd83f2b292ee6b94ac4f00caf162fe117fa Author: Xuanwo <[email protected]> AuthorDate: Wed Nov 22 18:46:41 2023 +0800 Fix cos Signed-off-by: Xuanwo <[email protected]> --- core/src/services/azfile/core.rs | 2 +- core/src/services/cos/backend.rs | 27 ++++++++-- core/src/services/cos/core.rs | 96 +++++++++++++++++++++++++++++++++++ core/src/services/cos/lister.rs | 105 ++------------------------------------- core/src/services/s3/backend.rs | 2 +- core/src/services/s3/core.rs | 14 +++--- core/src/services/s3/lister.rs | 5 +- 7 files changed, 134 insertions(+), 117 deletions(-) diff --git a/core/src/services/azfile/core.rs b/core/src/services/azfile/core.rs index 4887051b1..d976b19e2 100644 --- a/core/src/services/azfile/core.rs +++ b/core/src/services/azfile/core.rs @@ -438,7 +438,7 @@ impl AzfileCore { continue; } - return Err(parse_error(resp)); + return Err(parse_error(resp).await?); } Ok(()) diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index cf6616c34..0383de8ef 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -20,6 +20,7 @@ use std::fmt::Debug; use std::sync::Arc; use async_trait::async_trait; +use bytes::Buf; use http::StatusCode; use http::Uri; use log::debug; @@ -27,7 +28,7 @@ use reqsign::TencentCosConfig; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; -use super::core::CosCore; +use super::core::*; use super::error::parse_error; use super::lister::CosLister; use super::writer::CosWriter; @@ -373,6 +374,27 @@ impl Accessor for CosBackend { return Ok(RpStat::new(Metadata::new(EntryMode::DIR))); } + if path.ends_with('/') { + let resp = self.core.cos_list_objects(path, "", "", Some(1)).await?; + + if resp.status() != StatusCode::OK { + return Err(parse_error(resp).await?); + } + + let bs = resp.into_body().bytes().await?; + let output: ListObjectsOutput = + quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?; + + return if !output.contents.is_empty() { + Ok(RpStat::new(Metadata::new(EntryMode::DIR))) + } else { + Err( + Error::new(ErrorKind::NotFound, "The directory is not found") + .with_context("path", path), + ) + }; + } + let resp = self.core.cos_head_object(path, &args).await?; let status = resp.status(); @@ -380,9 +402,6 @@ impl Accessor for CosBackend { // The response is very similar to azblob. match status { StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new), - StatusCode::NOT_FOUND if path.ends_with('/') => { - Ok(RpStat::new(Metadata::new(EntryMode::DIR))) - } _ => Err(parse_error(resp).await?), } } diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs index e34f0a460..29659cfc1 100644 --- a/core/src/services/cos/core.rs +++ b/core/src/services/cos/core.rs @@ -482,3 +482,99 @@ pub struct CompleteMultipartUploadRequestPart { #[serde(rename = "ETag")] pub etag: String, } + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListObjectsOutput { + pub name: String, + pub prefix: String, + pub contents: Vec<ListObjectsOutputContent>, + pub common_prefixes: Vec<CommonPrefix>, + pub marker: String, + pub next_marker: Option<String>, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct CommonPrefix { + pub prefix: String, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListObjectsOutputContent { + pub key: String, + pub size: u64, +} + +#[cfg(test)] +mod tests { + use bytes::Buf; + + use super::*; + + #[test] + fn test_parse_xml() { + let bs = bytes::Bytes::from( + r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<ListBucketResult> + <Name>examplebucket</Name> + <Prefix>obj</Prefix> + <Marker>obj002</Marker> + <NextMarker>obj004</NextMarker> + <MaxKeys>1000</MaxKeys> + <IsTruncated>false</IsTruncated> + <Contents> + <Key>obj002</Key> + <LastModified>2015-07-01T02:11:19.775Z</LastModified> + <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> + <Size>9</Size> + <Owner> + <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> + </Owner> + <StorageClass>STANDARD</StorageClass> + </Contents> + <Contents> + <Key>obj003</Key> + <LastModified>2015-07-01T02:11:19.775Z</LastModified> + <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> + <Size>10</Size> + <Owner> + <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> + </Owner> + <StorageClass>STANDARD</StorageClass> + </Contents> + <CommonPrefixes> + <Prefix>hello</Prefix> + </CommonPrefixes> + <CommonPrefixes> + <Prefix>world</Prefix> + </CommonPrefixes> +</ListBucketResult>"#, + ); + let out: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()).expect("must success"); + + assert_eq!(out.name, "examplebucket".to_string()); + assert_eq!(out.prefix, "obj".to_string()); + assert_eq!(out.marker, "obj002".to_string()); + assert_eq!(out.next_marker, Some("obj004".to_string()),); + assert_eq!( + out.contents + .iter() + .map(|v| v.key.clone()) + .collect::<Vec<String>>(), + ["obj002", "obj003"], + ); + assert_eq!( + out.contents.iter().map(|v| v.size).collect::<Vec<u64>>(), + [9, 10], + ); + assert_eq!( + out.common_prefixes + .iter() + .map(|v| v.prefix.clone()) + .collect::<Vec<String>>(), + ["hello", "world"], + ) + } +} diff --git a/core/src/services/cos/lister.rs b/core/src/services/cos/lister.rs index a6a402a0d..830895a46 100644 --- a/core/src/services/cos/lister.rs +++ b/core/src/services/cos/lister.rs @@ -20,14 +20,11 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Buf; use quick_xml::de; -use serde::Deserialize; -use super::core::CosCore; +use super::core::*; use super::error::parse_error; use crate::raw::*; use crate::EntryMode; -use crate::Error; -use crate::ErrorKind; use crate::Metadata; use crate::Result; @@ -64,8 +61,8 @@ impl oio::PageList for CosLister { let bs = resp.into_body().bytes().await?; - let output: Output = de::from_reader(bs.reader()) - .map_err(|e| Error::new(ErrorKind::Unexpected, "deserialize xml").set_source(e))?; + let output: ListObjectsOutput = + de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?; // Try our best to check whether this list is done. // @@ -100,99 +97,3 @@ impl oio::PageList for CosLister { Ok(()) } } - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct Output { - name: String, - prefix: String, - contents: Vec<Content>, - common_prefixes: Vec<CommonPrefix>, - marker: String, - next_marker: Option<String>, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct CommonPrefix { - prefix: String, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct Content { - key: String, - size: u64, -} - -#[cfg(test)] -mod tests { - use bytes::Buf; - - use super::*; - - #[test] - fn test_parse_xml() { - let bs = bytes::Bytes::from( - r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?> -<ListBucketResult> - <Name>examplebucket</Name> - <Prefix>obj</Prefix> - <Marker>obj002</Marker> - <NextMarker>obj004</NextMarker> - <MaxKeys>1000</MaxKeys> - <IsTruncated>false</IsTruncated> - <Contents> - <Key>obj002</Key> - <LastModified>2015-07-01T02:11:19.775Z</LastModified> - <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> - <Size>9</Size> - <Owner> - <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> - </Owner> - <StorageClass>STANDARD</StorageClass> - </Contents> - <Contents> - <Key>obj003</Key> - <LastModified>2015-07-01T02:11:19.775Z</LastModified> - <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> - <Size>10</Size> - <Owner> - <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> - </Owner> - <StorageClass>STANDARD</StorageClass> - </Contents> - <CommonPrefixes> - <Prefix>hello</Prefix> - </CommonPrefixes> - <CommonPrefixes> - <Prefix>world</Prefix> - </CommonPrefixes> -</ListBucketResult>"#, - ); - let out: Output = de::from_reader(bs.reader()).expect("must success"); - - assert_eq!(out.name, "examplebucket".to_string()); - assert_eq!(out.prefix, "obj".to_string()); - assert_eq!(out.marker, "obj002".to_string()); - assert_eq!(out.next_marker, Some("obj004".to_string()),); - assert_eq!( - out.contents - .iter() - .map(|v| v.key.clone()) - .collect::<Vec<String>>(), - ["obj002", "obj003"], - ); - assert_eq!( - out.contents.iter().map(|v| v.size).collect::<Vec<u64>>(), - [9, 10], - ); - assert_eq!( - out.common_prefixes - .iter() - .map(|v| v.prefix.clone()) - .collect::<Vec<String>>(), - ["hello", "world"], - ) - } -} diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 18e0b96be..ac01b5af4 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -1113,7 +1113,7 @@ impl Accessor for S3Backend { } let bs = resp.into_body().bytes().await?; - let output: Output = + let output: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?; return if !output.contents.is_empty() { diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs index 66dbfce72..f20fc9a86 100644 --- a/core/src/services/s3/core.rs +++ b/core/src/services/s3/core.rs @@ -772,16 +772,16 @@ pub struct DeleteObjectsResultError { /// is not exist. #[derive(Default, Debug, Deserialize)] #[serde(default, rename_all = "PascalCase")] -pub struct Output { +pub struct ListObjectsOutput { pub is_truncated: Option<bool>, pub next_continuation_token: Option<String>, pub common_prefixes: Vec<OutputCommonPrefix>, - pub contents: Vec<OutputContent>, + pub contents: Vec<ListObjectsOutputContent>, } #[derive(Default, Debug, Eq, PartialEq, Deserialize)] #[serde(rename_all = "PascalCase")] -pub struct OutputContent { +pub struct ListObjectsOutputContent { pub key: String, pub size: u64, pub last_modified: String, @@ -966,7 +966,7 @@ mod tests { </ListBucketResult>"#, ); - let out: Output = quick_xml::de::from_reader(bs.reader()).expect("must success"); + let out: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()).expect("must success"); assert!(!out.is_truncated.unwrap()); assert!(out.next_continuation_token.is_none()); @@ -980,19 +980,19 @@ mod tests { assert_eq!( out.contents, vec