This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch fix-stat in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit cfdd0ea5a4690ce3ab44d7a622b794c6c047c916 Author: Xuanwo <[email protected]> AuthorDate: Wed Nov 22 19:23:00 2023 +0800 Fix obs Signed-off-by: Xuanwo <[email protected]> --- core/src/services/obs/backend.rs | 24 ++++++++- core/src/services/obs/core.rs | 96 +++++++++++++++++++++++++++++++++++ core/src/services/obs/lister.rs | 105 ++------------------------------------- 3 files changed, 122 insertions(+), 103 deletions(-) diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 4f84852b5..2449913a6 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -20,6 +20,7 @@ use std::fmt::Debug; use std::sync::Arc; use async_trait::async_trait; +use bytes::Buf; use http::StatusCode; use http::Uri; use log::debug; @@ -27,7 +28,7 @@ use reqsign::HuaweicloudObsConfig; use reqsign::HuaweicloudObsCredentialLoader; use reqsign::HuaweicloudObsSigner; -use super::core::ObsCore; +use super::core::{ListObjectsOutput, ObsCore}; use super::error::parse_error; use super::lister::ObsLister; use super::writer::ObsWriter; @@ -400,6 +401,27 @@ impl Accessor for ObsBackend { return Ok(RpStat::new(Metadata::new(EntryMode::DIR))); } + if path.ends_with('/') { + let resp = self.core.obs_list_objects(path, "", "", Some(1)).await?; + + if resp.status() != StatusCode::OK { + return Err(parse_error(resp).await?); + } + + let bs = resp.into_body().bytes().await?; + let output: ListObjectsOutput = + quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?; + + return if !output.contents.is_empty() { + Ok(RpStat::new(Metadata::new(EntryMode::DIR))) + } else { + Err( + Error::new(ErrorKind::NotFound, "The directory is not found") + .with_context("path", path), + ) + }; + } + let resp = self.core.obs_head_object(path, &args).await?; let status = resp.status(); diff --git a/core/src/services/obs/core.rs b/core/src/services/obs/core.rs index de9d2a735..344f10456 100644 --- a/core/src/services/obs/core.rs +++ b/core/src/services/obs/core.rs @@ -476,3 +476,99 @@ pub struct CompleteMultipartUploadRequestPart { #[serde(rename = "ETag")] pub etag: String, } + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListObjectsOutput { + pub name: String, + pub prefix: String, + pub contents: Vec<ListObjectsOutputContent>, + pub common_prefixes: Vec<CommonPrefix>, + pub marker: String, + pub next_marker: Option<String>, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct CommonPrefix { + pub prefix: String, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListObjectsOutputContent { + pub key: String, + pub size: u64, +} + +#[cfg(test)] +mod tests { + use bytes::Buf; + + use super::*; + + #[test] + fn test_parse_xml() { + let bs = bytes::Bytes::from( + r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<ListBucketResult xmlns="http://obs.cn-north-4.myhuaweicloud.com/doc/2015-06-30/";> + <Name>examplebucket</Name> + <Prefix>obj</Prefix> + <Marker>obj002</Marker> + <NextMarker>obj004</NextMarker> + <MaxKeys>1000</MaxKeys> + <IsTruncated>false</IsTruncated> + <Contents> + <Key>obj002</Key> + <LastModified>2015-07-01T02:11:19.775Z</LastModified> + <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> + <Size>9</Size> + <Owner> + <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> + </Owner> + <StorageClass>STANDARD</StorageClass> + </Contents> + <Contents> + <Key>obj003</Key> + <LastModified>2015-07-01T02:11:19.775Z</LastModified> + <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> + <Size>10</Size> + <Owner> + <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> + </Owner> + <StorageClass>STANDARD</StorageClass> + </Contents> + <CommonPrefixes> + <Prefix>hello</Prefix> + </CommonPrefixes> + <CommonPrefixes> + <Prefix>world</Prefix> + </CommonPrefixes> +</ListBucketResult>"#, + ); + let out: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()).expect("must success"); + + assert_eq!(out.name, "examplebucket".to_string()); + assert_eq!(out.prefix, "obj".to_string()); + assert_eq!(out.marker, "obj002".to_string()); + assert_eq!(out.next_marker, Some("obj004".to_string()),); + assert_eq!( + out.contents + .iter() + .map(|v| v.key.clone()) + .collect::<Vec<String>>(), + ["obj002", "obj003"], + ); + assert_eq!( + out.contents.iter().map(|v| v.size).collect::<Vec<u64>>(), + [9, 10], + ); + assert_eq!( + out.common_prefixes + .iter() + .map(|v| v.prefix.clone()) + .collect::<Vec<String>>(), + ["hello", "world"], + ) + } +} diff --git a/core/src/services/obs/lister.rs b/core/src/services/obs/lister.rs index 831427ab1..7aa6c7631 100644 --- a/core/src/services/obs/lister.rs +++ b/core/src/services/obs/lister.rs @@ -20,14 +20,11 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Buf; use quick_xml::de; -use serde::Deserialize; -use super::core::ObsCore; +use super::core::*; use super::error::parse_error; use crate::raw::*; use crate::EntryMode; -use crate::Error; -use crate::ErrorKind; use crate::Metadata; use crate::Result; @@ -65,8 +62,8 @@ impl oio::PageList for ObsLister { let bs = resp.into_body().bytes().await?; - let output: Output = de::from_reader(bs.reader()) - .map_err(|e| Error::new(ErrorKind::Unexpected, "deserialize xml").set_source(e))?; + let output: ListObjectsOutput = + de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?; // Try our best to check whether this list is done. // @@ -103,99 +100,3 @@ impl oio::PageList for ObsLister { Ok(()) } } - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct Output { - name: String, - prefix: String, - contents: Vec<Content>, - common_prefixes: Vec<CommonPrefix>, - marker: String, - next_marker: Option<String>, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct CommonPrefix { - prefix: String, -} - -#[derive(Default, Debug, Deserialize)] -#[serde(default, rename_all = "PascalCase")] -struct Content { - key: String, - size: u64, -} - -#[cfg(test)] -mod tests { - use bytes::Buf; - - use super::*; - - #[test] - fn test_parse_xml() { - let bs = bytes::Bytes::from( - r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?> -<ListBucketResult xmlns="http://obs.cn-north-4.myhuaweicloud.com/doc/2015-06-30/";> - <Name>examplebucket</Name> - <Prefix>obj</Prefix> - <Marker>obj002</Marker> - <NextMarker>obj004</NextMarker> - <MaxKeys>1000</MaxKeys> - <IsTruncated>false</IsTruncated> - <Contents> - <Key>obj002</Key> - <LastModified>2015-07-01T02:11:19.775Z</LastModified> - <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> - <Size>9</Size> - <Owner> - <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> - </Owner> - <StorageClass>STANDARD</StorageClass> - </Contents> - <Contents> - <Key>obj003</Key> - <LastModified>2015-07-01T02:11:19.775Z</LastModified> - <ETag>"a72e382246ac83e86bd203389849e71d"</ETag> - <Size>10</Size> - <Owner> - <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID> - </Owner> - <StorageClass>STANDARD</StorageClass> - </Contents> - <CommonPrefixes> - <Prefix>hello</Prefix> - </CommonPrefixes> - <CommonPrefixes> - <Prefix>world</Prefix> - </CommonPrefixes> -</ListBucketResult>"#, - ); - let out: Output = de::from_reader(bs.reader()).expect("must success"); - - assert_eq!(out.name, "examplebucket".to_string()); - assert_eq!(out.prefix, "obj".to_string()); - assert_eq!(out.marker, "obj002".to_string()); - assert_eq!(out.next_marker, Some("obj004".to_string()),); - assert_eq!( - out.contents - .iter() - .map(|v| v.key.clone()) - .collect::<Vec<String>>(), - ["obj002", "obj003"], - ); - assert_eq!( - out.contents.iter().map(|v| v.size).collect::<Vec<u64>>(), - [9, 10], - ); - assert_eq!( - out.common_prefixes - .iter() - .map(|v| v.prefix.clone()) - .collect::<Vec<String>>(), - ["hello", "world"], - ) - } -}
