This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch fix-stat in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit e746e74440c455ec725185383f47cffa3aa40509 Author: Xuanwo <[email protected]> AuthorDate: Wed Nov 22 19:41:33 2023 +0800 Fix swift Signed-off-by: Xuanwo <[email protected]> --- core/src/services/swift/backend.rs | 38 +++++++++++++--- core/src/services/swift/core.rs | 84 +++++++++++++++++++++++++++++++++- core/src/services/swift/lister.rs | 92 +++++--------------------------------- 3 files changed, 125 insertions(+), 89 deletions(-) diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs index 9fa58b7dd..11e8aa988 100644 --- a/core/src/services/swift/backend.rs +++ b/core/src/services/swift/backend.rs @@ -24,7 +24,7 @@ use async_trait::async_trait; use http::StatusCode; use log::debug; -use super::core::SwiftCore; +use super::core::*; use super::error::parse_error; use super::lister::SwiftLister; use super::writer::SwiftWriter; @@ -299,6 +299,31 @@ impl Accessor for SwiftBackend { } async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> { + // Stat root always returns a DIR. + if path == "/" { + return Ok(RpStat::new(Metadata::new(EntryMode::DIR))); + } + + if path.ends_with('/') { + let resp = self.core.swift_list(path, "", Some(1)).await?; + if resp.status() != StatusCode::OK { + return Err(parse_error(resp).await?); + } + + let bs = resp.into_body().bytes().await?; + let output: Vec<ListOpResponse> = + serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?; + + return if !output.is_empty() { + Ok(RpStat::new(Metadata::new(EntryMode::DIR))) + } else { + Err( + Error::new(ErrorKind::NotFound, "The directory is not found") + .with_context("path", path), + ) + }; + } + let resp = self.core.swift_get_metadata(path).await?; let status = resp.status(); @@ -308,10 +333,6 @@ impl Accessor for SwiftBackend { let meta = parse_into_metadata(path, resp.headers())?; Ok(RpStat::new(meta)) } - // If the path is a container, the server will return a 204 response. - StatusCode::NOT_FOUND if path.ends_with('/') => { - Ok(RpStat::new(Metadata::new(EntryMode::DIR))) - } _ => Err(parse_error(resp).await?), } } @@ -329,7 +350,12 @@ impl Accessor for SwiftBackend { } async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> { - let l = SwiftLister::new(self.core.clone(), path.to_string(), args.recursive()); + let l = SwiftLister::new( + self.core.clone(), + path.to_string(), + args.recursive(), + args.limit(), + ); Ok((RpList::default(), oio::PageLister::new(l))) } diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs index 3412bbd8e..8fbbf9b43 100644 --- a/core/src/services/swift/core.rs +++ b/core/src/services/swift/core.rs @@ -19,6 +19,7 @@ use std::fmt::Debug; use http::Request; use http::Response; +use serde::Deserialize; use crate::raw::*; use crate::*; @@ -70,12 +71,13 @@ impl SwiftCore { &self, path: &str, delimiter: &str, + limit: Option<usize>, ) -> Result<Response<IncomingAsyncBody>> { let p = build_abs_path(&self.root, path); // The delimiter is used to disable recursive listing. // Swift returns a 200 status code when there is no such pseudo directory in prefix. - let url = format!( + let mut url = format!( "{}/v1/{}/{}/?prefix={}&delimiter={}&format=json", self.endpoint, self.account, @@ -84,6 +86,10 @@ impl SwiftCore { delimiter ); + if let Some(limit) = limit { + url += &format!("&limit={}", limit); + } + let mut req = Request::get(&url); req = req.header("X-Auth-Token", &self.token); @@ -217,3 +223,79 @@ impl SwiftCore { self.client.send(req).await } } + +#[derive(Debug, Eq, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum ListOpResponse { + Subdir { + subdir: String, + }, + FileInfo { + bytes: u64, + hash: String, + name: String, + content_type: String, + last_modified: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_list_response_test() -> Result<()> { + let resp = bytes::Bytes::from( + r#" + [ + { + "subdir": "animals/" + }, + { + "subdir": "fruit/" + }, + { + "bytes": 147, + "hash": "5e6b5b70b0426b1cc1968003e1afa5ad", + "name": "test.txt", + "content_type": "text/plain", + "last_modified": "2023-11-01T03:00:23.147480" + } + ] + "#, + ); + + let mut out = serde_json::from_slice::<Vec<ListOpResponse>>(&resp) + .map_err(new_json_deserialize_error)?; + + assert_eq!(out.len(), 3); + assert_eq!( + out.pop().unwrap(), + ListOpResponse::FileInfo { + bytes: 147, + hash: "5e6b5b70b0426b1cc1968003e1afa5ad".to_string(), + name: "test.txt".to_string(), + content_type: + "multipart/form-data;boundary=------------------------25004a866ee9c0cb" + .to_string(), + last_modified: "2023-11-01T03:00:23.147480".to_string(), + } + ); + + assert_eq!( + out.pop().unwrap(), + ListOpResponse::Subdir { + subdir: "fruit/".to_string() + } + ); + + assert_eq!( + out.pop().unwrap(), + ListOpResponse::Subdir { + subdir: "animals/".to_string() + } + ); + + Ok(()) + } +} diff --git a/core/src/services/swift/lister.rs b/core/src/services/swift/lister.rs index 6291c866d..077fa3292 100644 --- a/core/src/services/swift/lister.rs +++ b/core/src/services/swift/lister.rs @@ -18,9 +18,8 @@ use std::sync::Arc; use async_trait::async_trait; -use serde::Deserialize; -use super::core::SwiftCore; +use super::core::*; use super::error::parse_error; use crate::raw::*; use crate::*; @@ -29,15 +28,17 @@ pub struct SwiftLister { core: Arc<SwiftCore>, path: String, delimiter: &'static str, + limit: Option<usize>, } impl SwiftLister { - pub fn new(core: Arc<SwiftCore>, path: String, recursive: bool) -> Self { + pub fn new(core: Arc<SwiftCore>, path: String, recursive: bool, limit: Option<usize>) -> Self { let delimiter = if recursive { "" } else { "/" }; Self { core, path, delimiter, + limit, } } } @@ -45,7 +46,10 @@ impl SwiftLister { #[async_trait] impl oio::PageList for SwiftLister { async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> { - let response = self.core.swift_list(&self.path, self.delimiter).await?; + let response = self + .core + .swift_list(&self.path, self.delimiter, self.limit) + .await?; let status_code = response.status(); @@ -57,8 +61,8 @@ impl oio::PageList for SwiftLister { ctx.done = true; let bytes = response.into_body().bytes().await?; - let decoded_response = serde_json::from_slice::<Vec<ListOpResponse>>(&bytes) - .map_err(new_json_deserialize_error)?; + let decoded_response: Vec<ListOpResponse> = + serde_json::from_slice(&bytes).map_err(new_json_deserialize_error)?; for status in decoded_response { let entry: oio::Entry = match status { @@ -96,79 +100,3 @@ impl oio::PageList for SwiftLister { Ok(()) } } - -#[derive(Debug, Eq, PartialEq, Deserialize)] -#[serde(untagged)] -pub(super) enum ListOpResponse { - Subdir { - subdir: String, - }, - FileInfo { - bytes: u64, - hash: String, - name: String, - content_type: String, - last_modified: String, - }, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_list_response_test() -> Result<()> { - let resp = bytes::Bytes::from( - r#" - [ - { - "subdir": "animals/" - }, - { - "subdir": "fruit/" - }, - { - "bytes": 147, - "hash": "5e6b5b70b0426b1cc1968003e1afa5ad", - "name": "test.txt", - "content_type": "text/plain", - "last_modified": "2023-11-01T03:00:23.147480" - } - ] - "#, - ); - - let mut out = serde_json::from_slice::<Vec<ListOpResponse>>(&resp) - .map_err(new_json_deserialize_error)?; - - assert_eq!(out.len(), 3); - assert_eq!( - out.pop().unwrap(), - ListOpResponse::FileInfo { - bytes: 147, - hash: "5e6b5b70b0426b1cc1968003e1afa5ad".to_string(), - name: "test.txt".to_string(), - content_type: - "multipart/form-data;boundary=------------------------25004a866ee9c0cb" - .to_string(), - last_modified: "2023-11-01T03:00:23.147480".to_string(), - } - ); - - assert_eq!( - out.pop().unwrap(), - ListOpResponse::Subdir { - subdir: "fruit/".to_string() - } - ); - - assert_eq!( - out.pop().unwrap(), - ListOpResponse::Subdir { - subdir: "animals/".to_string() - } - ); - - Ok(()) - } -}
