This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch refactor-lister in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 88a2fa5413c4c0a7f0679230f414a548f3d55522 Author: Xuanwo <[email protected]> AuthorDate: Thu Dec 7 14:11:12 2023 +0800 List must support list without recursive Signed-off-by: Xuanwo <[email protected]> --- bindings/c/include/opendal.h | 4 --- bindings/c/src/operator_info.rs | 3 -- bindings/java/src/lib.rs | 3 +- bindings/nodejs/src/capability.rs | 6 ---- bindings/python/src/capability.rs | 3 -- core/src/layers/complete.rs | 55 ++++++-------------------------- core/src/layers/immutable_index.rs | 1 - core/src/layers/retry.rs | 1 - core/src/raw/oio/list/flat_list.rs | 10 ------ core/src/services/alluxio/backend.rs | 1 - core/src/services/azblob/backend.rs | 1 - core/src/services/azdls/backend.rs | 1 - core/src/services/azfile/backend.rs | 1 - core/src/services/b2/backend.rs | 1 - core/src/services/cos/backend.rs | 1 - core/src/services/dbfs/backend.rs | 1 - core/src/services/fs/backend.rs | 1 - core/src/services/ftp/backend.rs | 1 - core/src/services/gcs/backend.rs | 1 - core/src/services/gdrive/backend.rs | 1 - core/src/services/hdfs/backend.rs | 1 - core/src/services/huggingface/backend.rs | 1 - core/src/services/ipfs/backend.rs | 1 - core/src/services/ipmfs/backend.rs | 1 - core/src/services/obs/backend.rs | 1 - core/src/services/onedrive/backend.rs | 1 - core/src/services/oss/backend.rs | 1 - core/src/services/s3/backend.rs | 1 - core/src/services/sftp/backend.rs | 1 - core/src/services/swift/backend.rs | 1 - core/src/services/webdav/backend.rs | 1 - core/src/services/webhdfs/backend.rs | 1 - core/src/types/capability.rs | 2 -- 33 files changed, 11 insertions(+), 100 deletions(-) diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h index 38b3e531e..36dab301f 100644 --- a/bindings/c/include/opendal.h +++ b/bindings/c/include/opendal.h @@ -635,10 +635,6 @@ typedef struct opendal_capability { * If backend supports list with start after. */ bool list_with_start_after; - /** - * If backend support list with using slash as delimiter. - */ - bool list_without_recursive; /** * If backend supports list without delimiter. */ diff --git a/bindings/c/src/operator_info.rs b/bindings/c/src/operator_info.rs index 4d805e4de..6c7fc69d1 100644 --- a/bindings/c/src/operator_info.rs +++ b/bindings/c/src/operator_info.rs @@ -116,8 +116,6 @@ pub struct opendal_capability { pub list_with_limit: bool, /// If backend supports list with start after. pub list_with_start_after: bool, - /// If backend support list with using slash as delimiter. - pub list_without_recursive: bool, /// If backend supports list without delimiter. pub list_with_recursive: bool, @@ -266,7 +264,6 @@ impl From<core::Capability> for opendal_capability { list_with_limit: value.list_with_limit, list_with_start_after: value.list_with_start_after, list_with_recursive: value.list_with_recursive, - list_without_recursive: value.list_without_recursive, presign: value.presign, presign_read: value.presign_read, presign_stat: value.presign_stat, diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index b3631d14e..d95c0debd 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -153,7 +153,7 @@ fn make_operator_info<'a>(env: &mut JNIEnv<'a>, info: OperatorInfo) -> Result<JO fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result<JObject<'a>> { let capability = env.new_object( "org/apache/opendal/Capability", - "(ZZZZZZZZZZZZZZZZZZJJJZZZZZZZZZZZZZZZJZ)V", + "(ZZZZZZZZZZZZZZZZZZJJJZZZZZZZZZZZZZZJZ)V", &[ JValue::Bool(cap.stat as jboolean), JValue::Bool(cap.stat_with_if_match as jboolean), @@ -183,7 +183,6 @@ fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result<JObject< JValue::Bool(cap.list as jboolean), JValue::Bool(cap.list_with_limit as jboolean), JValue::Bool(cap.list_with_start_after as jboolean), - JValue::Bool(cap.list_without_recursive as jboolean), JValue::Bool(cap.list_with_recursive as jboolean), JValue::Bool(cap.presign as jboolean), JValue::Bool(cap.presign_read as jboolean), diff --git a/bindings/nodejs/src/capability.rs b/bindings/nodejs/src/capability.rs index 8084eb796..742f3c9a6 100644 --- a/bindings/nodejs/src/capability.rs +++ b/bindings/nodejs/src/capability.rs @@ -230,12 +230,6 @@ impl Capability { self.0.list_with_recursive } - /// If backend supports list without recursive. - #[napi(getter)] - pub fn list_without_recursive(&self) -> bool { - self.0.list_without_recursive - } - /// If operator supports presign. #[napi(getter)] pub fn presign(&self) -> bool { diff --git a/bindings/python/src/capability.rs b/bindings/python/src/capability.rs index 533d016fa..723949272 100644 --- a/bindings/python/src/capability.rs +++ b/bindings/python/src/capability.rs @@ -96,8 +96,6 @@ pub struct Capability { pub list_with_limit: bool, /// If backend supports list with start after. pub list_with_start_after: bool, - /// If backend support list with using slash as delimiter. - pub list_without_recursive: bool, /// If backend supports list without delimiter. pub list_with_recursive: bool, @@ -155,7 +153,6 @@ impl Capability { list: capability.list, list_with_limit: capability.list_with_limit, list_with_start_after: capability.list_with_start_after, - list_without_recursive: capability.list_without_recursive, list_with_recursive: capability.list_with_recursive, presign: capability.presign, presign_read: capability.presign_read, diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index a536c6ca3..278a9e827 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -29,7 +29,6 @@ use bytes::Bytes; use crate::raw::oio::FileReader; use crate::raw::oio::FlatLister; -use crate::raw::oio::HierarchyLister; use crate::raw::oio::LazyReader; use crate::raw::oio::RangeReader; use crate::raw::oio::StreamableReader; @@ -366,33 +365,18 @@ impl<A: Accessor> CompleteAccessor<A> { let recursive = args.recursive(); - match ( - recursive, - cap.list_with_recursive, - cap.list_without_recursive, - ) { - // - If service can both list_with_recursive and list_without_recursive - // - If recursive is true while services can list_with_recursive - // - If recursive is false while services can list_without_recursive - (_, true, true) | (true, true, _) | (false, _, true) => { + match (recursive, cap.list_with_recursive) { + // - If service can list_with_recursive + // - If recursive is false + (_, true) | (false, _) => { let (rp, p) = self.inner.list(path, args).await?; Ok((rp, CompleteLister::AlreadyComplete(p))) } - // If services can't list_with_recursive nor list_without_recursive. - // - // It should be a service level bug. - (_, false, false) => Err(self.new_unsupported_error(Operation::List)), // If recursive is true but service can't list_with_recursive - (true, false, true) => { + (true, false) => { let p = FlatLister::new(self.inner.clone(), path); Ok((RpList::default(), CompleteLister::NeedFlat(p))) } - // If recursive is false but service can't list_without_recursive - (false, true, false) => { - let (_, p) = self.inner.list(path, args.with_recursive(true)).await?; - let p = HierarchyLister::new(p, path); - Ok((RpList::default(), CompleteLister::NeedHierarchy(p))) - } } } @@ -408,34 +392,18 @@ impl<A: Accessor> CompleteAccessor<A> { let recursive = args.recursive(); - match ( - recursive, - cap.list_with_recursive, - cap.list_without_recursive, - ) { - // - If service can both list_with_recursive and list_without_recursive - // - If recursive is true while services can list_with_recursive - // - If recursive is false while services can list_without_recursive - (_, true, true) | (true, true, _) | (false, _, true) => { + match (recursive, cap.list_with_recursive) { + // - If service can both list_with_recursive + // - If recursive is false + (_, true) | (false, _) => { let (rp, p) = self.inner.blocking_list(path, args)?; Ok((rp, CompleteLister::AlreadyComplete(p))) } - // If services can't list_with_recursive nor list_without_recursive. - // - // It should be a service level bug. - (_, false, false) => Err(self.new_unsupported_error(Operation::List)), // If recursive is true but service can't list_with_recursive - (true, false, true) => { + (true, false) => { let p = FlatLister::new(self.inner.clone(), path); Ok((RpList::default(), CompleteLister::NeedFlat(p))) } - // If recursive is false but service can't list_without_recursive - (false, true, false) => { - let (_, p) = self.inner.blocking_list(path, args.with_recursive(true))?; - let p: HierarchyLister<<A as Accessor>::BlockingLister> = - HierarchyLister::new(p, path); - Ok((RpList::default(), CompleteLister::NeedHierarchy(p))) - } } } } @@ -738,7 +706,6 @@ where pub enum CompleteLister<A: Accessor, P> { AlreadyComplete(P), NeedFlat(FlatLister<Arc<A>, P>), - NeedHierarchy(HierarchyLister<P>), } #[async_trait] @@ -753,7 +720,6 @@ where match self { AlreadyComplete(p) => p.poll_next(cx), NeedFlat(p) => p.poll_next(cx), - NeedHierarchy(p) => p.poll_next(cx), } } } @@ -769,7 +735,6 @@ where match self { AlreadyComplete(p) => p.next(), NeedFlat(p) => p.next(), - NeedHierarchy(p) => p.next(), } } } diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs index 4a57f4e19..993c7eba7 100644 --- a/core/src/layers/immutable_index.rs +++ b/core/src/layers/immutable_index.rs @@ -156,7 +156,6 @@ impl<A: Accessor> LayeredAccessor for ImmutableIndexAccessor<A> { let cap = meta.full_capability_mut(); cap.list = true; cap.list_with_recursive = true; - cap.list_without_recursive = true; meta } diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index 3db6f793b..8b0a44bb8 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -1173,7 +1173,6 @@ mod tests { am.set_native_capability(Capability { read: true, list: true, - list_without_recursive: true, list_with_recursive: true, batch: true, ..Default::default() diff --git a/core/src/raw/oio/list/flat_list.rs b/core/src/raw/oio/list/flat_list.rs index e6363bf80..9e8e2a5ac 100644 --- a/core/src/raw/oio/list/flat_list.rs +++ b/core/src/raw/oio/list/flat_list.rs @@ -84,15 +84,6 @@ where { /// Create a new flat lister pub fn new(acc: A, path: &str) -> FlatLister<A, L> { - #[cfg(debug_assertions)] - { - let meta = acc.info(); - debug_assert!( - meta.full_capability().list_without_recursive, - "service doesn't support list hierarchy, it must be a bug" - ); - } - FlatLister { acc: Some(acc), root: path.to_string(), @@ -254,7 +245,6 @@ mod tests { fn info(&self) -> AccessorInfo { let mut am = AccessorInfo::default(); am.full_capability_mut().list = true; - am.full_capability_mut().list_without_recursive = true; am } diff --git a/core/src/services/alluxio/backend.rs b/core/src/services/alluxio/backend.rs index 8cbf902d4..078a871da 100644 --- a/core/src/services/alluxio/backend.rs +++ b/core/src/services/alluxio/backend.rs @@ -204,7 +204,6 @@ impl Accessor for AlluxioBackend { delete: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index 7aca06dd2..bf77111e0 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -575,7 +575,6 @@ impl Accessor for AzblobBackend { copy: true, list: true, - list_without_recursive: true, list_with_recursive: true, presign: self.has_sas_token, diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index a494ece63..164a2d7f7 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -255,7 +255,6 @@ impl Accessor for AzdlsBackend { rename: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index ae3d8807b..f8b7f9418 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -270,7 +270,6 @@ impl Accessor for AzfileBackend { rename: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs index b689acb0b..4c6c8660d 100644 --- a/core/src/services/b2/backend.rs +++ b/core/src/services/b2/backend.rs @@ -310,7 +310,6 @@ impl Accessor for B2Backend { list_with_limit: true, list_with_start_after: true, list_with_recursive: true, - list_without_recursive: true, presign: true, presign_read: true, diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 9583a8b5b..1abc9c5a6 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -288,7 +288,6 @@ impl Accessor for CosBackend { copy: true, list: true, - list_without_recursive: true, list_with_recursive: true, presign: true, diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs index 4d85c3aad..6a5fcb602 100644 --- a/core/src/services/dbfs/backend.rs +++ b/core/src/services/dbfs/backend.rs @@ -177,7 +177,6 @@ impl Accessor for DbfsBackend { rename: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index 892a0f79b..e2205e03c 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -265,7 +265,6 @@ impl Accessor for FsBackend { delete: true, list: true, - list_without_recursive: true, copy: true, rename: true, diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index 5b450875f..b0c2a4544 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -306,7 +306,6 @@ impl Accessor for FtpBackend { create_dir: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index 10259d939..8fe1a75c2 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -348,7 +348,6 @@ impl Accessor for GcsBackend { list: true, list_with_limit: true, list_with_start_after: true, - list_without_recursive: true, list_with_recursive: true, batch: true, diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index be8e4b88a..c2e6a874e 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -58,7 +58,6 @@ impl Accessor for GdriveBackend { read: true, list: true, - list_without_recursive: true, write: true, diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs index ea857d4f6..bdff53de9 100644 --- a/core/src/services/hdfs/backend.rs +++ b/core/src/services/hdfs/backend.rs @@ -196,7 +196,6 @@ impl Accessor for HdfsBackend { delete: true, list: true, - list_without_recursive: true, rename: true, blocking: true, diff --git a/core/src/services/huggingface/backend.rs b/core/src/services/huggingface/backend.rs index 37c0b5064..b9d98867e 100644 --- a/core/src/services/huggingface/backend.rs +++ b/core/src/services/huggingface/backend.rs @@ -261,7 +261,6 @@ impl Accessor for HuggingfaceBackend { read_with_range: true, list: true, - list_without_recursive: true, list_with_recursive: true, ..Default::default() diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs index 915e2b798..081a56e73 100644 --- a/core/src/services/ipfs/backend.rs +++ b/core/src/services/ipfs/backend.rs @@ -180,7 +180,6 @@ impl Accessor for IpfsBackend { read_with_range: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/ipmfs/backend.rs b/core/src/services/ipmfs/backend.rs index 7a9da9b4e..0433028f0 100644 --- a/core/src/services/ipmfs/backend.rs +++ b/core/src/services/ipmfs/backend.rs @@ -84,7 +84,6 @@ impl Accessor for IpmfsBackend { delete: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index e753e800d..d72cca73c 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -294,7 +294,6 @@ impl Accessor for ObsBackend { copy: true, list: true, - list_without_recursive: true, list_with_recursive: true, presign: true, diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index ba4e5b8d3..db7ed22c3 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -80,7 +80,6 @@ impl Accessor for OnedriveBackend { delete: true, create_dir: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 15fc23f6b..42a8b9d19 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -425,7 +425,6 @@ impl Accessor for OssBackend { list: true, list_with_limit: true, list_with_start_after: true, - list_without_recursive: true, list_with_recursive: true, presign: true, diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index a6317a6cd..4d9522620 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -1016,7 +1016,6 @@ impl Accessor for S3Backend { list_with_limit: true, list_with_start_after: true, list_with_recursive: true, - list_without_recursive: true, presign: true, presign_stat: true, diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index ec9e2de4f..572bb4dd1 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -267,7 +267,6 @@ impl Accessor for SftpBackend { list: true, list_with_limit: true, - list_without_recursive: true, copy: self.copyable, rename: true, diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs index 4e9e60540..6ae2a3b6a 100644 --- a/core/src/services/swift/backend.rs +++ b/core/src/services/swift/backend.rs @@ -237,7 +237,6 @@ impl Accessor for SwiftBackend { delete: true, list: true, - list_without_recursive: true, list_with_recursive: true, ..Default::default() diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 1361a11fd..b751cf026 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -248,7 +248,6 @@ impl Accessor for WebdavBackend { rename: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs index 9fa9bd150..8a6a9f297 100644 --- a/core/src/services/webhdfs/backend.rs +++ b/core/src/services/webhdfs/backend.rs @@ -417,7 +417,6 @@ impl Accessor for WebhdfsBackend { delete: true, list: true, - list_without_recursive: true, ..Default::default() }); diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index c6f4fc52c..0847afa5e 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -132,8 +132,6 @@ pub struct Capability { pub list_with_start_after: bool, /// If backend supports list with recursive. pub list_with_recursive: bool, - /// If backend supports list without recursive. - pub list_without_recursive: bool, /// If operator supports presign. pub presign: bool,
