This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 0920ac379 refactor: List must support list without recursive (#3721)
0920ac379 is described below
commit 0920ac37909eeaf5d18e15c98df860bf5ff47c15
Author: Xuanwo <[email protected]>
AuthorDate: Thu Dec 7 16:44:30 2023 +0800
refactor: List must support list without recursive (#3721)
* List must support list without recursive
Signed-off-by: Xuanwo <[email protected]>
* Add list nest dir test
Signed-off-by: Xuanwo <[email protected]>
* Remove non-native feature
Signed-off-by: Xuanwo <[email protected]>
* Remove non-native feature
Signed-off-by: Xuanwo <[email protected]>
* Fix kv
Signed-off-by: Xuanwo <[email protected]>
* Fix java
Signed-off-by: Xuanwo <[email protected]>
* Fix list
Signed-off-by: Xuanwo <[email protected]>
* Fix lister
Signed-off-by: Xuanwo <[email protected]>
* Fix binding c
Signed-off-by: Xuanwo <[email protected]>
* Fix nodejs
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
bindings/c/include/opendal.h | 4 --
bindings/c/src/operator_info.rs | 3 -
bindings/c/tests/bdd.cpp | 20 ------
bindings/c/tests/opinfo.cpp | 4 --
bindings/java/src/lib.rs | 3 +-
.../main/java/org/apache/opendal/Capability.java | 15 ++--
bindings/nodejs/generated.d.ts | 2 -
bindings/nodejs/src/capability.rs | 6 --
bindings/python/src/capability.rs | 3 -
core/src/layers/complete.rs | 55 +++------------
core/src/layers/immutable_index.rs | 1 -
core/src/layers/retry.rs | 1 -
core/src/raw/adapters/kv/backend.rs | 81 ++--------------------
core/src/raw/adapters/typed_kv/backend.rs | 78 ++-------------------
core/src/raw/oio/list/flat_list.rs | 10 ---
core/src/raw/oio/list/hierarchy_list.rs | 18 ++++-
core/src/services/alluxio/backend.rs | 1 -
core/src/services/azblob/backend.rs | 1 -
core/src/services/azdls/backend.rs | 1 -
core/src/services/azfile/backend.rs | 1 -
core/src/services/b2/backend.rs | 1 -
core/src/services/cos/backend.rs | 1 -
core/src/services/dashmap/backend.rs | 12 +++-
core/src/services/dbfs/backend.rs | 1 -
core/src/services/fs/backend.rs | 1 -
core/src/services/ftp/backend.rs | 1 -
core/src/services/gcs/backend.rs | 1 -
core/src/services/gdrive/backend.rs | 1 -
core/src/services/hdfs/backend.rs | 1 -
core/src/services/huggingface/backend.rs | 1 -
core/src/services/ipfs/backend.rs | 1 -
core/src/services/ipmfs/backend.rs | 1 -
core/src/services/obs/backend.rs | 1 -
core/src/services/onedrive/backend.rs | 1 -
core/src/services/oss/backend.rs | 1 -
core/src/services/s3/backend.rs | 1 -
core/src/services/sftp/backend.rs | 1 -
core/src/services/swift/backend.rs | 1 -
core/src/services/webdav/backend.rs | 1 -
core/src/services/webhdfs/backend.rs | 1 -
core/src/types/capability.rs | 2 -
core/tests/behavior/list.rs | 8 ++-
42 files changed, 62 insertions(+), 287 deletions(-)
diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h
index 38b3e531e..36dab301f 100644
--- a/bindings/c/include/opendal.h
+++ b/bindings/c/include/opendal.h
@@ -635,10 +635,6 @@ typedef struct opendal_capability {
* If backend supports list with start after.
*/
bool list_with_start_after;
- /**
- * If backend support list with using slash as delimiter.
- */
- bool list_without_recursive;
/**
* If backend supports list without delimiter.
*/
diff --git a/bindings/c/src/operator_info.rs b/bindings/c/src/operator_info.rs
index 4d805e4de..6c7fc69d1 100644
--- a/bindings/c/src/operator_info.rs
+++ b/bindings/c/src/operator_info.rs
@@ -116,8 +116,6 @@ pub struct opendal_capability {
pub list_with_limit: bool,
/// If backend supports list with start after.
pub list_with_start_after: bool,
- /// If backend support list with using slash as delimiter.
- pub list_without_recursive: bool,
/// If backend supports list without delimiter.
pub list_with_recursive: bool,
@@ -266,7 +264,6 @@ impl From<core::Capability> for opendal_capability {
list_with_limit: value.list_with_limit,
list_with_start_after: value.list_with_start_after,
list_with_recursive: value.list_with_recursive,
- list_without_recursive: value.list_without_recursive,
presign: value.presign,
presign_read: value.presign_read,
presign_stat: value.presign_stat,
diff --git a/bindings/c/tests/bdd.cpp b/bindings/c/tests/bdd.cpp
index 67b8f56b8..370c17fd5 100644
--- a/bindings/c/tests/bdd.cpp
+++ b/bindings/c/tests/bdd.cpp
@@ -75,26 +75,6 @@ TEST_F(OpendalBddTest, FeatureTest)
opendal_metadata* meta = s.meta;
EXPECT_TRUE(opendal_metadata_is_file(meta));
- // The blocking file "test" should be renamed to "test-copy"
- error = opendal_operator_rename(this->p, this->path.c_str(), "test-copy");
- EXPECT_EQ(error, nullptr);
- e = opendal_operator_is_exist(this->p, "test-copy");
- EXPECT_EQ(e.error, nullptr);
- EXPECT_TRUE(e.is_exist);
- e = opendal_operator_is_exist(this->p, this->path.c_str());
- EXPECT_EQ(e.error, nullptr);
- EXPECT_FALSE(e.is_exist);
-
- // The blocking file "test-copy" should be copied to "test"
- error = opendal_operator_copy(this->p, "test-copy", this->path.c_str());
- EXPECT_EQ(error, nullptr);
- e = opendal_operator_is_exist(this->p, this->path.c_str());
- EXPECT_EQ(e.error, nullptr);
- EXPECT_TRUE(e.is_exist);
- e = opendal_operator_is_exist(this->p, "test-copy");
- EXPECT_EQ(e.error, nullptr);
- EXPECT_TRUE(e.is_exist);
-
// The blocking file "test" content length must be 13
EXPECT_EQ(opendal_metadata_content_length(meta), 13);
diff --git a/bindings/c/tests/opinfo.cpp b/bindings/c/tests/opinfo.cpp
index 480364017..42c70fde2 100644
--- a/bindings/c/tests/opinfo.cpp
+++ b/bindings/c/tests/opinfo.cpp
@@ -75,8 +75,6 @@ TEST_F(OpendalOperatorInfoTest, CapabilityTest)
EXPECT_TRUE(full_cap.delete_);
EXPECT_TRUE(full_cap.list);
EXPECT_TRUE(full_cap.list_with_recursive);
- EXPECT_TRUE(full_cap.copy);
- EXPECT_TRUE(full_cap.rename);
EXPECT_TRUE(native_cap.blocking);
EXPECT_TRUE(native_cap.read);
@@ -90,8 +88,6 @@ TEST_F(OpendalOperatorInfoTest, CapabilityTest)
EXPECT_TRUE(native_cap.delete_);
EXPECT_TRUE(native_cap.list);
EXPECT_TRUE(native_cap.list_with_recursive);
- EXPECT_TRUE(native_cap.copy);
- EXPECT_TRUE(native_cap.rename);
}
TEST_F(OpendalOperatorInfoTest, InfoTest)
diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs
index b3631d14e..d95c0debd 100644
--- a/bindings/java/src/lib.rs
+++ b/bindings/java/src/lib.rs
@@ -153,7 +153,7 @@ fn make_operator_info<'a>(env: &mut JNIEnv<'a>, info:
OperatorInfo) -> Result<JO
fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) ->
Result<JObject<'a>> {
let capability = env.new_object(
"org/apache/opendal/Capability",
- "(ZZZZZZZZZZZZZZZZZZJJJZZZZZZZZZZZZZZZJZ)V",
+ "(ZZZZZZZZZZZZZZZZZZJJJZZZZZZZZZZZZZZJZ)V",
&[
JValue::Bool(cap.stat as jboolean),
JValue::Bool(cap.stat_with_if_match as jboolean),
@@ -183,7 +183,6 @@ fn make_capability<'a>(env: &mut JNIEnv<'a>, cap:
Capability) -> Result<JObject<
JValue::Bool(cap.list as jboolean),
JValue::Bool(cap.list_with_limit as jboolean),
JValue::Bool(cap.list_with_start_after as jboolean),
- JValue::Bool(cap.list_without_recursive as jboolean),
JValue::Bool(cap.list_with_recursive as jboolean),
JValue::Bool(cap.presign as jboolean),
JValue::Bool(cap.presign_read as jboolean),
diff --git a/bindings/java/src/main/java/org/apache/opendal/Capability.java
b/bindings/java/src/main/java/org/apache/opendal/Capability.java
index dfde61276..45a76e2b7 100644
--- a/bindings/java/src/main/java/org/apache/opendal/Capability.java
+++ b/bindings/java/src/main/java/org/apache/opendal/Capability.java
@@ -167,14 +167,9 @@ public class Capability {
public final boolean listWithStartAfter;
/**
- * If backend support list with using slash as delimiter.
+ * If backend support list with recursive.
*/
- public final boolean listWithDelimiterSlash;
-
- /**
- * If backend supports list without delimiter.
- */
- public final boolean listWithoutDelimiter;
+ public final boolean listWithRecursive;
/**
* If operator supports presign.
@@ -245,8 +240,7 @@ public class Capability {
boolean list,
boolean listWithLimit,
boolean listWithStartAfter,
- boolean listWithDelimiterSlash,
- boolean listWithoutDelimiter,
+ boolean listWithRecursive,
boolean presign,
boolean presignRead,
boolean presignStat,
@@ -283,8 +277,7 @@ public class Capability {
this.list = list;
this.listWithLimit = listWithLimit;
this.listWithStartAfter = listWithStartAfter;
- this.listWithDelimiterSlash = listWithDelimiterSlash;
- this.listWithoutDelimiter = listWithoutDelimiter;
+ this.listWithRecursive = listWithRecursive;
this.presign = presign;
this.presignRead = presignRead;
this.presignStat = presignStat;
diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts
index c5e1d1ba8..9f45b248d 100644
--- a/bindings/nodejs/generated.d.ts
+++ b/bindings/nodejs/generated.d.ts
@@ -128,8 +128,6 @@ export class Capability {
get listWithStartAfter(): boolean
/** If backend supports list with recursive. */
get listWithRecursive(): boolean
- /** If backend supports list without recursive. */
- get listWithoutRecursive(): boolean
/** If operator supports presign. */
get presign(): boolean
/** If operator supports presign read. */
diff --git a/bindings/nodejs/src/capability.rs
b/bindings/nodejs/src/capability.rs
index 8084eb796..742f3c9a6 100644
--- a/bindings/nodejs/src/capability.rs
+++ b/bindings/nodejs/src/capability.rs
@@ -230,12 +230,6 @@ impl Capability {
self.0.list_with_recursive
}
- /// If backend supports list without recursive.
- #[napi(getter)]
- pub fn list_without_recursive(&self) -> bool {
- self.0.list_without_recursive
- }
-
/// If operator supports presign.
#[napi(getter)]
pub fn presign(&self) -> bool {
diff --git a/bindings/python/src/capability.rs
b/bindings/python/src/capability.rs
index 533d016fa..723949272 100644
--- a/bindings/python/src/capability.rs
+++ b/bindings/python/src/capability.rs
@@ -96,8 +96,6 @@ pub struct Capability {
pub list_with_limit: bool,
/// If backend supports list with start after.
pub list_with_start_after: bool,
- /// If backend support list with using slash as delimiter.
- pub list_without_recursive: bool,
/// If backend supports list without delimiter.
pub list_with_recursive: bool,
@@ -155,7 +153,6 @@ impl Capability {
list: capability.list,
list_with_limit: capability.list_with_limit,
list_with_start_after: capability.list_with_start_after,
- list_without_recursive: capability.list_without_recursive,
list_with_recursive: capability.list_with_recursive,
presign: capability.presign,
presign_read: capability.presign_read,
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index a536c6ca3..278a9e827 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -29,7 +29,6 @@ use bytes::Bytes;
use crate::raw::oio::FileReader;
use crate::raw::oio::FlatLister;
-use crate::raw::oio::HierarchyLister;
use crate::raw::oio::LazyReader;
use crate::raw::oio::RangeReader;
use crate::raw::oio::StreamableReader;
@@ -366,33 +365,18 @@ impl<A: Accessor> CompleteAccessor<A> {
let recursive = args.recursive();
- match (
- recursive,
- cap.list_with_recursive,
- cap.list_without_recursive,
- ) {
- // - If service can both list_with_recursive and
list_without_recursive
- // - If recursive is true while services can list_with_recursive
- // - If recursive is false while services can
list_without_recursive
- (_, true, true) | (true, true, _) | (false, _, true) => {
+ match (recursive, cap.list_with_recursive) {
+ // - If service can list_with_recursive
+ // - If recursive is false
+ (_, true) | (false, _) => {
let (rp, p) = self.inner.list(path, args).await?;
Ok((rp, CompleteLister::AlreadyComplete(p)))
}
- // If services can't list_with_recursive nor
list_without_recursive.
- //
- // It should be a service level bug.
- (_, false, false) =>
Err(self.new_unsupported_error(Operation::List)),
// If recursive is true but service can't list_with_recursive
- (true, false, true) => {
+ (true, false) => {
let p = FlatLister::new(self.inner.clone(), path);
Ok((RpList::default(), CompleteLister::NeedFlat(p)))
}
- // If recursive is false but service can't list_without_recursive
- (false, true, false) => {
- let (_, p) = self.inner.list(path,
args.with_recursive(true)).await?;
- let p = HierarchyLister::new(p, path);
- Ok((RpList::default(), CompleteLister::NeedHierarchy(p)))
- }
}
}
@@ -408,34 +392,18 @@ impl<A: Accessor> CompleteAccessor<A> {
let recursive = args.recursive();
- match (
- recursive,
- cap.list_with_recursive,
- cap.list_without_recursive,
- ) {
- // - If service can both list_with_recursive and
list_without_recursive
- // - If recursive is true while services can list_with_recursive
- // - If recursive is false while services can
list_without_recursive
- (_, true, true) | (true, true, _) | (false, _, true) => {
+ match (recursive, cap.list_with_recursive) {
+ // - If service can both list_with_recursive
+ // - If recursive is false
+ (_, true) | (false, _) => {
let (rp, p) = self.inner.blocking_list(path, args)?;
Ok((rp, CompleteLister::AlreadyComplete(p)))
}
- // If services can't list_with_recursive nor
list_without_recursive.
- //
- // It should be a service level bug.
- (_, false, false) =>
Err(self.new_unsupported_error(Operation::List)),
// If recursive is true but service can't list_with_recursive
- (true, false, true) => {
+ (true, false) => {
let p = FlatLister::new(self.inner.clone(), path);
Ok((RpList::default(), CompleteLister::NeedFlat(p)))
}
- // If recursive is false but service can't list_without_recursive
- (false, true, false) => {
- let (_, p) = self.inner.blocking_list(path,
args.with_recursive(true))?;
- let p: HierarchyLister<<A as Accessor>::BlockingLister> =
- HierarchyLister::new(p, path);
- Ok((RpList::default(), CompleteLister::NeedHierarchy(p)))
- }
}
}
}
@@ -738,7 +706,6 @@ where
pub enum CompleteLister<A: Accessor, P> {
AlreadyComplete(P),
NeedFlat(FlatLister<Arc<A>, P>),
- NeedHierarchy(HierarchyLister<P>),
}
#[async_trait]
@@ -753,7 +720,6 @@ where
match self {
AlreadyComplete(p) => p.poll_next(cx),
NeedFlat(p) => p.poll_next(cx),
- NeedHierarchy(p) => p.poll_next(cx),
}
}
}
@@ -769,7 +735,6 @@ where
match self {
AlreadyComplete(p) => p.next(),
NeedFlat(p) => p.next(),
- NeedHierarchy(p) => p.next(),
}
}
}
diff --git a/core/src/layers/immutable_index.rs
b/core/src/layers/immutable_index.rs
index 4a57f4e19..993c7eba7 100644
--- a/core/src/layers/immutable_index.rs
+++ b/core/src/layers/immutable_index.rs
@@ -156,7 +156,6 @@ impl<A: Accessor> LayeredAccessor for
ImmutableIndexAccessor<A> {
let cap = meta.full_capability_mut();
cap.list = true;
cap.list_with_recursive = true;
- cap.list_without_recursive = true;
meta
}
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 3db6f793b..8b0a44bb8 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -1173,7 +1173,6 @@ mod tests {
am.set_native_capability(Capability {
read: true,
list: true,
- list_without_recursive: true,
list_with_recursive: true,
batch: true,
..Default::default()
diff --git a/core/src/raw/adapters/kv/backend.rs
b/core/src/raw/adapters/kv/backend.rs
index c4e68d8c7..a05b1da67 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -28,6 +28,7 @@ use futures::future::BoxFuture;
use futures::FutureExt;
use super::Adapter;
+use crate::raw::oio::HierarchyLister;
use crate::raw::*;
use crate::*;
@@ -69,8 +70,8 @@ impl<S: Adapter> Accessor for Backend<S> {
type BlockingReader = oio::Cursor;
type Writer = KvWriter<S>;
type BlockingWriter = KvWriter<S>;
- type Lister = KvLister;
- type BlockingLister = KvLister;
+ type Lister = HierarchyLister<KvLister>;
+ type BlockingLister = HierarchyLister<KvLister>;
fn info(&self) -> AccessorInfo {
let mut am: AccessorInfo = self.kv.metadata().into();
@@ -89,14 +90,6 @@ impl<S: Adapter> Accessor for Backend<S> {
cap.delete = true;
}
- if cap.read && cap.write {
- cap.copy = true;
- }
-
- if cap.read && cap.write && cap.delete {
- cap.rename = true;
- }
-
if cap.list {
cap.list_with_recursive = true;
}
@@ -189,83 +182,23 @@ impl<S: Adapter> Accessor for Backend<S> {
Ok(RpDelete::default())
}
- async fn list(&self, path: &str, _: OpList) -> Result<(RpList,
Self::Lister)> {
+ async fn list(&self, path: &str, args: OpList) -> Result<(RpList,
Self::Lister)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.scan(&p).await?;
let lister = KvLister::new(&self.root, res);
+ let lister = HierarchyLister::new(lister, path, args.recursive());
Ok((RpList::default(), lister))
}
- fn blocking_list(&self, path: &str, _: OpList) -> Result<(RpList,
Self::BlockingLister)> {
+ fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList,
Self::BlockingLister)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.blocking_scan(&p)?;
let lister = KvLister::new(&self.root, res);
+ let lister = HierarchyLister::new(lister, path, args.recursive());
Ok((RpList::default(), lister))
}
-
- async fn rename(&self, from: &str, to: &str, _: OpRename) ->
Result<RpRename> {
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.get(&from).await? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.set(&to, &bs).await?;
-
- self.kv.delete(&from).await?;
- Ok(RpRename::default())
- }
-
- fn blocking_rename(&self, from: &str, to: &str, _: OpRename) ->
Result<RpRename> {
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.blocking_get(&from)? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.blocking_set(&to, &bs)?;
-
- self.kv.blocking_delete(&from)?;
- Ok(RpRename::default())
- }
-
- async fn copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy> {
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.get(&from).await? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.set(&to, &bs).await?;
-
- Ok(RpCopy::default())
- }
-
- fn blocking_copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy>
{
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.blocking_get(&from)? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.blocking_set(&to, &bs)?;
-
- Ok(RpCopy::default())
- }
}
impl<S> Backend<S>
diff --git a/core/src/raw/adapters/typed_kv/backend.rs
b/core/src/raw/adapters/typed_kv/backend.rs
index 266d41724..b02c99ec5 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -28,6 +28,7 @@ use futures::FutureExt;
use super::Adapter;
use super::Value;
+use crate::raw::oio::HierarchyLister;
use crate::raw::*;
use crate::*;
@@ -63,8 +64,8 @@ impl<S: Adapter> Accessor for Backend<S> {
type BlockingReader = oio::Cursor;
type Writer = KvWriter<S>;
type BlockingWriter = KvWriter<S>;
- type Lister = KvLister;
- type BlockingLister = KvLister;
+ type Lister = HierarchyLister<KvLister>;
+ type BlockingLister = HierarchyLister<KvLister>;
fn info(&self) -> AccessorInfo {
let kv_info = self.kv.info();
@@ -97,13 +98,6 @@ impl<S: Adapter> Accessor for Backend<S> {
cap.list_with_recursive = true;
}
- if cap.read && cap.write {
- cap.copy = true;
- }
-
- if cap.read && cap.write && cap.delete {
- cap.rename = true;
- }
cap.blocking = true;
am.set_native_capability(cap);
@@ -192,81 +186,23 @@ impl<S: Adapter> Accessor for Backend<S> {
Ok(RpDelete::default())
}
- async fn list(&self, path: &str, _: OpList) -> Result<(RpList,
Self::Lister)> {
+ async fn list(&self, path: &str, args: OpList) -> Result<(RpList,
Self::Lister)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.scan(&p).await?;
let lister = KvLister::new(&self.root, res);
+ let lister = HierarchyLister::new(lister, path, args.recursive());
Ok((RpList::default(), lister))
}
- fn blocking_list(&self, path: &str, _: OpList) -> Result<(RpList,
Self::BlockingLister)> {
+ fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList,
Self::BlockingLister)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.blocking_scan(&p)?;
let lister = KvLister::new(&self.root, res);
+ let lister = HierarchyLister::new(lister, path, args.recursive());
Ok((RpList::default(), lister))
}
-
- async fn rename(&self, from: &str, to: &str, _: OpRename) ->
Result<RpRename> {
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.get(&from).await? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.set(&to, bs).await?;
- self.kv.delete(&from).await?;
- Ok(RpRename::default())
- }
-
- fn blocking_rename(&self, from: &str, to: &str, _: OpRename) ->
Result<RpRename> {
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.blocking_get(&from)? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.blocking_set(&to, bs)?;
- self.kv.blocking_delete(&from)?;
- Ok(RpRename::default())
- }
-
- async fn copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy> {
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.get(&from).await? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.set(&to, bs).await?;
-
- Ok(RpCopy::default())
- }
-
- fn blocking_copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy>
{
- let from = build_abs_path(&self.root, from);
- let to = build_abs_path(&self.root, to);
-
- let bs = match self.kv.blocking_get(&from)? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't
have this path")),
- };
-
- self.kv.blocking_set(&to, bs)?;
-
- Ok(RpCopy::default())
- }
}
impl<S> Backend<S>
diff --git a/core/src/raw/oio/list/flat_list.rs
b/core/src/raw/oio/list/flat_list.rs
index e6363bf80..9e8e2a5ac 100644
--- a/core/src/raw/oio/list/flat_list.rs
+++ b/core/src/raw/oio/list/flat_list.rs
@@ -84,15 +84,6 @@ where
{
/// Create a new flat lister
pub fn new(acc: A, path: &str) -> FlatLister<A, L> {
- #[cfg(debug_assertions)]
- {
- let meta = acc.info();
- debug_assert!(
- meta.full_capability().list_without_recursive,
- "service doesn't support list hierarchy, it must be a bug"
- );
- }
-
FlatLister {
acc: Some(acc),
root: path.to_string(),
@@ -254,7 +245,6 @@ mod tests {
fn info(&self) -> AccessorInfo {
let mut am = AccessorInfo::default();
am.full_capability_mut().list = true;
- am.full_capability_mut().list_without_recursive = true;
am
}
diff --git a/core/src/raw/oio/list/hierarchy_list.rs
b/core/src/raw/oio/list/hierarchy_list.rs
index 3c67abc11..259e45532 100644
--- a/core/src/raw/oio/list/hierarchy_list.rs
+++ b/core/src/raw/oio/list/hierarchy_list.rs
@@ -37,11 +37,12 @@ pub struct HierarchyLister<P> {
lister: P,
path: String,
visited: HashSet<String>,
+ recursive: bool,
}
impl<P> HierarchyLister<P> {
/// Create a new hierarchy lister
- pub fn new(lister: P, path: &str) -> HierarchyLister<P> {
+ pub fn new(lister: P, path: &str, recursive: bool) -> HierarchyLister<P> {
let path = if path == "/" {
"".to_string()
} else {
@@ -52,6 +53,7 @@ impl<P> HierarchyLister<P> {
lister,
path,
visited: HashSet::default(),
+ recursive,
}
}
@@ -75,6 +77,11 @@ impl<P> HierarchyLister<P> {
return false;
}
+ // Don't return already visited path.
+ if self.visited.contains(e.path()) {
+ return false;
+ }
+
let prefix_len = self.path.len();
let idx = if let Some(idx) = e.path()[prefix_len..].find('/') {
@@ -125,6 +132,9 @@ impl<P: oio::List> oio::List for HierarchyLister<P> {
None => return Poll::Ready(Ok(None)),
};
+ if self.recursive {
+ return Poll::Ready(Ok(Some(entry)));
+ }
if self.keep_entry(&mut entry) {
return Poll::Ready(Ok(Some(entry)));
}
@@ -140,6 +150,10 @@ impl<P: oio::BlockingList> oio::BlockingList for
HierarchyLister<P> {
None => return Ok(None),
};
+ if self.recursive {
+ return Ok(Some(entry));
+ }
+
if self.keep_entry(&mut entry) {
return Ok(Some(entry));
}
@@ -191,7 +205,7 @@ mod tests {
let lister = MockLister::new(vec![
"x/x/", "x/y/", "y/", "x/x/x", "y/y", "xy/", "z", "y/a",
]);
- let mut lister = HierarchyLister::new(lister, "");
+ let mut lister = HierarchyLister::new(lister, "", false);
let mut entries = Vec::default();
diff --git a/core/src/services/alluxio/backend.rs
b/core/src/services/alluxio/backend.rs
index 8cbf902d4..078a871da 100644
--- a/core/src/services/alluxio/backend.rs
+++ b/core/src/services/alluxio/backend.rs
@@ -204,7 +204,6 @@ impl Accessor for AlluxioBackend {
delete: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/azblob/backend.rs
b/core/src/services/azblob/backend.rs
index 7aca06dd2..bf77111e0 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -575,7 +575,6 @@ impl Accessor for AzblobBackend {
copy: true,
list: true,
- list_without_recursive: true,
list_with_recursive: true,
presign: self.has_sas_token,
diff --git a/core/src/services/azdls/backend.rs
b/core/src/services/azdls/backend.rs
index a494ece63..164a2d7f7 100644
--- a/core/src/services/azdls/backend.rs
+++ b/core/src/services/azdls/backend.rs
@@ -255,7 +255,6 @@ impl Accessor for AzdlsBackend {
rename: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/azfile/backend.rs
b/core/src/services/azfile/backend.rs
index ae3d8807b..f8b7f9418 100644
--- a/core/src/services/azfile/backend.rs
+++ b/core/src/services/azfile/backend.rs
@@ -270,7 +270,6 @@ impl Accessor for AzfileBackend {
rename: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
index b689acb0b..4c6c8660d 100644
--- a/core/src/services/b2/backend.rs
+++ b/core/src/services/b2/backend.rs
@@ -310,7 +310,6 @@ impl Accessor for B2Backend {
list_with_limit: true,
list_with_start_after: true,
list_with_recursive: true,
- list_without_recursive: true,
presign: true,
presign_read: true,
diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs
index 9583a8b5b..1abc9c5a6 100644
--- a/core/src/services/cos/backend.rs
+++ b/core/src/services/cos/backend.rs
@@ -288,7 +288,6 @@ impl Accessor for CosBackend {
copy: true,
list: true,
- list_without_recursive: true,
list_with_recursive: true,
presign: true,
diff --git a/core/src/services/dashmap/backend.rs
b/core/src/services/dashmap/backend.rs
index f64dd2865..b85b528e2 100644
--- a/core/src/services/dashmap/backend.rs
+++ b/core/src/services/dashmap/backend.rs
@@ -16,7 +16,7 @@
// under the License.
use std::collections::HashMap;
-use std::fmt::Debug;
+use std::fmt::{Debug, Formatter};
use async_trait::async_trait;
use dashmap::DashMap;
@@ -62,11 +62,19 @@ impl Builder for DashmapBuilder {
/// Backend is used to serve `Accessor` support in dashmap.
pub type DashmapBackend = typed_kv::Backend<Adapter>;
-#[derive(Debug, Clone)]
+#[derive(Clone)]
pub struct Adapter {
inner: DashMap<String, typed_kv::Value>,
}
+impl Debug for Adapter {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("DashmapAdapter")
+ .field("size", &self.inner.len())
+ .finish_non_exhaustive()
+ }
+}
+
#[async_trait]
impl typed_kv::Adapter for Adapter {
fn info(&self) -> typed_kv::Info {
diff --git a/core/src/services/dbfs/backend.rs
b/core/src/services/dbfs/backend.rs
index 4d85c3aad..6a5fcb602 100644
--- a/core/src/services/dbfs/backend.rs
+++ b/core/src/services/dbfs/backend.rs
@@ -177,7 +177,6 @@ impl Accessor for DbfsBackend {
rename: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index 892a0f79b..e2205e03c 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -265,7 +265,6 @@ impl Accessor for FsBackend {
delete: true,
list: true,
- list_without_recursive: true,
copy: true,
rename: true,
diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs
index 5b450875f..b0c2a4544 100644
--- a/core/src/services/ftp/backend.rs
+++ b/core/src/services/ftp/backend.rs
@@ -306,7 +306,6 @@ impl Accessor for FtpBackend {
create_dir: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 10259d939..8fe1a75c2 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -348,7 +348,6 @@ impl Accessor for GcsBackend {
list: true,
list_with_limit: true,
list_with_start_after: true,
- list_without_recursive: true,
list_with_recursive: true,
batch: true,
diff --git a/core/src/services/gdrive/backend.rs
b/core/src/services/gdrive/backend.rs
index be8e4b88a..c2e6a874e 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -58,7 +58,6 @@ impl Accessor for GdriveBackend {
read: true,
list: true,
- list_without_recursive: true,
write: true,
diff --git a/core/src/services/hdfs/backend.rs
b/core/src/services/hdfs/backend.rs
index ea857d4f6..bdff53de9 100644
--- a/core/src/services/hdfs/backend.rs
+++ b/core/src/services/hdfs/backend.rs
@@ -196,7 +196,6 @@ impl Accessor for HdfsBackend {
delete: true,
list: true,
- list_without_recursive: true,
rename: true,
blocking: true,
diff --git a/core/src/services/huggingface/backend.rs
b/core/src/services/huggingface/backend.rs
index 37c0b5064..b9d98867e 100644
--- a/core/src/services/huggingface/backend.rs
+++ b/core/src/services/huggingface/backend.rs
@@ -261,7 +261,6 @@ impl Accessor for HuggingfaceBackend {
read_with_range: true,
list: true,
- list_without_recursive: true,
list_with_recursive: true,
..Default::default()
diff --git a/core/src/services/ipfs/backend.rs
b/core/src/services/ipfs/backend.rs
index 915e2b798..081a56e73 100644
--- a/core/src/services/ipfs/backend.rs
+++ b/core/src/services/ipfs/backend.rs
@@ -180,7 +180,6 @@ impl Accessor for IpfsBackend {
read_with_range: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/ipmfs/backend.rs
b/core/src/services/ipmfs/backend.rs
index 7a9da9b4e..0433028f0 100644
--- a/core/src/services/ipmfs/backend.rs
+++ b/core/src/services/ipmfs/backend.rs
@@ -84,7 +84,6 @@ impl Accessor for IpmfsBackend {
delete: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs
index e753e800d..d72cca73c 100644
--- a/core/src/services/obs/backend.rs
+++ b/core/src/services/obs/backend.rs
@@ -294,7 +294,6 @@ impl Accessor for ObsBackend {
copy: true,
list: true,
- list_without_recursive: true,
list_with_recursive: true,
presign: true,
diff --git a/core/src/services/onedrive/backend.rs
b/core/src/services/onedrive/backend.rs
index ba4e5b8d3..db7ed22c3 100644
--- a/core/src/services/onedrive/backend.rs
+++ b/core/src/services/onedrive/backend.rs
@@ -80,7 +80,6 @@ impl Accessor for OnedriveBackend {
delete: true,
create_dir: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index 15fc23f6b..42a8b9d19 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -425,7 +425,6 @@ impl Accessor for OssBackend {
list: true,
list_with_limit: true,
list_with_start_after: true,
- list_without_recursive: true,
list_with_recursive: true,
presign: true,
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index a6317a6cd..4d9522620 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -1016,7 +1016,6 @@ impl Accessor for S3Backend {
list_with_limit: true,
list_with_start_after: true,
list_with_recursive: true,
- list_without_recursive: true,
presign: true,
presign_stat: true,
diff --git a/core/src/services/sftp/backend.rs
b/core/src/services/sftp/backend.rs
index ec9e2de4f..572bb4dd1 100644
--- a/core/src/services/sftp/backend.rs
+++ b/core/src/services/sftp/backend.rs
@@ -267,7 +267,6 @@ impl Accessor for SftpBackend {
list: true,
list_with_limit: true,
- list_without_recursive: true,
copy: self.copyable,
rename: true,
diff --git a/core/src/services/swift/backend.rs
b/core/src/services/swift/backend.rs
index 4e9e60540..6ae2a3b6a 100644
--- a/core/src/services/swift/backend.rs
+++ b/core/src/services/swift/backend.rs
@@ -237,7 +237,6 @@ impl Accessor for SwiftBackend {
delete: true,
list: true,
- list_without_recursive: true,
list_with_recursive: true,
..Default::default()
diff --git a/core/src/services/webdav/backend.rs
b/core/src/services/webdav/backend.rs
index 1361a11fd..b751cf026 100644
--- a/core/src/services/webdav/backend.rs
+++ b/core/src/services/webdav/backend.rs
@@ -248,7 +248,6 @@ impl Accessor for WebdavBackend {
rename: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/webhdfs/backend.rs
b/core/src/services/webhdfs/backend.rs
index 9fa9bd150..8a6a9f297 100644
--- a/core/src/services/webhdfs/backend.rs
+++ b/core/src/services/webhdfs/backend.rs
@@ -417,7 +417,6 @@ impl Accessor for WebhdfsBackend {
delete: true,
list: true,
- list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index c6f4fc52c..0847afa5e 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -132,8 +132,6 @@ pub struct Capability {
pub list_with_start_after: bool,
/// If backend supports list with recursive.
pub list_with_recursive: bool,
- /// If backend supports list without recursive.
- pub list_without_recursive: bool,
/// If operator supports presign.
pub presign: bool,
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index 64911f8fa..17f4a695a 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -261,7 +261,8 @@ pub async fn test_list_sub_dir(op: Operator) -> Result<()> {
/// List dir should also to list nested dir.
pub async fn test_list_nested_dir(op: Operator) -> Result<()> {
- let dir = format!("{}/{}/", uuid::Uuid::new_v4(), uuid::Uuid::new_v4());
+ let parent = format!("{}/", uuid::Uuid::new_v4());
+ let dir = format!("{parent}{}/", uuid::Uuid::new_v4());
let file_name = uuid::Uuid::new_v4().to_string();
let file_path = format!("{dir}{file_name}");
@@ -274,6 +275,11 @@ pub async fn test_list_nested_dir(op: Operator) ->
Result<()> {
.expect("creat must succeed");
op.create_dir(&dir_path).await.expect("creat must succeed");
+ let obs = op.list(&parent).await?;
+ assert_eq!(obs.len(), 1, "parent should only got 1 entry");
+ assert_eq!(obs[0].path(), dir);
+ assert_eq!(obs[0].metadata().mode(), EntryMode::DIR);
+
let mut obs = op.lister(&dir).await?;
let mut objects = HashMap::new();
