This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch change-list-api in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 63942a98c54888f191d6feb1ec6c3ab25ff3f2e1 Author: Xuanwo <[email protected]> AuthorDate: Sat Aug 5 10:44:17 2023 +0800 Save work Signed-off-by: Xuanwo <[email protected]> --- Cargo.lock | 87 ------------------------ Cargo.toml | 2 +- bin/oay/src/services/s3/service.rs | 2 +- bin/oay/src/services/webdav/webdavfs.rs | 2 +- bin/oli/src/commands/cp.rs | 2 +- bin/oli/src/commands/ls.rs | 4 +- bindings/nodejs/src/lib.rs | 4 +- bindings/object_store/src/lib.rs | 2 +- bindings/python/src/asyncio.rs | 2 +- core/src/layers/complete.rs | 2 +- core/src/layers/immutable_index.rs | 8 +-- core/src/layers/retry.rs | 2 +- core/src/types/operator/operator.rs | 101 ++++++++++++++-------------- core/src/types/operator/operator_futures.rs | 2 +- core/tests/behavior/list.rs | 16 ++--- core/tests/behavior/list_only.rs | 2 +- 16 files changed, 78 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 505dc8753..f08c347d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1174,12 +1174,6 @@ dependencies = [ "syn 2.0.23", ] -[[package]] -name = "cty" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" - [[package]] name = "darling" version = "0.14.4" @@ -3212,78 +3206,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "ocaml" -version = "1.0.0-beta.5" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "871da589fac01351dda8c8b26a5132b90d8accd5382424cd072902584ffe0c02" -dependencies = [ - "ocaml-boxroot-sys", - "ocaml-derive", - "ocaml-interop", - "ocaml-sys 0.23.0", -] - -[[package]] -name = "ocaml-boxroot-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "5186393bfbee4ce2bc5bbb82beafb77e85c1d0a557e3cfc8c8a0d63d7845fed5" -dependencies = [ - "cc", -] - -[[package]] -name = "ocaml-build" -version = "1.0.0-beta.4" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "ae2b0b155a81c2f3f9e71cb1f47e84d705d852e9ea7c10955d8bce1fa2d26aa0" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "ocaml-derive" -version = "1.0.0-beta.5" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "5cdffd393e5e346c52509608ac6c3be178523931bf9f50c18d23836474521d17" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "ocaml-interop" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "2e01e08412a7e072a90a225d2ae49a2860aeea853ce673bc63891dbf86aed063" -dependencies = [ - "ocaml-boxroot-sys", - "ocaml-sys 0.22.3", - "static_assertions", -] - -[[package]] -name = "ocaml-sys" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "73ec6ca7d41458442627435afb8f4671e83fd642e8a560171d671a1f679aa3cf" -dependencies = [ - "cty", -] - -[[package]] -name = "ocaml-sys" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index"; -checksum = "dc814766094de5f0f1c7e027027eb05c603cd7e235cc04ae195185fddecc1285" -dependencies = [ - "cc", - "cty", -] - [[package]] name = "oli" version = "0.39.0" @@ -3460,15 +3382,6 @@ dependencies = [ "opendal", ] -[[package]] -name = "opendal-ocaml" -version = "0.39.0" -dependencies = [ - "ocaml", - "ocaml-build", - "opendal", -] - [[package]] name = "opendal-python" version = "0.39.0" diff --git a/Cargo.toml b/Cargo.toml index 2c99a9699..1bb700005 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ members = [ "bindings/haskell", "bindings/lua", "bindings/dotnet", - "bindings/ocaml", +# "bindings/ocaml", "bin/oli", "bin/oay", diff --git a/bin/oay/src/services/s3/service.rs b/bin/oay/src/services/s3/service.rs index f8bf88280..3fec8625f 100644 --- a/bin/oay/src/services/s3/service.rs +++ b/bin/oay/src/services/s3/service.rs @@ -101,7 +101,7 @@ async fn handle_list_objects( let mut lister = state .op - .list_with(¶ms.prefix) + .lister_with(¶ms.prefix) .start_after(¶ms.start_after) .await?; diff --git a/bin/oay/src/services/webdav/webdavfs.rs b/bin/oay/src/services/webdav/webdavfs.rs index b9dea58a7..3eda5c539 100644 --- a/bin/oay/src/services/webdav/webdavfs.rs +++ b/bin/oay/src/services/webdav/webdavfs.rs @@ -62,7 +62,7 @@ impl DavFileSystem for WebdavFs { ) -> dav_server::fs::FsFuture<dav_server::fs::FsStream<Box<dyn dav_server::fs::DavDirEntry>>> { async move { - let lister = self.op.list(path.as_url_string().as_str()).await.unwrap(); + let lister = self.op.lister(path.as_url_string().as_str()).await.unwrap(); Ok(DavStream::new(self.op.clone(), lister).boxed()) } .boxed() diff --git a/bin/oli/src/commands/cp.rs b/bin/oli/src/commands/cp.rs index 0c4468922..9bb40bad4 100644 --- a/bin/oli/src/commands/cp.rs +++ b/bin/oli/src/commands/cp.rs @@ -57,7 +57,7 @@ pub async fn main(args: &ArgMatches) -> Result<()> { } let dst_root = Path::new(&dst_path); - let mut ds = src_op.scan(&src_path).await?; + let mut ds = src_op.lister_with(&src_path).delimiter("").await?; while let Some(de) = ds.try_next().await? { let meta = src_op.metadata(&de, Metakey::Mode).await?; if meta.mode().is_dir() { diff --git a/bin/oli/src/commands/ls.rs b/bin/oli/src/commands/ls.rs index 70747cffd..719b7dcb4 100644 --- a/bin/oli/src/commands/ls.rs +++ b/bin/oli/src/commands/ls.rs @@ -41,14 +41,14 @@ pub async fn main(args: &ArgMatches) -> Result<()> { let (op, path) = cfg.parse_location(target)?; if !recursive { - let mut ds = op.list(&path).await?; + let mut ds = op.lister(&path).await?; while let Some(de) = ds.try_next().await? { println!("{}", de.name()); } return Ok(()); } - let mut ds = op.scan(&path).await?; + let mut ds = op.lister_with(&path).delimiter("").await?; while let Some(de) = ds.try_next().await? { println!("{}", de.path()); } diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs index 0cebdcad5..5cc773dec 100644 --- a/bindings/nodejs/src/lib.rs +++ b/bindings/nodejs/src/lib.rs @@ -408,7 +408,9 @@ impl Operator { /// ``` #[napi] pub async fn list(&self, path: String) -> Result<Lister> { - Ok(Lister(self.0.list(&path).await.map_err(format_napi_error)?)) + Ok(Lister( + self.0.lister(&path).await.map_err(format_napi_error)?, + )) } /// List given path synchronously. diff --git a/bindings/object_store/src/lib.rs b/bindings/object_store/src/lib.rs index 421b6a686..8733033e2 100644 --- a/bindings/object_store/src/lib.rs +++ b/bindings/object_store/src/lib.rs @@ -170,7 +170,7 @@ impl ObjectStore for OpendalStore { let path = prefix.map_or("".into(), |x| format!("{}/", x)); let mut stream = self .inner - .list(&path) + .lister(&path) .await .map_err(|err| format_object_store_error(err, &path))?; diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs index 02dad1c18..709b81f83 100644 --- a/bindings/python/src/asyncio.rs +++ b/bindings/python/src/asyncio.rs @@ -139,7 +139,7 @@ impl AsyncOperator { pub fn list<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> { let this = self.0.clone(); future_into_py(py, async move { - let lister = this.list(&path).await.map_err(format_pyerr)?; + let lister = this.lister(&path).await.map_err(format_pyerr)?; let pylister: PyObject = Python::with_gil(|py| AsyncLister::new(lister).into_py(py)); Ok(pylister) }) diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index 20a1dd7b2..25048b6c1 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -1014,7 +1014,7 @@ mod tests { capability_test!(rename, |op| { op.rename("/path/to/mock_file", "/path/to/mock_file_2") }); - capability_test!(list, |op| { op.list("/path/to/mock_dir/") }); + capability_test!(list, |op| { op.lister("/path/to/mock_dir/") }); capability_test!(presign, |op| { op.presign_read("/path/to/mock_file", Duration::from_secs(1)) }); diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs index 1c7ed8992..665d6f45d 100644 --- a/core/src/layers/immutable_index.rs +++ b/core/src/layers/immutable_index.rs @@ -302,7 +302,7 @@ mod tests { let mut map = HashMap::new(); let mut set = HashSet::new(); - let mut ds = op.list("").await?; + let mut ds = op.lister("").await?; while let Some(entry) = ds.try_next().await? { debug!("got entry: {}", entry.path()); assert!( @@ -341,7 +341,7 @@ mod tests { .layer(iil) .finish(); - let mut ds = op.scan("/").await?; + let mut ds = op.lister_with("/").delimiter("").await?; let mut set = HashSet::new(); let mut map = HashMap::new(); while let Some(entry) = ds.try_next().await? { @@ -391,7 +391,7 @@ mod tests { // List / let mut map = HashMap::new(); let mut set = HashSet::new(); - let mut ds = op.list("/").await?; + let mut ds = op.lister("/").await?; while let Some(entry) = ds.try_next().await? { assert!( set.insert(entry.path().to_string()), @@ -410,7 +410,7 @@ mod tests { // List dataset/stateful/ let mut map = HashMap::new(); let mut set = HashSet::new(); - let mut ds = op.list("dataset/stateful/").await?; + let mut ds = op.lister("dataset/stateful/").await?; while let Some(entry) = ds.try_next().await? { assert!( set.insert(entry.path().to_string()), diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index 1408af035..be21adbd1 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -1356,7 +1356,7 @@ mod tests { let expected = vec!["hello", "world", "2023/", "0208/"]; let mut lister = op - .list("retryable_error/") + .lister("retryable_error/") .await .expect("service must support list"); let mut actual = Vec::new(); diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index c36bf20c5..49bc5e851 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -151,7 +151,7 @@ impl Operator { /// # } /// ``` pub async fn check(&self) -> Result<()> { - let mut ds = self.list("/").await?; + let mut ds = self.lister("/").await?; match ds.next().await { Some(Err(e)) if e.kind() != ErrorKind::NotFound => Err(e), @@ -1266,6 +1266,49 @@ impl Operator { Ok(()) } + /// List dir in flat way. + /// + /// Also, this function can be used to list a prefix. + /// + /// An error will be returned if given path doesn't end with `/`. + /// + /// # Notes + /// + /// - `scan` will not return the prefix itself. + /// - `scan` is an alias of `list_with(path).delimiter("")` + /// + /// # Examples + /// + /// ```no_run + /// # use anyhow::Result; + /// # use futures::io; + /// use futures::TryStreamExt; + /// use opendal::EntryMode; + /// use opendal::Metakey; + /// use opendal::Operator; + /// # + /// # #[tokio::main] + /// # async fn test(op: Operator) -> Result<()> { + /// let mut ds = op.scan("/path/to/dir/").await?; + /// while let Some(mut de) = ds.try_next().await? { + /// let meta = op.metadata(&de, Metakey::Mode).await?; + /// match meta.mode() { + /// EntryMode::FILE => { + /// println!("Handling file") + /// } + /// EntryMode::DIR => { + /// println!("Handling dir like start a new list via meta.path()") + /// } + /// EntryMode::Unknown => continue, + /// } + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn scan(&self, path: &str) -> Result<Lister> { + self.lister_with(path).delimiter("").await + } + /// List given path. /// /// This function will create a new handle to list entries. @@ -1283,7 +1326,7 @@ impl Operator { /// use opendal::Operator; /// # #[tokio::main] /// # async fn test(op: Operator) -> Result<()> { - /// let mut ds = op.list("path/to/dir/").await?; + /// let mut ds = op.lister("path/to/dir/").await?; /// while let Some(mut de) = ds.try_next().await? { /// let meta = op.metadata(&de, Metakey::Mode).await?; /// match meta.mode() { @@ -1299,8 +1342,8 @@ impl Operator { /// # Ok(()) /// # } /// ``` - pub async fn list(&self, path: &str) -> Result<Lister> { - self.list_with(path).await + pub async fn lister(&self, path: &str) -> Result<Lister> { + self.lister_with(path).await } /// List given path with OpList. @@ -1323,7 +1366,7 @@ impl Operator { /// # #[tokio::main] /// # async fn test(op: Operator) -> Result<()> { /// let mut ds = op - /// .list_with("path/to/dir/") + /// .lister_with("path/to/dir/") /// .limit(10) /// .start_after("start") /// .await?; @@ -1356,7 +1399,7 @@ impl Operator { /// use opendal::Operator; /// # #[tokio::main] /// # async fn test(op: Operator) -> Result<()> { - /// let mut ds = op.list_with("path/to/dir/").delimiter("").await?; + /// let mut ds = op.lister_with("path/to/dir/").delimiter("").await?; /// while let Some(mut de) = ds.try_next().await? { /// let meta = op.metadata(&de, Metakey::Mode).await?; /// match meta.mode() { @@ -1372,7 +1415,7 @@ impl Operator { /// # Ok(()) /// # } /// ``` - pub fn list_with(&self, path: &str) -> FutureList { + pub fn lister_with(&self, path: &str) -> FutureList { let path = normalize_path(path); let fut = FutureList(OperatorFuture::new( @@ -1400,50 +1443,8 @@ impl Operator { )); fut } - - /// List dir in flat way. - /// - /// Also, this function can be used to list a prefix. - /// - /// An error will be returned if given path doesn't end with `/`. - /// - /// # Notes - /// - /// - `scan` will not return the prefix itself. - /// - `scan` is an alias of `list_with(path).delimiter("")` - /// - /// # Examples - /// - /// ```no_run - /// # use anyhow::Result; - /// # use futures::io; - /// use futures::TryStreamExt; - /// use opendal::EntryMode; - /// use opendal::Metakey; - /// use opendal::Operator; - /// # - /// # #[tokio::main] - /// # async fn test(op: Operator) -> Result<()> { - /// let mut ds = op.scan("/path/to/dir/").await?; - /// while let Some(mut de) = ds.try_next().await? { - /// let meta = op.metadata(&de, Metakey::Mode).await?; - /// match meta.mode() { - /// EntryMode::FILE => { - /// println!("Handling file") - /// } - /// EntryMode::DIR => { - /// println!("Handling dir like start a new list via meta.path()") - /// } - /// EntryMode::Unknown => continue, - /// } - /// } - /// # Ok(()) - /// # } - /// ``` - pub async fn scan(&self, path: &str) -> Result<Lister> { - self.list_with(path).delimiter("").await - } } + /// Operator presign API. impl Operator { /// Presign an operation for stat(head). diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index 8fd52a31e..91440a476 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -565,7 +565,7 @@ impl Future for FutureDelete { } } -/// Future that generated by [`Operator::list_with`]. +/// Future that generated by [`Operator::lister_with`]. /// /// Users can add more options by public functions provided by this struct. pub struct FutureList(pub(crate) OperatorFuture<OpList, Lister>); diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs index bf8aa8339..808748ce5 100644 --- a/core/tests/behavior/list.rs +++ b/core/tests/behavior/list.rs @@ -66,7 +66,7 @@ pub async fn test_list_dir(op: Operator) -> Result<()> { op.write(&path, content).await.expect("write must succeed"); - let mut obs = op.list(&format!("{parent}/")).await?; + let mut obs = op.lister(&format!("{parent}/")).await?; let mut found = false; while let Some(de) = obs.try_next().await? { let meta = op.stat(de.path()).await?; @@ -105,7 +105,7 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()> { .collect::<Vec<_>>() .await; - let mut objects = op.with_limit(10).list("test_list_rich_dir/").await?; + let mut objects = op.with_limit(10).lister("test_list_rich_dir/").await?; let mut actual = vec![]; while let Some(o) = objects.try_next().await? { let path = o.path().to_string(); @@ -126,7 +126,7 @@ pub async fn test_list_empty_dir(op: Operator) -> Result<()> { op.create_dir(&dir).await.expect("write must succeed"); - let mut obs = op.list(&dir).await?; + let mut obs = op.lister(&dir).await?; let mut objects = HashMap::new(); while let Some(de) = obs.try_next().await? { objects.insert(de.path().to_string(), de); @@ -143,7 +143,7 @@ pub async fn test_list_empty_dir(op: Operator) -> Result<()> { pub async fn test_list_non_exist_dir(op: Operator) -> Result<()> { let dir = format!("{}/", uuid::Uuid::new_v4()); - let mut obs = op.list(&dir).await?; + let mut obs = op.lister(&dir).await?; let mut objects = HashMap::new(); while let Some(de) = obs.try_next().await? { objects.insert(de.path().to_string(), de); @@ -160,7 +160,7 @@ pub async fn test_list_sub_dir(op: Operator) -> Result<()> { op.create_dir(&path).await.expect("creat must succeed"); - let mut obs = op.list("/").await?; + let mut obs = op.lister("/").await?; let mut found = false; while let Some(de) = obs.try_next().await? { if de.path() == path { @@ -192,7 +192,7 @@ pub async fn test_list_nested_dir(op: Operator) -> Result<()> { .expect("creat must succeed"); op.create_dir(&dir_path).await.expect("creat must succeed"); - let mut obs = op.list(&dir).await?; + let mut obs = op.lister(&dir).await?; let mut objects = HashMap::new(); while let Some(de) = obs.try_next().await? { @@ -235,7 +235,7 @@ pub async fn test_list_nested_dir(op: Operator) -> Result<()> { pub async fn test_list_dir_with_file_path(op: Operator) -> Result<()> { let parent = uuid::Uuid::new_v4().to_string(); - let obs = op.list(&parent).await.map(|_| ()); + let obs = op.lister(&parent).await.map(|_| ()); assert!(obs.is_err()); assert_eq!(obs.unwrap_err().kind(), ErrorKind::NotADirectory); @@ -267,7 +267,7 @@ pub async fn test_list_with_start_after(op: Operator) -> Result<()> { .collect::<Vec<_>>() .await; - let mut objects = op.list_with(dir).start_after(&given[2]).await?; + let mut objects = op.lister_with(dir).start_after(&given[2]).await?; let mut actual = vec![]; while let Some(o) = objects.try_next().await? { let path = o.path().to_string(); diff --git a/core/tests/behavior/list_only.rs b/core/tests/behavior/list_only.rs index 6ddf64cb3..e7778461d 100644 --- a/core/tests/behavior/list_only.rs +++ b/core/tests/behavior/list_only.rs @@ -36,7 +36,7 @@ pub fn behavior_list_only_tests(op: &Operator) -> Vec<Trial> { pub async fn test_list_only(op: Operator) -> Result<()> { let mut entries = HashMap::new(); - let mut ds = op.list("/").await?; + let mut ds = op.lister("/").await?; while let Some(de) = ds.try_next().await? { entries.insert(de.path().to_string(), op.stat(de.path()).await?.mode()); }
