This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch list-with-metakey-for-blocking in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 57c96f1f6ac360a2c3a6d70239c2e55b0f354af1 Author: Xuanwo <[email protected]> AuthorDate: Mon Aug 14 15:18:31 2023 +0800 Implement list for blocking operator Signed-off-by: Xuanwo <[email protected]> --- core/src/types/operator/blocking_operator.rs | 153 ++++++++++++++++++++++++++ core/src/types/operator/operator_functions.rs | 37 +++++++ 2 files changed, 190 insertions(+) diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs index 143592762..c964bf78a 100644 --- a/core/src/types/operator/blocking_operator.rs +++ b/core/src/types/operator/blocking_operator.rs @@ -711,6 +711,159 @@ impl BlockingOperator { Ok(()) } + /// List entries within a given directory. + /// + /// # Notes + /// + /// ## Listing recursively + /// + /// This function only read the children of the given directory. To read + /// all entries recursively, use `BlockingOperator::list_with("path").delimiter("")` + /// instead. + /// + /// ## Streaming + /// + /// This function will read all entries in the given directory. It could + /// take very long time and consume a lot of memory if the directory + /// contains a lot of entries. + /// + /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in + /// a streaming way. + /// + /// ## Metadata + /// + /// The only metadata that is guaranteed to be available is the `Mode`. + /// For fetching more metadata, please use [`BlockingOperator::list_with`] and `metakey`. + /// + /// # Examples + /// + /// ```no_run + /// # use anyhow::Result; + /// use opendal::EntryMode; + /// use opendal::Metakey; + /// use opendal::BlockingOperator; + /// # fn test(op: BlockingOperator) -> Result<()> { + /// let mut entries = op.list("path/to/dir/")?; + /// for entry in entries { + /// match entry.metadata().mode() { + /// EntryMode::FILE => { + /// println!("Handling file") + /// } + /// EntryMode::DIR => { + /// println!("Handling dir {}", entry.path()) + /// } + /// EntryMode::Unknown => continue, + /// } + /// } + /// # Ok(()) + /// # } + /// ``` + pub fn list(&self, path: &str) -> Result<Vec<Entry>> { + self.list_with(path).call() + } + + /// List entries within a given directory with options. + /// + /// # Notes + /// + /// ## For streaming + /// + /// This function will read all entries in the given directory. It could + /// take very long time and consume a lot of memory if the directory + /// contains a lot of entries. + /// + /// In order to avoid this, you can use [`Operator::lister`] to list entries in + /// a streaming way. + /// + /// ## Metadata + /// + /// The only metadata that is guaranteed to be available is the `Mode`. + /// For fetching more metadata, please specify the `metakey`. + /// + /// # Examples + /// + /// ## List entries with prefix + /// + /// This function can also be used to list entries in recursive way. + /// + /// ```no_run + /// # use anyhow::Result; + /// use opendal::EntryMode; + /// use opendal::Metakey; + /// use opendal::BlockingOperator; + /// # fn test(op: BlockingOperator) -> Result<()> { + /// let mut entries = op.list_with("prefix/").delimiter("").call()?; + /// for entry in entries { + /// match entry.metadata().mode() { + /// EntryMode::FILE => { + /// println!("Handling file") + /// } + /// EntryMode::DIR => { + /// println!("Handling dir like start a new list via meta.path()") + /// } + /// EntryMode::Unknown => continue, + /// } + /// } + /// # Ok(()) + /// # } + /// ``` + /// + /// ## List entries with metakey for more metadata + /// + /// ```no_run + /// # use anyhow::Result; + /// use opendal::EntryMode; + /// use opendal::Metakey; + /// use opendal::BlockingOperator; + /// # fn test(op: BlockingOperator) -> Result<()> { + /// let mut entries = op + /// .list_with("dir/") + /// .metakey(Metakey::ContentLength | Metakey::LastModified) + /// .call()?; + /// for entry in entries { + /// let meta = entry.metadata(); + /// match meta.mode() { + /// EntryMode::FILE => { + /// println!( + /// "Handling file {} with size {}", + /// entry.path(), + /// meta.content_length() + /// ) + /// } + /// EntryMode::DIR => { + /// println!("Handling dir {}", entry.path()) + /// } + /// EntryMode::Unknown => continue, + /// } + /// } + /// # Ok(()) + /// # } + /// ``` + pub fn list_with(&self, path: &str) -> FunctionList { + let path = normalize_path(path); + + FunctionList(OperatorFunction::new( + self.inner().clone(), + path, + OpList::default(), + |inner, path, args| { + if !validate_path(&path, EntryMode::FILE) { + return Err( + Error::new(ErrorKind::IsADirectory, "write path is a directory") + .with_operation("BlockingOperator::write_with") + .with_context("service", inner.info().scheme().into_static()) + .with_context("path", &path), + ); + } + + let (_, pager) = inner.blocking_list(&path, args)?; + let lister = BlockingLister::new(pager); + + lister.collect() + }, + )) + } + /// List entries within a given directory as an iterator. /// /// This function will create a new handle to list entries. diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs index 5734950e0..2164bcafb 100644 --- a/core/src/types/operator/operator_functions.rs +++ b/core/src/types/operator/operator_functions.rs @@ -116,6 +116,43 @@ impl FunctionDelete { } } +/// Function that generated by [`BlockingOperator::list_with`]. +/// +/// Users can add more options by public functions provided by this struct. +pub struct FunctionList(pub(crate) OperatorFunction<OpList, Vec<Entry>>); + +impl FunctionList { + /// Change the limit of this list operation. + pub fn limit(mut self, v: usize) -> Self { + self.0 = self.0.map_args(|args| args.with_limit(v)); + self + } + + /// Change the start_after of this list operation. + pub fn start_after(mut self, v: &str) -> Self { + self.0 = self.0.map_args(|args| args.with_start_after(v)); + self + } + + /// Change the delimiter. The default delimiter is "/" + pub fn delimiter(mut self, v: &str) -> Self { + self.0 = self.0.map_args(|args| args.with_delimiter(v)); + self + } + + /// Change the metakey. The default metakey is `Metakey::Mode`. + pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self { + self.0 = self.0.map_args(|args| args.with_metakey(v)); + self + } + + /// Call the function to consume all the input and generate a + /// result. + pub fn call(self) -> Result<Vec<Entry>> { + self.0.call() + } +} + /// Function that generated by [`BlockingOperator::lister_with`]. /// /// Users can add more options by public functions provided by this struct.
