stayrascal commented on code in PR #324:
URL:
https://github.com/apache/arrow-rs-object-store/pull/324#discussion_r2052581960
##########
src/lib.rs:
##########
@@ -722,6 +722,64 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync +
Debug + 'static {
/// Note: the order of returned [`ObjectMeta`] is not guaranteed
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static,
Result<ObjectMeta>>;
+ /// List all the objects with given options defined in [`ListOpts`]
+ ///
+ /// Prefixes are evaluated on a path segment basis, i.e. `foo/bar` is a
prefix of `foo/bar/x` but not of
+ /// `foo/bar_baz/x`. List is recursive, i.e. `foo/bar/more/x` will be
included.
+ fn list_opts(
+ &self,
+ prefix: Option<&Path>,
+ options: ListOpts,
+ ) -> BoxStream<'static, Result<ListResult>> {
+ // Add the default implementation via calling list_with_offset methods
avoid bring
+ // a breaking change, the default implementation and other list
methods will be removed
+ // in the future. All list request should call this method.
+ use std::collections::BTreeSet;
+
+ let prefix = prefix.cloned().unwrap_or_default();
+ let offset = options.offset.unwrap_or(prefix.clone());
+ self.list_with_offset(Some(&prefix), &offset)
+ .try_chunks(1000)
+ .map(move |batch| {
+ let batch = batch.map_err(|e| e.1)?;
+ if options.delimiter {
+ let mut common_prefixes = BTreeSet::new();
+ let mut objects = Vec::new();
+ for obj in batch.into_iter() {
+ let location = obj.location.clone();
+ let mut parts = match location.prefix_match(&prefix) {
+ Some(parts) => parts,
+ None => continue,
+ };
+
+ match parts.next() {
+ None => {}
+ Some(p) => match parts.next() {
+ None => objects.push(obj),
+ Some(_) => {
+ let common_prefix = prefix.child(p);
+ if common_prefix > offset {
+ common_prefixes.insert(common_prefix);
+ }
+ }
+ },
+ }
+ drop(parts)
+ }
+ Ok(ListResult {
+ common_prefixes: common_prefixes.into_iter().collect(),
+ objects,
+ })
+ } else {
+ Ok(ListResult {
+ common_prefixes: vec![],
+ objects: batch,
+ })
+ }
+ })
+ .boxed()
Review Comment:
sure, throw `NotImplemented` error is a simple way,
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
