This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch xuanwo/reduce-list-stat-fallback in repository https://gitbox.apache.org/repos/asf/opendal.git
commit e8711541b4ecda2193540d7f8641408d31f4327f Author: Xuanwo <[email protected]> AuthorDate: Mon Feb 23 20:44:17 2026 +0800 feat(services): reduce list fallback stat for local kv backends --- core/services/compfs/src/lister.rs | 8 ++++++-- core/services/dashmap/src/lister.rs | 19 ++++++++----------- core/services/mini_moka/src/lister.rs | 23 ++++++----------------- core/services/moka/src/lister.rs | 25 ++++++++++--------------- core/services/rocksdb/src/core.rs | 11 ++++++----- core/services/rocksdb/src/lister.rs | 10 +++++++--- core/services/sled/src/core.rs | 9 +++++---- core/services/sled/src/lister.rs | 10 +++++++--- 8 files changed, 55 insertions(+), 60 deletions(-) diff --git a/core/services/compfs/src/lister.rs b/core/services/compfs/src/lister.rs index 78a27fd3a..fddb4be61 100644 --- a/core/services/compfs/src/lister.rs +++ b/core/services/compfs/src/lister.rs @@ -58,10 +58,14 @@ fn next_entry(read_dir: &mut ReadDir, root: &Path) -> std::io::Result<Option<oio let path = entry.path(); let rel_path = normalize(&path, root); - let file_type = entry.file_type()?; + let de_metadata = entry.metadata()?; + let file_type = de_metadata.file_type(); let entry = if file_type.is_file() { - oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE)) + oio::Entry::new( + &rel_path, + Metadata::new(EntryMode::FILE).with_content_length(de_metadata.len()), + ) } else if file_type.is_dir() { oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR)) } else { diff --git a/core/services/dashmap/src/lister.rs b/core/services/dashmap/src/lister.rs index d74cc5891..b8ace7631 100644 --- a/core/services/dashmap/src/lister.rs +++ b/core/services/dashmap/src/lister.rs @@ -26,12 +26,16 @@ use super::core::DashmapCore; pub struct DashmapLister { root: String, path: String, - iter: IntoIter<String>, + iter: IntoIter<(String, Metadata)>, } impl DashmapLister { pub fn new(core: Arc<DashmapCore>, root: String, path: String) -> Self { - let entries: Vec<_> = core.cache.iter().map(|item| item.key().clone()).collect(); + let entries: Vec<_> = core + .cache + .iter() + .map(|item| (item.key().clone(), item.value().metadata.clone())) + .collect(); let path = build_abs_path(&root, &path); Self { @@ -44,17 +48,10 @@ impl DashmapLister { impl oio::List for DashmapLister { async fn next(&mut self) -> Result<Option<oio::Entry>> { - for key in self.iter.by_ref() { + for (key, metadata) in self.iter.by_ref() { if key.starts_with(&self.path) { let path = build_rel_path(&self.root, &key); - - // Determine if it's a file or directory based on trailing slash - let mode = if key.ends_with('/') { - EntryMode::DIR - } else { - EntryMode::FILE - }; - let entry = oio::Entry::new(&path, Metadata::new(mode)); + let entry = oio::Entry::new(&path, metadata); return Ok(Some(entry)); } } diff --git a/core/services/mini_moka/src/lister.rs b/core/services/mini_moka/src/lister.rs index d608d8b22..67a1d168e 100644 --- a/core/services/mini_moka/src/lister.rs +++ b/core/services/mini_moka/src/lister.rs @@ -25,41 +25,30 @@ use super::core::MiniMokaCore; pub struct MiniMokaLister { root: String, - keys: IntoIter<String>, + entries: IntoIter<(String, Metadata)>, } impl MiniMokaLister { pub fn new(core: Arc<MiniMokaCore>, root: String, _path: String) -> Self { - // Get all keys from the cache - let keys: Vec<String> = core + let entries: Vec<(String, Metadata)> = core .cache .iter() - .map(|entry| entry.key().to_string()) + .map(|entry| (entry.key().to_string(), entry.value().metadata.clone())) .collect(); Self { root, - keys: keys.into_iter(), + entries: entries.into_iter(), } } } impl oio::List for MiniMokaLister { async fn next(&mut self) -> Result<Option<oio::Entry>> { - match self.keys.next() { - Some(key) => { - // Convert absolute path to relative path + match self.entries.next() { + Some((key, metadata)) => { let rel_path = build_rel_path(&self.root, &key); - // Determine if it's a file or directory based on trailing slash - let mode = if key.ends_with('/') { - EntryMode::DIR - } else { - EntryMode::FILE - }; - - let metadata = Metadata::new(mode); - Ok(Some(oio::Entry::new(&rel_path, metadata))) } None => Ok(None), diff --git a/core/services/moka/src/lister.rs b/core/services/moka/src/lister.rs index 4c60e5784..fef81e605 100644 --- a/core/services/moka/src/lister.rs +++ b/core/services/moka/src/lister.rs @@ -25,39 +25,34 @@ use opendal_core::*; pub struct MokaLister { root: String, - keys: IntoIter<String>, + entries: IntoIter<(String, Metadata)>, } impl MokaLister { pub fn new(core: Arc<MokaCore>, root: String, _path: String) -> Self { - // Get all keys from the cache - let keys: Vec<String> = core.cache.iter().map(|kv| kv.0.to_string()).collect(); + let entries: Vec<(String, Metadata)> = core + .cache + .iter() + .map(|(key, value)| (key.to_string(), value.metadata.clone())) + .collect(); Self { root, - keys: keys.into_iter(), + entries: entries.into_iter(), } } } impl oio::List for MokaLister { async fn next(&mut self) -> Result<Option<oio::Entry>> { - match self.keys.next() { - Some(key) => { - // Convert absolute path to relative path + match self.entries.next() { + Some((key, metadata)) => { let mut path = build_rel_path(&self.root, &key); if path.is_empty() { path = "/".to_string(); } - // Determine if it's a file or directory based on trailing slash - let mode = if key.ends_with('/') { - EntryMode::DIR - } else { - EntryMode::FILE - }; - - Ok(Some(oio::Entry::new(&path, Metadata::new(mode)))) + Ok(Some(oio::Entry::new(&path, metadata))) } None => Ok(None), } diff --git a/core/services/rocksdb/src/core.rs b/core/services/rocksdb/src/core.rs index 6beb2d63d..15ef21dc8 100644 --- a/core/services/rocksdb/src/core.rs +++ b/core/services/rocksdb/src/core.rs @@ -51,17 +51,18 @@ impl RocksdbCore { self.db.delete(path).map_err(parse_rocksdb_error) } - pub fn list(&self, path: &str) -> Result<Vec<String>> { - let it = self.db.prefix_iterator(path).map(|r| r.map(|(k, _)| k)); + pub fn list(&self, path: &str) -> Result<Vec<(String, u64)>> { + let it = self.db.prefix_iterator(path); let mut res = Vec::default(); - for key in it { - let key = key.map_err(parse_rocksdb_error)?; + for entry in it { + let (key, value) = entry.map_err(parse_rocksdb_error)?; let key = String::from_utf8_lossy(&key); if !key.starts_with(path) { break; } - res.push(key.to_string()); + + res.push((key.to_string(), value.len() as u64)); } Ok(res) diff --git a/core/services/rocksdb/src/lister.rs b/core/services/rocksdb/src/lister.rs index b39548cec..387346223 100644 --- a/core/services/rocksdb/src/lister.rs +++ b/core/services/rocksdb/src/lister.rs @@ -25,7 +25,7 @@ use super::core::*; pub struct RocksdbLister { root: String, - iter: IntoIter<String>, + iter: IntoIter<(String, u64)>, } impl RocksdbLister { @@ -41,7 +41,7 @@ impl RocksdbLister { impl oio::List for RocksdbLister { async fn next(&mut self) -> Result<Option<oio::Entry>> { - if let Some(key) = self.iter.next() { + if let Some((key, value_len)) = self.iter.next() { let path = build_rel_path(&self.root, &key); // Determine if it's a file or directory based on trailing slash @@ -50,7 +50,11 @@ impl oio::List for RocksdbLister { } else { EntryMode::FILE }; - let entry = oio::Entry::new(&path, Metadata::new(mode)); + let mut metadata = Metadata::new(mode); + if metadata.mode().is_file() { + metadata.set_content_length(value_len); + } + let entry = oio::Entry::new(&path, metadata); return Ok(Some(entry)); } diff --git a/core/services/sled/src/core.rs b/core/services/sled/src/core.rs index d00b8d82f..49c9fe610 100644 --- a/core/services/sled/src/core.rs +++ b/core/services/sled/src/core.rs @@ -51,18 +51,19 @@ impl SledCore { Ok(()) } - pub fn list(&self, path: &str) -> Result<Vec<String>> { - let it = self.tree.scan_prefix(path).keys(); + pub fn list(&self, path: &str) -> Result<Vec<(String, u64)>> { + let it = self.tree.scan_prefix(path); let mut res = Vec::default(); for i in it { - let bs = i.map_err(parse_error)?.to_vec(); + let (key, value) = i.map_err(parse_error)?; + let bs = key.to_vec(); let v = String::from_utf8(bs).map_err(|err| { Error::new(ErrorKind::Unexpected, "store key is not valid utf-8 string") .set_source(err) })?; - res.push(v); + res.push((v, value.len() as u64)); } Ok(res) diff --git a/core/services/sled/src/lister.rs b/core/services/sled/src/lister.rs index 04c648cce..a5111d6b9 100644 --- a/core/services/sled/src/lister.rs +++ b/core/services/sled/src/lister.rs @@ -25,7 +25,7 @@ use opendal_core::*; pub struct SledLister { root: String, - iter: IntoIter<String>, + iter: IntoIter<(String, u64)>, } impl SledLister { @@ -41,7 +41,7 @@ impl SledLister { impl oio::List for SledLister { async fn next(&mut self) -> Result<Option<oio::Entry>> { - if let Some(key) = self.iter.next() { + if let Some((key, value_len)) = self.iter.next() { let path = build_rel_path(&self.root, &key); // Determine if it's a file or directory based on trailing slash @@ -50,7 +50,11 @@ impl oio::List for SledLister { } else { EntryMode::FILE }; - let entry = oio::Entry::new(&path, Metadata::new(mode)); + let mut metadata = Metadata::new(mode); + if metadata.mode().is_file() { + metadata.set_content_length(value_len); + } + let entry = oio::Entry::new(&path, metadata); return Ok(Some(entry)); }
