This is an automated email from the ASF dual-hosted git repository. erickguan pushed a commit to branch fix-stat-impl in repository https://gitbox.apache.org/repos/asf/opendal.git
commit cbc3aefe70e6f599e15e69cd4e02a262c1baddd0 Author: Erick Guan <[email protected]> AuthorDate: Sun Jul 5 14:51:30 2026 +0800 Progress --- core/services/etcd/src/backend.rs | 6 +++--- core/services/etcd/src/core.rs | 13 +++++++++++++ core/services/foundationdb/src/backend.rs | 7 +++---- core/services/foundationdb/src/core.rs | 19 +++++++++++++++++++ core/services/foyer/src/backend.rs | 6 +++--- core/services/foyer/src/core.rs | 24 ++++++++++++++++++++++++ core/services/gridfs/src/backend.rs | 7 +++---- core/services/gridfs/src/core.rs | 13 +++++++++++++ core/services/memcached/src/backend.rs | 7 +++---- core/services/memcached/src/core.rs | 10 ++++++++++ core/services/persy/src/backend.rs | 7 +++---- core/services/persy/src/core.rs | 16 ++++++++++++++++ core/services/redb/src/backend.rs | 7 +++---- core/services/redb/src/core.rs | 20 ++++++++++++++++++++ core/services/rocksdb/src/backend.rs | 7 +++---- core/services/rocksdb/src/core.rs | 8 ++++++++ core/services/sled/src/backend.rs | 7 +++---- core/services/sled/src/core.rs | 8 ++++++++ core/services/tikv/src/backend.rs | 7 +++---- core/services/tikv/src/core.rs | 14 ++++++++++++++ 20 files changed, 175 insertions(+), 38 deletions(-) diff --git a/core/services/etcd/src/backend.rs b/core/services/etcd/src/backend.rs index 675605ce0..274b1feda 100644 --- a/core/services/etcd/src/backend.rs +++ b/core/services/etcd/src/backend.rs @@ -244,10 +244,10 @@ impl Service for EtcdBackend { let abs_path = build_abs_path(&self.info.root(), path); // First check if it's a direct key - match self.core.get(&abs_path).await? { - Some(buffer) => { + match self.core.get_length(&abs_path).await? { + Some(length) => { let mut metadata = Metadata::new(EntryMode::from_path(&abs_path)); - metadata.set_content_length(buffer.len() as u64); + metadata.set_content_length(length as u64); Ok(RpStat::new(metadata)) } None => { diff --git a/core/services/etcd/src/core.rs b/core/services/etcd/src/core.rs index 90a8f4587..cae672001 100644 --- a/core/services/etcd/src/core.rs +++ b/core/services/etcd/src/core.rs @@ -132,6 +132,19 @@ impl EtcdCore { } } + /// Get the byte length of the value for the given key without copying + /// the value data. Uses `kv.value().len()` which returns the length + /// from the etcd response without allocating a `Vec<u8>`. + pub async fn get_length(&self, key: &str) -> Result<Option<usize>> { + let mut client = self.conn().await?; + let resp = client.get(key, None).await.map_err(format_etcd_error)?; + if let Some(kv) = resp.kvs().first() { + Ok(Some(kv.value().len())) + } else { + Ok(None) + } + } + pub async fn set(&self, key: &str, value: Buffer) -> Result<()> { let mut client = self.conn().await?; let _ = client diff --git a/core/services/foundationdb/src/backend.rs b/core/services/foundationdb/src/backend.rs index fea7ec9cc..f57256269 100644 --- a/core/services/foundationdb/src/backend.rs +++ b/core/services/foundationdb/src/backend.rs @@ -153,10 +153,9 @@ impl Service for FoundationdbBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p).await?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p).await? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new( ErrorKind::NotFound, diff --git a/core/services/foundationdb/src/core.rs b/core/services/foundationdb/src/core.rs index 86ac15953..1b9649f84 100644 --- a/core/services/foundationdb/src/core.rs +++ b/core/services/foundationdb/src/core.rs @@ -55,6 +55,25 @@ impl FoundationdbCore { } } + /// Get the byte length of the value for the given key without copying + /// the value data. Uses `FdbSlice::len()` which returns the length + /// without materializing the full value into a `Vec<u8>`. + pub async fn get_length(&self, path: &str) -> Result<Option<usize>> { + let transaction = self.db.create_trx().expect("Unable to create transaction"); + + match transaction.get(path.as_bytes(), false).await { + Ok(Some(data)) => Ok(Some(data.len())), + Ok(None) => Err(Error::new( + ErrorKind::NotFound, + "foundationdb: key not found", + )), + Err(_) => Err(Error::new( + ErrorKind::NotFound, + "foundationdb: key not found", + )), + } + } + pub async fn set(&self, path: &str, value: Buffer) -> Result<()> { let transaction = self.db.create_trx().expect("Unable to create transaction"); diff --git a/core/services/foyer/src/backend.rs b/core/services/foyer/src/backend.rs index 3591ccf03..75e6aab04 100644 --- a/core/services/foyer/src/backend.rs +++ b/core/services/foyer/src/backend.rs @@ -272,9 +272,9 @@ impl Service for FoyerBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - match self.core.get(&p).await? { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p).await? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "key not found in foyer")), } diff --git a/core/services/foyer/src/core.rs b/core/services/foyer/src/core.rs index 216170ba1..b95ce64e6 100644 --- a/core/services/foyer/src/core.rs +++ b/core/services/foyer/src/core.rs @@ -174,6 +174,30 @@ impl FoyerCore { } } + /// Get the byte length of the value for the given key without cloning + /// the value data. Uses `entry.value().0.len()` which reads the + /// length from the `Buffer` without cloning it. + pub async fn get_length(&self, key: &str) -> Result<Option<usize>> { + let cache = self.get_cache().await?; + let foyer_key = FoyerKey { + path: key.to_string(), + }; + if self + .deleted_keys + .lock() + .expect("deleted keys lock poisoned") + .contains(&foyer_key) + { + return Ok(None); + } + + match cache.get(&foyer_key).await { + Ok(Some(entry)) => Ok(Some(entry.value().0.len())), + Ok(None) => Ok(None), + Err(_) => Ok(None), + } + } + pub async fn insert(&self, key: &str, value: Buffer) -> Result<()> { let cache = self.get_cache().await?; let key = FoyerKey { diff --git a/core/services/gridfs/src/backend.rs b/core/services/gridfs/src/backend.rs index 2a14a879e..88200ca19 100644 --- a/core/services/gridfs/src/backend.rs +++ b/core/services/gridfs/src/backend.rs @@ -223,10 +223,9 @@ impl Service for GridfsBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p).await?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p).await? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in gridfs")), } diff --git a/core/services/gridfs/src/core.rs b/core/services/gridfs/src/core.rs index 3f5f9a9bb..489f1eab6 100644 --- a/core/services/gridfs/src/core.rs +++ b/core/services/gridfs/src/core.rs @@ -87,6 +87,19 @@ impl GridfsCore { Ok(Some(Buffer::from(destination))) } + /// Get the byte length of the file without downloading its contents. + /// Queries the GridFS files collection for the `length` metadata field, + /// avoiding the need to download and reassemble all chunks. + pub async fn get_length(&self, path: &str) -> Result<Option<usize>> { + let bucket = self.get_bucket().await?; + let filter = doc! { "filename": path }; + let Some(doc) = bucket.find_one(filter).await.map_err(parse_mongodb_error)? else { + return Ok(None); + }; + + Ok(Some(doc.length as usize)) + } + pub async fn set(&self, path: &str, value: Buffer) -> Result<()> { let bucket = self.get_bucket().await?; diff --git a/core/services/memcached/src/backend.rs b/core/services/memcached/src/backend.rs index 7d6e4a2d8..27d2f5be4 100644 --- a/core/services/memcached/src/backend.rs +++ b/core/services/memcached/src/backend.rs @@ -248,10 +248,9 @@ impl Service for MemcachedBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p).await?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p).await? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in memcached")), } diff --git a/core/services/memcached/src/core.rs b/core/services/memcached/src/core.rs index a272a2b51..1189c8aac 100644 --- a/core/services/memcached/src/core.rs +++ b/core/services/memcached/src/core.rs @@ -199,6 +199,16 @@ impl MemcachedCore { Ok(result.map(Buffer::from)) } + /// Get the byte length of the value for the given key without converting + /// the value into a `Buffer`. The memcached protocol does not provide a + /// length-only command, so the full value is fetched, but the + /// `Buffer::from()` allocation is avoided. + pub async fn get_length(&self, key: &str) -> Result<Option<usize>> { + let mut conn = self.conn().await?; + let result = conn.get(&percent_encode_path(key)).await?; + Ok(result.as_ref().map(|v| v.len())) + } + pub async fn set(&self, key: &str, value: Buffer) -> Result<()> { let mut conn = self.conn().await?; diff --git a/core/services/persy/src/backend.rs b/core/services/persy/src/backend.rs index a27c62633..303470510 100644 --- a/core/services/persy/src/backend.rs +++ b/core/services/persy/src/backend.rs @@ -182,10 +182,9 @@ impl Service for PersyBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p)?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p)? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in persy")), } diff --git a/core/services/persy/src/core.rs b/core/services/persy/src/core.rs index db9a05bd6..36a7b1895 100644 --- a/core/services/persy/src/core.rs +++ b/core/services/persy/src/core.rs @@ -51,6 +51,22 @@ impl PersyCore { Ok(None) } + /// Get the byte length of the value for the given key without converting + /// the value into a `Buffer`. Persy does not provide a length-only API, + /// so the full record is read, but the `Buffer::from()` allocation is avoided. + pub fn get_length(&self, path: &str) -> Result<Option<usize>> { + let mut read_id = self + .persy + .get::<String, persy::PersyId>(&self.index, &path.to_string()) + .map_err(parse_error)?; + if let Some(id) = read_id.next() { + let value = self.persy.read(&self.segment, &id).map_err(parse_error)?; + return Ok(value.as_ref().map(|v| v.len())); + } + + Ok(None) + } + pub fn set(&self, path: &str, value: Buffer) -> Result<()> { let mut tx = self.persy.begin().map_err(parse_error)?; let id = tx diff --git a/core/services/redb/src/backend.rs b/core/services/redb/src/backend.rs index 8469c542d..f9d6ee836 100644 --- a/core/services/redb/src/backend.rs +++ b/core/services/redb/src/backend.rs @@ -200,10 +200,9 @@ impl Service for RedbBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p)?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p)? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in redb")), } diff --git a/core/services/redb/src/core.rs b/core/services/redb/src/core.rs index 99ff2f0f6..64f826e55 100644 --- a/core/services/redb/src/core.rs +++ b/core/services/redb/src/core.rs @@ -56,6 +56,26 @@ impl RedbCore { Ok(result.map(Buffer::from)) } + /// Get the byte length of the value for the given key without copying + /// the value data. Uses `AccessGuard::value().len()` which reads the + /// length from the B-tree record header without allocating. + pub fn get_length(&self, path: &str) -> Result<Option<usize>> { + let read_txn = self.db.begin_read().map_err(parse_transaction_error)?; + + let table_define: redb::TableDefinition<&str, &[u8]> = + redb::TableDefinition::new(&self.table); + + let table = read_txn + .open_table(table_define) + .map_err(parse_table_error)?; + + match table.get(path) { + Ok(Some(v)) => Ok(Some(v.value().len())), + Ok(None) => Ok(None), + Err(e) => Err(parse_storage_error(e)), + } + } + pub fn set(&self, path: &str, value: Buffer) -> Result<()> { let write_txn = self.db.begin_write().map_err(parse_transaction_error)?; diff --git a/core/services/rocksdb/src/backend.rs b/core/services/rocksdb/src/backend.rs index a03eb4ff7..88dc813c7 100644 --- a/core/services/rocksdb/src/backend.rs +++ b/core/services/rocksdb/src/backend.rs @@ -150,10 +150,9 @@ impl Service for RocksdbBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p)?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p)? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in rocksdb")), } diff --git a/core/services/rocksdb/src/core.rs b/core/services/rocksdb/src/core.rs index 15ef21dc8..573105c98 100644 --- a/core/services/rocksdb/src/core.rs +++ b/core/services/rocksdb/src/core.rs @@ -41,6 +41,14 @@ impl RocksdbCore { Ok(result.map(Buffer::from)) } + /// Get the byte length of the value for the given key without copying + /// the value into a new allocation. Uses `get_pinned` which returns a + /// `PinnableSlice` that borrows from rocksdb's internal buffer. + pub fn get_length(&self, path: &str) -> Result<Option<usize>> { + let result = self.db.get_pinned(path).map_err(parse_rocksdb_error)?; + Ok(result.as_ref().map(|v| v.len())) + } + pub fn set(&self, path: &str, value: Buffer) -> Result<()> { self.db .put(path, value.to_vec()) diff --git a/core/services/sled/src/backend.rs b/core/services/sled/src/backend.rs index 79353ce47..93c414909 100644 --- a/core/services/sled/src/backend.rs +++ b/core/services/sled/src/backend.rs @@ -174,10 +174,9 @@ impl Service for SledBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p)?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p)? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in sled")), } diff --git a/core/services/sled/src/core.rs b/core/services/sled/src/core.rs index 49c9fe610..0c14c81e0 100644 --- a/core/services/sled/src/core.rs +++ b/core/services/sled/src/core.rs @@ -39,6 +39,14 @@ impl SledCore { Ok(res.map(|v| Buffer::from(v.to_vec()))) } + /// Get the byte length of the value for the given key without copying + /// the value data into a new allocation. Uses `IVec::len()` which + /// returns the length from the inline metadata. + pub fn get_length(&self, path: &str) -> Result<Option<usize>> { + let res = self.tree.get(path).map_err(parse_error)?; + Ok(res.as_ref().map(|v| v.len())) + } + pub fn set(&self, path: &str, value: Buffer) -> Result<()> { self.tree .insert(path, value.to_vec()) diff --git a/core/services/tikv/src/backend.rs b/core/services/tikv/src/backend.rs index 565de1673..311f11cd5 100644 --- a/core/services/tikv/src/backend.rs +++ b/core/services/tikv/src/backend.rs @@ -176,10 +176,9 @@ impl Service for TikvBackend { if p == build_abs_path(&self.root, "") { Ok(RpStat::new(Metadata::new(EntryMode::DIR))) } else { - let bs = self.core.get(&p).await?; - match bs { - Some(bs) => Ok(RpStat::new( - Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64), + match self.core.get_length(&p).await? { + Some(len) => Ok(RpStat::new( + Metadata::new(EntryMode::FILE).with_content_length(len as u64), )), None => Err(Error::new(ErrorKind::NotFound, "kv not found in tikv")), } diff --git a/core/services/tikv/src/core.rs b/core/services/tikv/src/core.rs index 730bd09c3..f32cc8ffb 100644 --- a/core/services/tikv/src/core.rs +++ b/core/services/tikv/src/core.rs @@ -83,6 +83,20 @@ impl TikvCore { Ok(result.map(Buffer::from)) } + /// Get the byte length of the value for the given key without converting + /// the value into a `Buffer`. The full value is still transferred over + /// the network (TiKV protocol limitation), but the `Buffer::from()` + /// allocation is avoided. + pub async fn get_length(&self, path: &str) -> Result<Option<usize>> { + let result = self + .get_connection() + .await? + .get(path.to_owned()) + .await + .map_err(parse_tikv_error)?; + Ok(result.as_ref().map(|v| v.len())) + } + pub async fn set(&self, path: &str, value: Buffer) -> Result<()> { self.get_connection() .await?
