This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch fix-cap in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit ad4e8e3c7827085ecf6d961b6a81ceeaa6ef976a Author: Xuanwo <[email protected]> AuthorDate: Sat Aug 26 16:32:31 2023 +0800 Save work Signed-off-by: Xuanwo <[email protected]> --- core/benches/ops/write.rs | 37 ++++++++++++++++++++++ core/src/layers/complete.rs | 4 +-- core/src/layers/madsim.rs | 2 +- core/src/layers/retry.rs | 2 +- core/src/raw/accessor.rs | 18 +++++++---- core/src/raw/adapters/kv/api.rs | 2 +- .../raw/oio/read/into_seekable_read_by_range.rs | 2 +- core/src/services/azblob/backend.rs | 2 +- core/src/services/azdfs/backend.rs | 2 +- core/src/services/cos/backend.rs | 2 +- core/src/services/dropbox/backend.rs | 2 +- core/src/services/fs/backend.rs | 2 +- core/src/services/ftp/backend.rs | 2 +- core/src/services/gcs/backend.rs | 2 +- core/src/services/gdrive/backend.rs | 2 +- core/src/services/ghac/backend.rs | 2 +- core/src/services/hdfs/backend.rs | 2 +- core/src/services/http/backend.rs | 2 +- core/src/services/ipfs/backend.rs | 2 +- core/src/services/ipmfs/backend.rs | 2 +- core/src/services/obs/backend.rs | 2 +- core/src/services/onedrive/backend.rs | 2 +- core/src/services/oss/backend.rs | 2 +- core/src/services/s3/backend.rs | 2 +- core/src/services/sftp/backend.rs | 2 +- core/src/services/supabase/backend.rs | 2 +- core/src/services/vercel_artifacts/backend.rs | 2 +- core/src/services/wasabi/backend.rs | 2 +- core/src/services/webdav/backend.rs | 2 +- core/src/services/webhdfs/backend.rs | 2 +- core/src/types/operator/operator.rs | 2 +- 31 files changed, 79 insertions(+), 36 deletions(-) diff --git a/core/benches/ops/write.rs b/core/benches/ops/write.rs index 8b56f9fc6..f178eedf9 100644 --- a/core/benches/ops/write.rs +++ b/core/benches/ops/write.rs @@ -32,6 +32,7 @@ pub fn bench(c: &mut Criterion) { let op = case.1.unwrap(); bench_write_once(c, case.0, op.clone()); + bench_read_lazy(c, case.0, op.clone()); } } @@ -66,3 +67,39 @@ fn bench_write_once(c: &mut Criterion, name: &str, op: Operator) { group.finish() } + +fn bench_read_lazy(c: &mut Criterion, name: &str, op: Operator) { + let mut group = c.benchmark_group(format!("service_{name}_read_lazy")); + + let mut rng = thread_rng(); + + let size = Size::from_mebibytes(16); + let content = gen_bytes(&mut rng, size.bytes() as usize); + let path = uuid::Uuid::new_v4().to_string(); + let temp_data = TempData::generate(op.clone(), &path, content.clone()); + + group.bench_with_input("read", &(op.clone(), &path), |b, (op, path)| { + b.to_async(&*TOKIO).iter(|| async { + op.reader(path).await.unwrap(); + }) + }); + group.bench_with_input("reader_zero", &(op.clone(), &path), |b, (op, path)| { + b.to_async(&*TOKIO).iter(|| async { + op.reader_with(path).range(0..1).await.unwrap(); + }) + }); + group.bench_with_input("read_zero", &(op.clone(), &path), |b, (op, path)| { + b.to_async(&*TOKIO).iter(|| async { + op.read_with(path).range(0..1).await.unwrap(); + }) + }); + group.bench_with_input("stat", &(op.clone(), &path), |b, (op, path)| { + b.to_async(&*TOKIO).iter(|| async { + op.stat(path).await.unwrap(); + }) + }); + + std::mem::drop(temp_data); + + group.finish() +} diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs index f95ba0851..4c58b6e30 100644 --- a/core/src/layers/complete.rs +++ b/core/src/layers/complete.rs @@ -156,7 +156,7 @@ impl<A: Accessor> CompleteReaderAccessor<A> { path: &str, args: OpRead, ) -> Result<(RpRead, CompleteReader<A, A::Reader>)> { - let capability = self.meta.full_capability(); + let capability = self.meta.native_capability(); if !capability.read { return new_capability_unsupported_error(Operation::Read); } @@ -899,7 +899,7 @@ mod tests { fn info(&self) -> AccessorInfo { let mut info = AccessorInfo::default(); - info.set_full_capability(self.capability); + info.set_native_capability(self.capability); info } diff --git a/core/src/layers/madsim.rs b/core/src/layers/madsim.rs index d7d483989..fdf0ec5de 100644 --- a/core/src/layers/madsim.rs +++ b/core/src/layers/madsim.rs @@ -160,7 +160,7 @@ impl LayeredAccessor for MadsimAccessor { let mut info = AccessorInfo::default(); info.set_name("madsim"); - info.set_full_capability(Capability { + info.set_native_capability(Capability { read: true, write: true, ..Default::default() diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index bd56fa42c..7517c2c21 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -1151,7 +1151,7 @@ mod tests { fn info(&self) -> AccessorInfo { let mut am = AccessorInfo::default(); - am.set_full_capability(Capability { + am.set_native_capability(Capability { read: true, list: true, list_with_delimiter_slash: true, diff --git a/core/src/raw/accessor.rs b/core/src/raw/accessor.rs index 9c8905482..6fed93b58 100644 --- a/core/src/raw/accessor.rs +++ b/core/src/raw/accessor.rs @@ -534,6 +534,18 @@ impl AccessorInfo { self.native_capability } + /// Set native capabilities for service. + /// + /// # NOTES + /// + /// Set native capability will also flush the full capability. The only way to change + /// full_capability is via `full_capability_mut`. + pub fn set_native_capability(&mut self, capability: Capability) -> &mut Self { + self.native_capability = capability; + self.full_capability = capability; + self + } + /// Get service's full capabilities. pub fn full_capability(&self) -> Capability { self.full_capability @@ -543,10 +555,4 @@ impl AccessorInfo { pub fn full_capability_mut(&mut self) -> &mut Capability { &mut self.full_capability } - - /// Set full capabilities for service. - pub fn set_full_capability(&mut self, capability: Capability) -> &mut Self { - self.full_capability = capability; - self - } } diff --git a/core/src/raw/adapters/kv/api.rs b/core/src/raw/adapters/kv/api.rs index 7ae2fe97e..9adaabae6 100644 --- a/core/src/raw/adapters/kv/api.rs +++ b/core/src/raw/adapters/kv/api.rs @@ -169,7 +169,7 @@ impl From<Metadata> for AccessorInfo { let mut am = AccessorInfo::default(); am.set_name(m.name()); am.set_scheme(m.scheme()); - am.set_full_capability(m.capabilities()); + am.set_native_capability(m.capabilities()); am } diff --git a/core/src/raw/oio/read/into_seekable_read_by_range.rs b/core/src/raw/oio/read/into_seekable_read_by_range.rs index 052b300ec..9f623d010 100644 --- a/core/src/raw/oio/read/into_seekable_read_by_range.rs +++ b/core/src/raw/oio/read/into_seekable_read_by_range.rs @@ -420,7 +420,7 @@ mod tests { fn info(&self) -> AccessorInfo { let mut am = AccessorInfo::default(); - am.set_full_capability(Capability { + am.set_native_capability(Capability { read: true, ..Default::default() }); diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index e773d4199..fbd480de6 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -516,7 +516,7 @@ impl Accessor for AzblobBackend { am.set_scheme(Scheme::Azblob) .set_root(&self.core.root) .set_name(&self.core.container) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/azdfs/backend.rs b/core/src/services/azdfs/backend.rs index 0d567266b..3882f4fe2 100644 --- a/core/src/services/azdfs/backend.rs +++ b/core/src/services/azdfs/backend.rs @@ -240,7 +240,7 @@ impl Accessor for AzdfsBackend { am.set_scheme(Scheme::Azdfs) .set_root(&self.core.root) .set_name(&self.core.filesystem) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 66aa65449..3360d9dd5 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -259,7 +259,7 @@ impl Accessor for CosBackend { am.set_scheme(Scheme::Cos) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/dropbox/backend.rs b/core/src/services/dropbox/backend.rs index e24f0b36f..ebb468728 100644 --- a/core/src/services/dropbox/backend.rs +++ b/core/src/services/dropbox/backend.rs @@ -58,7 +58,7 @@ impl Accessor for DropboxBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::Dropbox) .set_root(&self.core.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index b8fac1813..f94f59d96 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -254,7 +254,7 @@ impl Accessor for FsBackend { let mut am = AccessorInfo::default(); am.set_scheme(Scheme::Fs) .set_root(&self.root.to_string_lossy()) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index 0c9d27557..ed960c5f7 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -273,7 +273,7 @@ impl Accessor for FtpBackend { let mut am = AccessorInfo::default(); am.set_scheme(Scheme::Ftp) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index 27140c78f..02a88a8ac 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -347,7 +347,7 @@ impl Accessor for GcsBackend { am.set_scheme(Scheme::Gcs) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { create_dir: true, stat: true, diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index 246e32f33..fa270b1e8 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -49,7 +49,7 @@ impl Accessor for GdriveBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::Gdrive) .set_root(&self.core.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/ghac/backend.rs b/core/src/services/ghac/backend.rs index 396b14000..8adce69a3 100644 --- a/core/src/services/ghac/backend.rs +++ b/core/src/services/ghac/backend.rs @@ -303,7 +303,7 @@ impl Accessor for GhacBackend { am.set_scheme(Scheme::Ghac) .set_root(&self.root) .set_name(&self.version) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs index 0888589c7..a6078c89e 100644 --- a/core/src/services/hdfs/backend.rs +++ b/core/src/services/hdfs/backend.rs @@ -171,7 +171,7 @@ impl Accessor for HdfsBackend { let mut am = AccessorInfo::default(); am.set_scheme(Scheme::Hdfs) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 25f06f993..da29b3ca8 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -262,7 +262,7 @@ impl Accessor for HttpBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::Http) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs index 7fd9b4eab..ebf1e327e 100644 --- a/core/src/services/ipfs/backend.rs +++ b/core/src/services/ipfs/backend.rs @@ -172,7 +172,7 @@ impl Accessor for IpfsBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::Ipfs) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/ipmfs/backend.rs b/core/src/services/ipmfs/backend.rs index 95163f2c2..aebab57e5 100644 --- a/core/src/services/ipmfs/backend.rs +++ b/core/src/services/ipmfs/backend.rs @@ -74,7 +74,7 @@ impl Accessor for IpmfsBackend { let mut am = AccessorInfo::default(); am.set_scheme(Scheme::Ipmfs) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 78ecf0976..cb43a41c1 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -322,7 +322,7 @@ impl Accessor for ObsBackend { am.set_scheme(Scheme::Obs) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 5590a7a37..09ef04765 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -73,7 +73,7 @@ impl Accessor for OnedriveBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::Onedrive) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { read: true, write: true, stat: true, diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 1aef92b77..ee1131ee1 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -391,7 +391,7 @@ impl Accessor for OssBackend { am.set_scheme(Scheme::Oss) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index db8c4db00..d80e5d71d 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -902,7 +902,7 @@ impl Accessor for S3Backend { am.set_scheme(Scheme::S3) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index 609496012..f594b2d38 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -232,7 +232,7 @@ impl Accessor for SftpBackend { let mut am = AccessorInfo::default(); am.set_root(self.root.as_str()) .set_scheme(Scheme::Sftp) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs index 3872623c2..15c0354dd 100644 --- a/core/src/services/supabase/backend.rs +++ b/core/src/services/supabase/backend.rs @@ -217,7 +217,7 @@ impl Accessor for SupabaseBackend { am.set_scheme(Scheme::Supabase) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/vercel_artifacts/backend.rs b/core/src/services/vercel_artifacts/backend.rs index 34501827b..70cd9048b 100644 --- a/core/src/services/vercel_artifacts/backend.rs +++ b/core/src/services/vercel_artifacts/backend.rs @@ -54,7 +54,7 @@ impl Accessor for VercelArtifactsBackend { fn info(&self) -> AccessorInfo { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::VercelArtifacts) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/wasabi/backend.rs b/core/src/services/wasabi/backend.rs index a8a8da91c..28969898a 100644 --- a/core/src/services/wasabi/backend.rs +++ b/core/src/services/wasabi/backend.rs @@ -901,7 +901,7 @@ impl Accessor for WasabiBackend { am.set_scheme(Scheme::Wasabi) .set_root(&self.core.root) .set_name(&self.core.bucket) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, stat_with_if_match: true, stat_with_if_none_match: true, diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index b4fef1db5..752af5f75 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -281,7 +281,7 @@ impl Accessor for WebdavBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(Scheme::Webdav) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs index 4167e1aee..db8e0490a 100644 --- a/core/src/services/webhdfs/backend.rs +++ b/core/src/services/webhdfs/backend.rs @@ -408,7 +408,7 @@ impl Accessor for WebhdfsBackend { let mut am = AccessorInfo::default(); am.set_scheme(Scheme::Webhdfs) .set_root(&self.root) - .set_full_capability(Capability { + .set_native_capability(Capability { stat: true, read: true, diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index b95f4b78d..b2c6e815c 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -369,7 +369,7 @@ impl Operator { let br = args.range(); let (rp, mut s) = inner.read(&path, args).await?; - let length = rp.into_metadata().content_length() as usize; + let length = rp.into_metadata().contentngth() as usize; let mut buffer = Vec::with_capacity(length); let dst = buffer.spare_capacity_mut();
