This is an automated email from the ASF dual-hosted git repository. psiace pushed a commit to branch feature/layer-hotpath in repository https://gitbox.apache.org/repos/asf/opendal.git
commit be41b5178cafd1084e8b019bd4338b1874ff1958 Author: Chojan Shang <[email protected]> AuthorDate: Mon Dec 22 18:10:08 2025 +0800 feat(layers/hotpath): add HotpathLayer integration Signed-off-by: Chojan Shang <[email protected]> --- core/Cargo.lock | 133 ++++++++++++++++++++++++++++ core/Cargo.toml | 2 + core/layers/hotpath/Cargo.toml | 38 ++++++++ core/layers/hotpath/src/lib.rs | 193 +++++++++++++++++++++++++++++++++++++++++ core/src/lib.rs | 2 + 5 files changed, 368 insertions(+) diff --git a/core/Cargo.lock b/core/Cargo.lock index c5ca7a3a8..72f4515e7 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -245,6 +245,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -1645,6 +1651,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "ciborium" version = "0.2.2" @@ -2767,6 +2779,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2892,6 +2910,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fail" version = "0.4.0" @@ -3711,6 +3739,16 @@ dependencies = [ "java-locator", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "hdrs" version = "0.3.2" @@ -3851,6 +3889,44 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hotpath" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "daa7e73e5b9dd467b2cbe7cc50ba224cd43bfcc0a618fd75f4adc32cf3a95d66" +dependencies = [ + "arc-swap", + "base64 0.22.1", + "cfg-if", + "clap", + "colored", + "crossbeam-channel", + "eyre", + "futures-util", + "hdrhistogram", + "hotpath-macros", + "libc", + "mach2", + "pin-project-lite", + "prettytable-rs", + "quanta", + "regex", + "serde", + "serde_json", + "tiny_http", +] + +[[package]] +name = "hotpath-macros" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "4aca1659ca1e70edc8f96de48530a104bec11a267f663625098137bfd072d601" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "html5ever" version = "0.35.0" @@ -4215,6 +4291,12 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexmap" version = "1.9.3" @@ -4341,6 +4423,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -4829,6 +4922,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index"; checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "macro_magic" version = "0.5.1" @@ -5542,6 +5641,7 @@ dependencies = [ "opendal-layer-dtrace", "opendal-layer-fastmetrics", "opendal-layer-fastrace", + "opendal-layer-hotpath", "opendal-layer-immutable-index", "opendal-layer-logging", "opendal-layer-metrics", @@ -5804,6 +5904,14 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-layer-hotpath" +version = "0.55.0" +dependencies = [ + "hotpath", + "opendal-core", +] + [[package]] name = "opendal-layer-immutable-index" version = "0.55.0" @@ -7517,6 +7625,19 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "prettytable-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" +dependencies = [ + "encode_unicode", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "probe" version = "0.5.2" @@ -10250,6 +10371,18 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.8.2" diff --git a/core/Cargo.toml b/core/Cargo.toml index 43b061a1e..9a1ab443e 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -99,6 +99,7 @@ layers-concurrent-limit = ["dep:opendal-layer-concurrent-limit"] layers-dtrace = ["dep:opendal-layer-dtrace"] layers-fastmetrics = ["dep:opendal-layer-fastmetrics"] layers-fastrace = ["dep:opendal-layer-fastrace"] +layers-hotpath = ["dep:opendal-layer-hotpath"] layers-immutable-index = ["dep:opendal-layer-immutable-index"] layers-logging = ["dep:opendal-layer-logging"] layers-metrics = ["dep:opendal-layer-metrics"] @@ -206,6 +207,7 @@ opendal-layer-concurrent-limit = { path = "layers/concurrent-limit", version = " opendal-layer-dtrace = { path = "layers/dtrace", version = "0.55.0", optional = true, default-features = false } opendal-layer-fastmetrics = { path = "layers/fastmetrics", version = "0.55.0", optional = true, default-features = false } opendal-layer-fastrace = { path = "layers/fastrace", version = "0.55.0", optional = true, default-features = false } +opendal-layer-hotpath = { path = "layers/hotpath", version = "0.55.0", optional = true, default-features = false } opendal-layer-immutable-index = { path = "layers/immutable-index", version = "0.55.0", optional = true, default-features = false } opendal-layer-logging = { path = "layers/logging", version = "0.55.0", optional = true, default-features = false } opendal-layer-metrics = { path = "layers/metrics", version = "0.55.0", optional = true, default-features = false } diff --git a/core/layers/hotpath/Cargo.toml b/core/layers/hotpath/Cargo.toml new file mode 100644 index 000000000..7e1af4738 --- /dev/null +++ b/core/layers/hotpath/Cargo.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +description = "Apache OpenDAL hotpath layer" +name = "opendal-layer-hotpath" + +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +hotpath = { version = "0.9.1", features = ["hotpath"] } +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } + +[dev-dependencies] +opendal-core = { path = "../../core", version = "0.55.0" } diff --git a/core/layers/hotpath/src/lib.rs b/core/layers/hotpath/src/lib.rs new file mode 100644 index 000000000..f0b9c3d8b --- /dev/null +++ b/core/layers/hotpath/src/lib.rs @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hotpath::MeasurementGuard; +use opendal_core::raw::*; +use opendal_core::*; + +const LABEL_CREATE_DIR: &str = "opendal.create_dir"; +const LABEL_READ: &str = "opendal.read"; +const LABEL_WRITE: &str = "opendal.write"; +const LABEL_COPY: &str = "opendal.copy"; +const LABEL_RENAME: &str = "opendal.rename"; +const LABEL_STAT: &str = "opendal.stat"; +const LABEL_DELETE: &str = "opendal.delete"; +const LABEL_LIST: &str = "opendal.list"; +const LABEL_PRESIGN: &str = "opendal.presign"; + +const LABEL_READER_READ: &str = "opendal.reader.read"; +const LABEL_WRITER_WRITE: &str = "opendal.writer.write"; +const LABEL_WRITER_CLOSE: &str = "opendal.writer.close"; +const LABEL_WRITER_ABORT: &str = "opendal.writer.abort"; +const LABEL_LISTER_NEXT: &str = "opendal.lister.next"; +const LABEL_DELETER_DELETE: &str = "opendal.deleter.delete"; +const LABEL_DELETER_CLOSE: &str = "opendal.deleter.close"; + +/// Add [hotpath](https://docs.rs/hotpath/) profiling for every operation. +/// +/// # Notes +/// +/// When `hotpath` profiling is enabled, initialize a guard via +/// [`hotpath::FunctionsGuardBuilder`] or `#[hotpath::main]` before running +/// operations. Otherwise, hotpath will panic on the first measurement. +/// +/// # Examples +/// +/// ```no_run +/// # use opendal_core::services; +/// # use opendal_core::Operator; +/// # use opendal_core::Result; +/// # use opendal_layer_hotpath::HotpathLayer; +/// # +/// # #[tokio::main] +/// # async fn main() -> Result<()> { +/// let _guard = hotpath::FunctionsGuardBuilder::new("opendal").build(); +/// let op = Operator::new(services::Memory::default())? +/// .layer(HotpathLayer) +/// .finish(); +/// op.write("test", "hello").await?; +/// # Ok(()) +/// # } +/// ``` +pub struct HotpathLayer; + +impl<A: Access> Layer<A> for HotpathLayer { + type LayeredAccess = HotpathAccessor<A>; + + fn layer(&self, inner: A) -> Self::LayeredAccess { + HotpathAccessor { inner } + } +} + +#[derive(Debug)] +pub struct HotpathAccessor<A> { + inner: A, +} + +impl<A: Access> LayeredAccess for HotpathAccessor<A> { + type Inner = A; + type Reader = HotpathWrapper<A::Reader>; + type Writer = HotpathWrapper<A::Writer>; + type Lister = HotpathWrapper<A::Lister>; + type Deleter = HotpathWrapper<A::Deleter>; + + fn inner(&self) -> &Self::Inner { + &self.inner + } + + async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> { + let _guard = MeasurementGuard::build(LABEL_CREATE_DIR, false, true); + self.inner.create_dir(path, args).await + } + + async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { + let _guard = MeasurementGuard::build(LABEL_READ, false, true); + let (rp, reader) = self.inner.read(path, args).await?; + Ok((rp, HotpathWrapper::new(reader))) + } + + async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> { + let _guard = MeasurementGuard::build(LABEL_WRITE, false, true); + let (rp, writer) = self.inner.write(path, args).await?; + Ok((rp, HotpathWrapper::new(writer))) + } + + async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> { + let _guard = MeasurementGuard::build(LABEL_COPY, false, true); + self.inner().copy(from, to, args).await + } + + async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> { + let _guard = MeasurementGuard::build(LABEL_RENAME, false, true); + self.inner().rename(from, to, args).await + } + + async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> { + let _guard = MeasurementGuard::build(LABEL_STAT, false, true); + self.inner.stat(path, args).await + } + + async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> { + let _guard = MeasurementGuard::build(LABEL_DELETE, false, true); + let (rp, deleter) = self.inner.delete().await?; + Ok((rp, HotpathWrapper::new(deleter))) + } + + async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> { + let _guard = MeasurementGuard::build(LABEL_LIST, false, true); + let (rp, lister) = self.inner.list(path, args).await?; + Ok((rp, HotpathWrapper::new(lister))) + } + + async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> { + let _guard = MeasurementGuard::build(LABEL_PRESIGN, false, true); + self.inner.presign(path, args).await + } +} + +pub struct HotpathWrapper<R> { + inner: R, +} + +impl<R> HotpathWrapper<R> { + fn new(inner: R) -> Self { + Self { inner } + } +} + +impl<R: oio::Read> oio::Read for HotpathWrapper<R> { + async fn read(&mut self) -> Result<Buffer> { + let _guard = MeasurementGuard::build(LABEL_READER_READ, false, true); + self.inner.read().await + } +} + +impl<R: oio::Write> oio::Write for HotpathWrapper<R> { + async fn write(&mut self, bs: Buffer) -> Result<()> { + let _guard = MeasurementGuard::build(LABEL_WRITER_WRITE, false, true); + self.inner.write(bs).await + } + + async fn close(&mut self) -> Result<Metadata> { + let _guard = MeasurementGuard::build(LABEL_WRITER_CLOSE, false, true); + self.inner.close().await + } + + async fn abort(&mut self) -> Result<()> { + let _guard = MeasurementGuard::build(LABEL_WRITER_ABORT, false, true); + self.inner.abort().await + } +} + +impl<R: oio::List> oio::List for HotpathWrapper<R> { + async fn next(&mut self) -> Result<Option<oio::Entry>> { + let _guard = MeasurementGuard::build(LABEL_LISTER_NEXT, false, true); + self.inner.next().await + } +} + +impl<R: oio::Delete> oio::Delete for HotpathWrapper<R> { + async fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> { + let _guard = MeasurementGuard::build(LABEL_DELETER_DELETE, false, true); + self.inner.delete(path, args).await + } + + async fn close(&mut self) -> Result<()> { + let _guard = MeasurementGuard::build(LABEL_DELETER_CLOSE, false, true); + self.inner.close().await + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index a17f8f792..b69107c8e 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -167,6 +167,8 @@ pub mod layers { pub use opendal_layer_fastmetrics::*; #[cfg(feature = "layers-fastrace")] pub use opendal_layer_fastrace::*; + #[cfg(feature = "layers-hotpath")] + pub use opendal_layer_hotpath::*; #[cfg(feature = "layers-immutable-index")] pub use opendal_layer_immutable_index::*; #[cfg(feature = "layers-logging")]
