This is an automated email from the ASF dual-hosted git repository.
chitralverma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 6eaf4b638 refactor(bindings/python): split layers into a package for
extensibility (#7878)
6eaf4b638 is described below
commit 6eaf4b638097105dfcbfccf23beb9750b0c06eb4
Author: Chitral Verma <[email protected]>
AuthorDate: Tue Jul 7 17:27:28 2026 +0530
refactor(bindings/python): split layers into a package for extensibility
(#7878)
Split the single layers.rs into a src/layers/ package with one module per
layer plus a mod.rs holding the shared PythonLayer trait, the base Layer,
and
re-exports. The Python surface stays flat on opendal.layers, so all existing
imports are unchanged and stub generation needs no special handling.
Adding a layer later is just: drop src/layers/<name>.rs, register it in
mod.rs and lib.rs, and run just stub-gen.
---
bindings/python/src/layers.rs | 223 ----------------------
bindings/python/src/layers/capability_override.rs | 56 ++++++
bindings/python/src/layers/concurrent_limit.rs | 61 ++++++
bindings/python/src/layers/mime_guess.rs | 60 ++++++
bindings/python/src/layers/mod.rs | 46 +++++
bindings/python/src/layers/retry.rs | 103 ++++++++++
6 files changed, 326 insertions(+), 223 deletions(-)
diff --git a/bindings/python/src/layers.rs b/bindings/python/src/layers.rs
deleted file mode 100644
index f1fc8e190..000000000
--- a/bindings/python/src/layers.rs
+++ /dev/null
@@ -1,223 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::time::Duration;
-
-use crate::*;
-use opendal::Operator;
-
-pub trait PythonLayer: Send + Sync {
- fn layer(&self, op: Operator) -> Operator;
-}
-
-/// Layers are used to intercept the operations on the underlying storage.
-#[pyclass(module = "opendal.layers", subclass)]
-pub struct Layer(pub Box<dyn PythonLayer>);
-
-/// A layer that overrides the full capability exposed by an operator.
-#[pyclass(module = "opendal.layers", extends=Layer, skip_from_py_object)]
-#[derive(Clone)]
-pub struct CapabilityOverrideLayer(ocore::layers::CapabilityOverrideLayer);
-
-impl PythonLayer for CapabilityOverrideLayer {
- fn layer(&self, op: Operator) -> Operator {
- op.layer(self.0.clone())
- }
-}
-#[pymethods]
-impl CapabilityOverrideLayer {
- /// Create a new CapabilityOverrideLayer from capability override entries.
- ///
- /// Parameters
- /// ----------
- /// overrides : str
- /// Comma-separated capability override entries.
- ///
- /// Returns
- /// -------
- /// CapabilityOverrideLayer
- #[new]
- #[pyo3(signature = (overrides))]
- fn new(overrides: &str) -> PyResult<PyClassInitializer<Self>> {
- let layer = Self(
- ocore::layers::CapabilityOverrideLayer::from_overrides(overrides)
- .map_err(format_pyerr)?,
- );
- let class =
PyClassInitializer::from(Layer(Box::new(layer.clone()))).add_subclass(layer);
-
- Ok(class)
- }
-}
-
-/// A layer that retries operations that fail with temporary errors.
-///
-/// Operations are retried if they fail with an error for which
-/// `Error.is_temporary` returns `True`. If all retries are exhausted,
-/// the error is marked as persistent and then returned.
-///
-/// Notes
-/// -----
-/// After an operation on a `Reader` or `Writer` has failed through
-/// all retries, the object is in an undefined state. Reusing it
-/// can lead to exceptions.
-#[pyclass(module = "opendal.layers", extends=Layer, skip_from_py_object)]
-#[derive(Clone)]
-pub struct RetryLayer(ocore::layers::RetryLayer);
-
-impl PythonLayer for RetryLayer {
- fn layer(&self, op: Operator) -> Operator {
- op.layer(self.0.clone())
- }
-}
-#[pymethods]
-impl RetryLayer {
- /// Create a new RetryLayer.
- ///
- /// Parameters
- /// ----------
- /// max_times : Optional[int]
- /// Maximum number of retry attempts. Defaults to ``3``.
- /// factor : Optional[float]
- /// Backoff factor applied between retries. Defaults to ``2.0``.
- /// jitter : bool
- /// Whether to apply jitter to the backoff. Defaults to ``False``.
- /// max_delay : Optional[float]
- /// Maximum delay (in seconds) between retries. Defaults to ``60.0``.
- /// min_delay : Optional[float]
- /// Minimum delay (in seconds) between retries. Defaults to ``1.0``.
- ///
- /// Returns
- /// -------
- /// RetryLayer
- #[new]
- #[pyo3(signature = (
- max_times = None,
- factor = None,
- jitter = false,
- max_delay = None,
- min_delay = None
- ))]
- fn new(
- max_times: Option<usize>,
- factor: Option<f32>,
- jitter: bool,
- max_delay: Option<f64>,
- min_delay: Option<f64>,
- ) -> PyResult<PyClassInitializer<Self>> {
- let mut retry = ocore::layers::RetryLayer::default();
- if let Some(max_times) = max_times {
- retry = retry.with_max_times(max_times);
- }
- if let Some(factor) = factor {
- retry = retry.with_factor(factor);
- }
- if jitter {
- retry = retry.with_jitter();
- }
- if let Some(max_delay) = max_delay {
- retry = retry.with_max_delay(Duration::from_micros((max_delay *
1_000_000.0) as u64));
- }
- if let Some(min_delay) = min_delay {
- retry = retry.with_min_delay(Duration::from_micros((min_delay *
1_000_000.0) as u64));
- }
-
- let retry_layer = Self(retry);
- let class =
PyClassInitializer::from(Layer(Box::new(retry_layer.clone())))
- .add_subclass(retry_layer);
-
- Ok(class)
- }
-}
-
-/// A layer that limits the number of concurrent operations.
-///
-/// Notes
-/// -----
-/// All operators wrapped by this layer will share a common semaphore. This
-/// allows you to reuse the same layer across multiple operators, ensuring
-/// that the total number of concurrent requests across the entire
-/// application does not exceed the limit.
-#[pyclass(module = "opendal.layers", extends = Layer, skip_from_py_object)]
-#[derive(Clone)]
-pub struct ConcurrentLimitLayer(ocore::layers::ConcurrentLimitLayer);
-
-impl PythonLayer for ConcurrentLimitLayer {
- fn layer(&self, op: Operator) -> Operator {
- op.layer(self.0.clone())
- }
-}
-#[pymethods]
-impl ConcurrentLimitLayer {
- /// Create a new ConcurrentLimitLayer.
- ///
- /// Parameters
- /// ----------
- /// limit : int
- /// Maximum number of concurrent operations allowed.
- ///
- /// Returns
- /// -------
- /// ConcurrentLimitLayer
- #[new]
- #[pyo3(signature = (limit))]
- fn new(limit: usize) -> PyResult<PyClassInitializer<Self>> {
- let concurrent_limit =
Self(ocore::layers::ConcurrentLimitLayer::new(limit));
- let class =
PyClassInitializer::from(Layer(Box::new(concurrent_limit.clone())))
- .add_subclass(concurrent_limit);
-
- Ok(class)
- }
-}
-
-/// A layer that guesses MIME types for objects based on their paths or
content.
-///
-/// This layer uses the `mime_guess` crate
-/// (see https://crates.io/crates/mime_guess) to infer the
-/// ``Content-Type``.
-///
-/// Notes
-/// -----
-/// This layer will not override a ``Content-Type`` that has already
-/// been set, either manually or by the backend service. It is only
-/// applied if no content type is present.
-///
-/// A ``Content-Type`` is not guaranteed. If the file extension is
-/// uncommon or unknown, the content type will remain unset.
-#[pyclass(module = "opendal.layers", extends = Layer, skip_from_py_object)]
-#[derive(Clone)]
-pub struct MimeGuessLayer(ocore::layers::MimeGuessLayer);
-
-impl PythonLayer for MimeGuessLayer {
- fn layer(&self, op: Operator) -> Operator {
- op.layer(self.0.clone())
- }
-}
-#[pymethods]
-impl MimeGuessLayer {
- /// Create a new MimeGuessLayer.
- ///
- /// Returns
- /// -------
- /// MimeGuessLayer
- #[new]
- fn new() -> PyResult<PyClassInitializer<Self>> {
- let mime_guess = Self(ocore::layers::MimeGuessLayer::default());
- let class =
-
PyClassInitializer::from(Layer(Box::new(mime_guess.clone()))).add_subclass(mime_guess);
- Ok(class)
- }
-}
diff --git a/bindings/python/src/layers/capability_override.rs
b/bindings/python/src/layers/capability_override.rs
new file mode 100644
index 000000000..8a7b1a90a
--- /dev/null
+++ b/bindings/python/src/layers/capability_override.rs
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::Layer;
+use super::PythonLayer;
+use crate::*;
+use opendal::Operator;
+
+/// A layer that overrides the full capability exposed by an operator.
+#[pyclass(module = "opendal.layers", extends=Layer, skip_from_py_object)]
+#[derive(Clone)]
+pub struct CapabilityOverrideLayer(ocore::layers::CapabilityOverrideLayer);
+
+impl PythonLayer for CapabilityOverrideLayer {
+ fn layer(&self, op: Operator) -> Operator {
+ op.layer(self.0.clone())
+ }
+}
+#[pymethods]
+impl CapabilityOverrideLayer {
+ /// Create a new CapabilityOverrideLayer from capability override entries.
+ ///
+ /// Parameters
+ /// ----------
+ /// overrides : str
+ /// Comma-separated capability override entries.
+ ///
+ /// Returns
+ /// -------
+ /// CapabilityOverrideLayer
+ #[new]
+ #[pyo3(signature = (overrides))]
+ fn new(overrides: &str) -> PyResult<PyClassInitializer<Self>> {
+ let layer = Self(
+ ocore::layers::CapabilityOverrideLayer::from_overrides(overrides)
+ .map_err(format_pyerr)?,
+ );
+ let class =
PyClassInitializer::from(Layer(Box::new(layer.clone()))).add_subclass(layer);
+
+ Ok(class)
+ }
+}
diff --git a/bindings/python/src/layers/concurrent_limit.rs
b/bindings/python/src/layers/concurrent_limit.rs
new file mode 100644
index 000000000..bbdfc9de9
--- /dev/null
+++ b/bindings/python/src/layers/concurrent_limit.rs
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::Layer;
+use super::PythonLayer;
+use crate::*;
+use opendal::Operator;
+
+/// A layer that limits the number of concurrent operations.
+///
+/// Notes
+/// -----
+/// All operators wrapped by this layer will share a common semaphore. This
+/// allows you to reuse the same layer across multiple operators, ensuring
+/// that the total number of concurrent requests across the entire
+/// application does not exceed the limit.
+#[pyclass(module = "opendal.layers", extends = Layer, skip_from_py_object)]
+#[derive(Clone)]
+pub struct ConcurrentLimitLayer(ocore::layers::ConcurrentLimitLayer);
+
+impl PythonLayer for ConcurrentLimitLayer {
+ fn layer(&self, op: Operator) -> Operator {
+ op.layer(self.0.clone())
+ }
+}
+#[pymethods]
+impl ConcurrentLimitLayer {
+ /// Create a new ConcurrentLimitLayer.
+ ///
+ /// Parameters
+ /// ----------
+ /// limit : int
+ /// Maximum number of concurrent operations allowed.
+ ///
+ /// Returns
+ /// -------
+ /// ConcurrentLimitLayer
+ #[new]
+ #[pyo3(signature = (limit))]
+ fn new(limit: usize) -> PyResult<PyClassInitializer<Self>> {
+ let concurrent_limit =
Self(ocore::layers::ConcurrentLimitLayer::new(limit));
+ let class =
PyClassInitializer::from(Layer(Box::new(concurrent_limit.clone())))
+ .add_subclass(concurrent_limit);
+
+ Ok(class)
+ }
+}
diff --git a/bindings/python/src/layers/mime_guess.rs
b/bindings/python/src/layers/mime_guess.rs
new file mode 100644
index 000000000..af99855e2
--- /dev/null
+++ b/bindings/python/src/layers/mime_guess.rs
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::Layer;
+use super::PythonLayer;
+use crate::*;
+use opendal::Operator;
+
+/// A layer that guesses MIME types for objects based on their paths or
content.
+///
+/// This layer uses the `mime_guess` crate
+/// (see https://crates.io/crates/mime_guess) to infer the
+/// ``Content-Type``.
+///
+/// Notes
+/// -----
+/// This layer will not override a ``Content-Type`` that has already
+/// been set, either manually or by the backend service. It is only
+/// applied if no content type is present.
+///
+/// A ``Content-Type`` is not guaranteed. If the file extension is
+/// uncommon or unknown, the content type will remain unset.
+#[pyclass(module = "opendal.layers", extends = Layer, skip_from_py_object)]
+#[derive(Clone)]
+pub struct MimeGuessLayer(ocore::layers::MimeGuessLayer);
+
+impl PythonLayer for MimeGuessLayer {
+ fn layer(&self, op: Operator) -> Operator {
+ op.layer(self.0.clone())
+ }
+}
+#[pymethods]
+impl MimeGuessLayer {
+ /// Create a new MimeGuessLayer.
+ ///
+ /// Returns
+ /// -------
+ /// MimeGuessLayer
+ #[new]
+ fn new() -> PyResult<PyClassInitializer<Self>> {
+ let mime_guess = Self(ocore::layers::MimeGuessLayer::default());
+ let class =
+
PyClassInitializer::from(Layer(Box::new(mime_guess.clone()))).add_subclass(mime_guess);
+ Ok(class)
+ }
+}
diff --git a/bindings/python/src/layers/mod.rs
b/bindings/python/src/layers/mod.rs
new file mode 100644
index 000000000..419fd07ab
--- /dev/null
+++ b/bindings/python/src/layers/mod.rs
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Python layer bindings.
+//!
+//! Each layer lives in its own module so new layers can be added without
growing
+//! a single file. The concrete types are re-exported here (so paths such as
+//! `crate::RetryLayer` keep working) and registered on the flat
`opendal.layers`
+//! Python module.
+
+mod capability_override;
+mod concurrent_limit;
+mod mime_guess;
+mod retry;
+
+pub use capability_override::CapabilityOverrideLayer;
+pub use concurrent_limit::ConcurrentLimitLayer;
+pub use mime_guess::MimeGuessLayer;
+pub use retry::RetryLayer;
+
+use opendal::Operator;
+use pyo3::prelude::*;
+
+/// Shared behavior for every Python layer: wrap an [`Operator`] with the
+/// underlying core layer.
+pub trait PythonLayer: Send + Sync {
+ fn layer(&self, op: Operator) -> Operator;
+}
+
+/// Layers are used to intercept the operations on the underlying storage.
+#[pyclass(module = "opendal.layers", subclass)]
+pub struct Layer(pub Box<dyn PythonLayer>);
diff --git a/bindings/python/src/layers/retry.rs
b/bindings/python/src/layers/retry.rs
new file mode 100644
index 000000000..2376fd0c7
--- /dev/null
+++ b/bindings/python/src/layers/retry.rs
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::time::Duration;
+
+use super::Layer;
+use super::PythonLayer;
+use crate::*;
+use opendal::Operator;
+
+/// A layer that retries operations that fail with temporary errors.
+///
+/// Operations are retried if they fail with an error for which
+/// `Error.is_temporary` returns `True`. If all retries are exhausted,
+/// the error is marked as persistent and then returned.
+///
+/// Notes
+/// -----
+/// After an operation on a `Reader` or `Writer` has failed through
+/// all retries, the object is in an undefined state. Reusing it
+/// can lead to exceptions.
+#[pyclass(module = "opendal.layers", extends=Layer, skip_from_py_object)]
+#[derive(Clone)]
+pub struct RetryLayer(ocore::layers::RetryLayer);
+
+impl PythonLayer for RetryLayer {
+ fn layer(&self, op: Operator) -> Operator {
+ op.layer(self.0.clone())
+ }
+}
+#[pymethods]
+impl RetryLayer {
+ /// Create a new RetryLayer.
+ ///
+ /// Parameters
+ /// ----------
+ /// max_times : Optional[int]
+ /// Maximum number of retry attempts. Defaults to ``3``.
+ /// factor : Optional[float]
+ /// Backoff factor applied between retries. Defaults to ``2.0``.
+ /// jitter : bool
+ /// Whether to apply jitter to the backoff. Defaults to ``False``.
+ /// max_delay : Optional[float]
+ /// Maximum delay (in seconds) between retries. Defaults to ``60.0``.
+ /// min_delay : Optional[float]
+ /// Minimum delay (in seconds) between retries. Defaults to ``1.0``.
+ ///
+ /// Returns
+ /// -------
+ /// RetryLayer
+ #[new]
+ #[pyo3(signature = (
+ max_times = None,
+ factor = None,
+ jitter = false,
+ max_delay = None,
+ min_delay = None
+ ))]
+ fn new(
+ max_times: Option<usize>,
+ factor: Option<f32>,
+ jitter: bool,
+ max_delay: Option<f64>,
+ min_delay: Option<f64>,
+ ) -> PyResult<PyClassInitializer<Self>> {
+ let mut retry = ocore::layers::RetryLayer::default();
+ if let Some(max_times) = max_times {
+ retry = retry.with_max_times(max_times);
+ }
+ if let Some(factor) = factor {
+ retry = retry.with_factor(factor);
+ }
+ if jitter {
+ retry = retry.with_jitter();
+ }
+ if let Some(max_delay) = max_delay {
+ retry = retry.with_max_delay(Duration::from_micros((max_delay *
1_000_000.0) as u64));
+ }
+ if let Some(min_delay) = min_delay {
+ retry = retry.with_min_delay(Duration::from_micros((min_delay *
1_000_000.0) as u64));
+ }
+
+ let retry_layer = Self(retry);
+ let class =
PyClassInitializer::from(Layer(Box::new(retry_layer.clone())))
+ .add_subclass(retry_layer);
+
+ Ok(class)
+ }
+}
