This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 8169efc23 feat(bindings/python): expose presign api (#2950)
8169efc23 is described below
commit 8169efc2348c5e410f0abdbc455d5022cd93b487
Author: Mingzhuo Yin <[email protected]>
AuthorDate: Tue Aug 29 11:29:58 2023 +0800
feat(bindings/python): expose presign api (#2950)
* feat(bindings/python): expose presign api
Signed-off-by: silver-ymz <[email protected]>
* add test for presign
Signed-off-by: silver-ymz <[email protected]>
* fix HeaderMap to return error for invalid value
Signed-off-by: silver-ymz <[email protected]>
* update headers function signature && rename presign test
Signed-off-by: silver-ymz <[email protected]>
* typo
Signed-off-by: silver-ymz <[email protected]>
* update test for presign
Signed-off-by: silver-ymz <[email protected]>
* make clear for expire second
Signed-off-by: silver-ymz <[email protected]>
---------
Signed-off-by: silver-ymz <[email protected]>
---
bindings/python/CONTRIBUTING.md | 25 ++++++++----
bindings/python/python/opendal/__init__.pyi | 11 ++++++
bindings/python/src/asyncio.rs | 59 +++++++++++++++++++++++++++++
bindings/python/src/lib.rs | 38 +++++++++++++++++++
bindings/python/tests/steps/binding.py | 10 +++++
bindings/tests/features/binding.feature | 1 +
6 files changed, 136 insertions(+), 8 deletions(-)
diff --git a/bindings/python/CONTRIBUTING.md b/bindings/python/CONTRIBUTING.md
index 7e6a73a12..cac74fc11 100644
--- a/bindings/python/CONTRIBUTING.md
+++ b/bindings/python/CONTRIBUTING.md
@@ -1,12 +1,13 @@
# Contributing
-- [Setup](#setup)
- - [Using a dev container environment](#using-a-devcontainer-environment)
- - [Bring your own toolbox](#bring-your-own-toolbox)
-- [Prepare](#prepare)
-- [Build](#build)
-- [Test](#test)
-- [Docs](#docs)
+- [Contributing](#contributing)
+ - [Setup](#setup)
+ - [Using a dev container environment](#using-a-dev-container-environment)
+ - [Bring your own toolbox](#bring-your-own-toolbox)
+ - [Prepare](#prepare)
+ - [Build](#build)
+ - [Test](#test)
+ - [Docs](#docs)
## Setup
@@ -50,12 +51,20 @@ pip install maturin[patchelf]
## Build
-To build python binding:
+To build python binding only:
+
+```shell
+maturin build
+```
+
+To build and install python binding directly in the current virtualenv:
```shell
maturin develop
```
+Note: `maturin develop` will be faster, but doesn't support all the features.
In most development cases, we recommend using `maturin develop`.
+
## Test
OpenDAL adopts `behave` for behavior tests:
diff --git a/bindings/python/python/opendal/__init__.pyi
b/bindings/python/python/opendal/__init__.pyi
index 2f82aed6d..2b3501500 100644
--- a/bindings/python/python/opendal/__init__.pyi
+++ b/bindings/python/python/opendal/__init__.pyi
@@ -40,6 +40,9 @@ class AsyncOperator:
async def delete(self, path: str): ...
async def list(self, path: str) -> AsyncIterable[Entry]: ...
async def scan(self, path: str) -> AsyncIterable[Entry]: ...
+ async def presign_stat(self, path: str, expire_second: int) ->
PresignedRequest: ...
+ async def presign_read(self, path: str, expire_second: int) ->
PresignedRequest: ...
+ async def presign_write(self, path: str, expire_second: int) ->
PresignedRequest: ...
class Reader:
def read(self, size: Optional[int] = None) -> bytes: ...
@@ -76,3 +79,11 @@ class Metadata:
class EntryMode:
def is_file(self) -> bool: ...
def is_dir(self) -> bool: ...
+
+class PresignedRequest:
+ @property
+ def url(self) -> str: ...
+ @property
+ def method(self) -> str: ...
+ @property
+ def headers(self) -> dict[str, str]: ...
\ No newline at end of file
diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs
index 4b33015fa..b01ed930d 100644
--- a/bindings/python/src/asyncio.rs
+++ b/bindings/python/src/asyncio.rs
@@ -19,6 +19,7 @@ use std::collections::HashMap;
use std::io::SeekFrom;
use std::str::FromStr;
use std::sync::Arc;
+use std::time::Duration;
use ::opendal as od;
use futures::TryStreamExt;
@@ -39,6 +40,7 @@ use crate::format_pyerr;
use crate::layers;
use crate::Entry;
use crate::Metadata;
+use crate::PresignedRequest;
/// `AsyncOperator` is the entry for all public async APIs
///
@@ -159,6 +161,63 @@ impl AsyncOperator {
})
}
+ /// Presign an operation for stat(head) which expires after
`expire_second` seconds.
+ pub fn presign_stat<'p>(
+ &'p self,
+ py: Python<'p>,
+ path: String,
+ expire_second: u64,
+ ) -> PyResult<&'p PyAny> {
+ let this = self.0.clone();
+ future_into_py(py, async move {
+ let res = this
+ .presign_stat(&path, Duration::from_secs(expire_second))
+ .await
+ .map_err(format_pyerr)
+ .map(PresignedRequest)?;
+
+ Ok(res)
+ })
+ }
+
+ /// Presign an operation for read which expires after `expire_second`
seconds.
+ pub fn presign_read<'p>(
+ &'p self,
+ py: Python<'p>,
+ path: String,
+ expire_second: u64,
+ ) -> PyResult<&'p PyAny> {
+ let this = self.0.clone();
+ future_into_py(py, async move {
+ let res = this
+ .presign_read(&path, Duration::from_secs(expire_second))
+ .await
+ .map_err(format_pyerr)
+ .map(PresignedRequest)?;
+
+ Ok(res)
+ })
+ }
+
+ /// Presign an operation for write which expires after `expire_second`
seconds.
+ pub fn presign_write<'p>(
+ &'p self,
+ py: Python<'p>,
+ path: String,
+ expire_second: u64,
+ ) -> PyResult<&'p PyAny> {
+ let this = self.0.clone();
+ future_into_py(py, async move {
+ let res = this
+ .presign_write(&path, Duration::from_secs(expire_second))
+ .await
+ .map_err(format_pyerr)
+ .map(PresignedRequest)?;
+
+ Ok(res)
+ })
+ }
+
fn __repr__(&self) -> String {
let info = self.0.info();
let name = info.name();
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 81189b3b0..4e3dc11c2 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -27,9 +27,11 @@ use std::str::FromStr;
use ::opendal as od;
use pyo3::create_exception;
use pyo3::exceptions::PyException;
+use pyo3::exceptions::PyFileExistsError;
use pyo3::exceptions::PyFileNotFoundError;
use pyo3::exceptions::PyIOError;
use pyo3::exceptions::PyNotImplementedError;
+use pyo3::exceptions::PyPermissionError;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::PyBytes;
@@ -368,10 +370,45 @@ impl EntryMode {
}
}
+#[pyclass(module = "opendal")]
+struct PresignedRequest(od::raw::PresignedRequest);
+
+#[pymethods]
+impl PresignedRequest {
+ /// Return the URL of this request.
+ #[getter]
+ pub fn url(&self) -> String {
+ self.0.uri().to_string()
+ }
+
+ /// Return the HTTP method of this request.
+ #[getter]
+ pub fn method(&self) -> &str {
+ self.0.method().as_str()
+ }
+
+ /// Return the HTTP headers of this request.
+ #[getter]
+ pub fn headers(&self) -> PyResult<HashMap<&str, &str>> {
+ let mut headers = HashMap::new();
+ for (k, v) in self.0.header().iter() {
+ let k = k.as_str();
+ let v = v.to_str().map_err(|err| Error::new_err(err.to_string()))?;
+ if headers.insert(k, v).is_some() {
+ return Err(Error::new_err("duplicate header"));
+ }
+ }
+ Ok(headers)
+ }
+}
+
fn format_pyerr(err: od::Error) -> PyErr {
use od::ErrorKind::*;
match err.kind() {
NotFound => PyFileNotFoundError::new_err(err.to_string()),
+ AlreadyExists => PyFileExistsError::new_err(err.to_string()),
+ PermissionDenied => PyPermissionError::new_err(err.to_string()),
+ Unsupported => PyNotImplementedError::new_err(err.to_string()),
_ => Error::new_err(err.to_string()),
}
}
@@ -416,6 +453,7 @@ fn _opendal(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Entry>()?;
m.add_class::<EntryMode>()?;
m.add_class::<Metadata>()?;
+ m.add_class::<PresignedRequest>()?;
m.add("Error", py.get_type::<Error>())?;
let layers = layers::create_submodule(py)?;
diff --git a/bindings/python/tests/steps/binding.py
b/bindings/python/tests/steps/binding.py
index 4d343eaaf..06edb9f7f 100644
--- a/bindings/python/tests/steps/binding.py
+++ b/bindings/python/tests/steps/binding.py
@@ -88,3 +88,13 @@ async def step_impl(context, filename, size):
async def step_impl(context, filename, content):
bs = await context.op.read(filename)
assert bs == content.encode()
+
+@then("The presign operation should success or raise exception Unsupported")
+@async_run_until_complete
+async def step_impl(context):
+ try:
+ await context.op.presign_stat("test.txt", 10)
+ await context.op.presign_read("test.txt", 10)
+ await context.op.presign_write("test.txt", 10)
+ except NotImplementedError:
+ pass
\ No newline at end of file
diff --git a/bindings/tests/features/binding.feature
b/bindings/tests/features/binding.feature
index ad6832c09..f9f02a7a7 100644
--- a/bindings/tests/features/binding.feature
+++ b/bindings/tests/features/binding.feature
@@ -32,3 +32,4 @@ Feature: OpenDAL Binding
Then The async file "test" entry mode must be file
Then The async file "test" content length must be 13
Then The async file "test" must have content "Hello, World!"
+ Then The presign operation should success or raise exception
Unsupported
