This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new cdb9871d1 feat(bindings/python): Enhance Stat, Lister, Metadata &
Entry (#6232)
cdb9871d1 is described below
commit cdb9871d16117b7b0645b7e31011ee350c9895e2
Author: Chitral Verma <chitralve...@gmail.com>
AuthorDate: Mon Jun 9 15:50:56 2025 +0530
feat(bindings/python): Enhance Stat, Lister, Metadata & Entry (#6232)
* enhance metadata in lister for fs service
* updates to entry and metadata
* allow all options with list and scan
* clippy
* clippy
* add deprecation warning to scan
* fix docs for sync/ async list and scan
* fix formatting
* fix formatting
* add stat options
* use deprecated decorator
* use deprecated decorator
* Update bindings/python/python/opendal/__init__.pyi
Co-authored-by: Nadeshiko Manju <lizheao940...@gmail.com>
* fix formatting
* Reset lister.rs
---------
Co-authored-by: Asuka Minato <i...@asukaminato.eu.org>
Co-authored-by: Nadeshiko Manju <lizheao940...@gmail.com>
---
bindings/python/python/opendal/__init__.pyi | 258 +++++++++++++++++------
bindings/python/python/opendal/exceptions.pyi | 20 +-
bindings/python/src/lib.rs | 2 +
bindings/python/src/metadata.rs | 78 +++++--
bindings/python/src/operator.rs | 113 ++++++----
bindings/python/src/options.rs | 50 +++++
bindings/python/tests/test_async_pickle_types.py | 5 +-
bindings/python/tests/test_pickle_rw.py | 6 +-
bindings/python/tests/test_sync_pickle_types.py | 5 +-
bindings/python/tests/test_write.py | 26 ---
10 files changed, 392 insertions(+), 171 deletions(-)
diff --git a/bindings/python/python/opendal/__init__.pyi
b/bindings/python/python/opendal/__init__.pyi
index 994c91c65..cd7e24c80 100644
--- a/bindings/python/python/opendal/__init__.pyi
+++ b/bindings/python/python/opendal/__init__.pyi
@@ -17,9 +17,14 @@
import os
from collections.abc import AsyncIterable, Iterable
+from datetime import datetime
from types import TracebackType
from typing import Any, Union, final
+try:
+ from warnings import deprecated
+except ImportError:
+ from typing_extensions import deprecated
from opendal import exceptions as exceptions
from opendal import layers as layers
from opendal.__base import _Base
@@ -44,6 +49,7 @@ class Operator(_Base):
op.write("hello.txt", b"hello world")
```
"""
+
def __init__(self, scheme: str, **options: Any) -> None: ...
def layer(self, layer: Layer) -> Operator:
"""Add new layers upon the current operator.
@@ -51,7 +57,8 @@ class Operator(_Base):
Args:
layer (Layer): The layer to be added.
- Returns:
+ Returns
+ -------
The new operator with the layer added.
"""
def open(self, path: PathBuf, mode: str, **options: Any) -> File:
@@ -68,7 +75,8 @@ class Operator(_Base):
- If `mode == "wb"`: options match the
[OpenDAL
`WriteOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.WriteOptions.html).
- Returns:
+ Returns
+ -------
File: A file-like object that can be used to read or write the
file.
Example:
@@ -108,7 +116,8 @@ class Operator(_Base):
- if_unmodified_since (datetime): Only read if the object was
not
modified since this timestamp. This timestamp must be in
UTC.
- Returns:
+ Returns
+ -------
bytes: The content of the object as bytes.
"""
def write(self, path: PathBuf, bs: bytes, **options: Any) -> None:
@@ -139,17 +148,35 @@ class Operator(_Base):
- user_metadata (dict[str, str]): Custom user metadata to
associate
with the object.
- Returns:
+ Returns
+ -------
None
"""
- def stat(self, path: PathBuf) -> Metadata:
+ def stat(self, path: PathBuf, **kwargs) -> Metadata:
"""Get the metadata of the object at the given path.
Args:
- path (str|Path): The path to the object.
+ path (str | Path): The path to the object.
+ **kwargs (Any): Optional stat parameters matching the
+ [OpenDAL
`StatOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.StatOptions.html):
- Returns:
- The metadata of the object.
+ - version (str): Specify the version of the object to read, if
+ supported by the backend.
+ - if_match (str): Read only if the ETag matches the given
value.
+ - if_none_match (str): Read-only if the ETag does not match the
+ given value.
+ - if_modified_since (datetime): Only read if the object was
modified
+ since this timestamp. This timestamp must be in UTC.
+ - if_unmodified_since (datetime): Only read if the object was
not
+ modified since this timestamp. This timestamp must be in
UTC.
+ - cache_control (str): Override the cache-control header for
the object.
+ - content_type (str): Explicitly set the Content-Type header
for
+ the object.
+ - content_disposition (str): Sets how the object should be
presented
+ (e.g., as an attachment).
+ Returns
+ -------
+ Metadata: The metadata of the object.
"""
def create_dir(self, path: PathBuf) -> None:
"""Create a directory at the given path.
@@ -169,32 +196,59 @@ class Operator(_Base):
Args:
path (str|Path): The path to the object.
- Returns:
+ Returns
+ -------
True if the object exists, False otherwise.
"""
- def list(self, path: PathBuf, *, start_after: str | None = None) ->
Iterable[Entry]:
- """List the objects at the given path.
-
- Args:
- path (str|Path): The path to the directory.
- start_after (str | None): The key to start listing from.
-
- Returns:
- An iterable of entries representing the objects in the directory.
- """
- def scan(self, path: PathBuf) -> Iterable[Entry]:
+ def list(self, path: PathBuf, **kwargs) -> Iterable[Entry]:
+ """List objects at the given path.
+
+ Args:
+ path (str | Path): The path to the directory/ prefix.
+ **kwargs (Any): Optional listing parameters matching the
+ [OpenDAL
`ListOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.ListOptions.html):
+
+ - limit (int): The limit passed to the underlying service to
specify the
+ max results that could return per-request. Users could use
this to
+ control the memory usage of list operation. If not set,
all matching
+ entries will be listed.
+ - start_after (str): Start listing after this key. Useful for
pagination
+ or resuming interrupted listings.
+ - recursive (bool): Whether to list entries recursively
through all
+ subdirectories. If False, lists only top-level entries
(entries
+ under the given path).
+ - versions (bool): Whether to include all versions of objects,
if the
+ underlying service supports versioning.
+ - deleted (bool): Whether to include deleted objects, if the
underlying
+ service supports soft-deletes or versioning.
+
+ Returns
+ -------
+ Iterable[Entry]: An iterable of entries representing the objects
in the
+ directory or prefix.
+ """
+ @deprecated("Use `list()` instead.")
+ def scan(self, path: PathBuf, **kwargs) -> Iterable[Entry]:
"""Scan the objects at the given path recursively.
Args:
- path (str|Path): The path to the directory.
+ path (str | Path): The path to the directory/ prefix.
+ **kwargs (Any): Optional listing parameters matching the
+ [OpenDAL
`ListOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.ListOptions.html),
+ excluding `recursive` which is always enforced as `True`
- Returns:
- An iterable of entries representing the objects in the directory.
+ Returns
+ -------
+ Iterable[Entry]: An iterable of all entries under the given path,
+ recursively traversing all subdirectories. Each entry
represents
+ an object (e.g., file or directory) discovered within the full
+ descendant hierarchy of the specified path.
"""
def capability(self) -> Capability:
"""Get the capability of the operator.
- Returns:
+ Returns
+ -------
The capability of the operator.
"""
def copy(self, source: PathBuf, target: PathBuf) -> None:
@@ -212,7 +266,7 @@ class Operator(_Base):
target (str|Path): The target path.
"""
def remove_all(self, path: PathBuf) -> None:
- """Convert into an async operator"""
+ """Convert into an async operator."""
def to_async_operator(self) -> AsyncOperator: ...
@final
@@ -232,6 +286,7 @@ class AsyncOperator(_Base):
await op.write("hello.txt", b"hello world")
```
"""
+
def __init__(self, scheme: str, **options: Any) -> None: ...
def layer(self, layer: Layer) -> AsyncOperator: ...
async def open(self, path: PathBuf, mode: str, **options: Any) ->
AsyncFile:
@@ -248,7 +303,8 @@ class AsyncOperator(_Base):
- If `mode == "wb"`: options match the
[OpenDAL
`WriteOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.WriteOptions.html).
- Returns:
+ Returns
+ -------
AsyncFile: A file-like object that can be used to read or write
the file.
Example:
@@ -289,7 +345,8 @@ class AsyncOperator(_Base):
- if_unmodified_since (datetime): Only read if the object was
not
modified since this timestamp. This timestamp must be in
UTC.
- Returns:
+ Returns
+ -------
The content of the object as bytes.
"""
async def write(self, path: PathBuf, bs: bytes, **options: Any) -> None:
@@ -320,17 +377,35 @@ class AsyncOperator(_Base):
- user_metadata (dict[str, str]): Custom user metadata to
associate
with the object.
- Returns:
+ Returns
+ -------
None
"""
- async def stat(self, path: PathBuf) -> Metadata:
+ async def stat(self, path: PathBuf, **kwargs) -> Metadata:
"""Get the metadata of the object at the given path.
Args:
- path (str|Path): The path to the object.
+ path (str | Path): The path to the object.
+ **kwargs (Any): Optional stat parameters matching the
+ [OpenDAL
`StatOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.StatOptions.html):
- Returns:
- The metadata of the object.
+ - version (str): Specify the version of the object to read, if
+ supported by the backend.
+ - if_match (str): Read only if the ETag matches the given
value.
+ - if_none_match (str): Read-only if the ETag does not match the
+ given value.
+ - if_modified_since (datetime): Only read if the object was
modified
+ since this timestamp. This timestamp must be in UTC.
+ - if_unmodified_since (datetime): Only read if the object was
not
+ modified since this timestamp. This timestamp must be in
UTC.
+ - cache_control (str): Override the cache-control header for
the object.
+ - content_type (str): Explicitly set the Content-Type header
for
+ the object.
+ - content_disposition (str): Sets how the object should be
presented
+ (e.g., as an attachment).
+ Returns
+ -------
+ Metadata: The metadata of the object.
"""
async def create_dir(self, path: PathBuf) -> None:
"""Create a directory at the given path.
@@ -350,29 +425,54 @@ class AsyncOperator(_Base):
Args:
path (str|Path): The path to the object.
- Returns:
+ Returns
+ -------
True if the object exists, False otherwise.
"""
- async def list(
- self, path: PathBuf, *, start_after: str | None = None
- ) -> AsyncIterable[Entry]:
- """List the objects at the given path.
-
- Args:
- path (str|Path): The path to the directory.
- start_after (str | None): The key to start listing from.
-
- Returns:
- An iterable of entries representing the objects in the directory.
- """
- async def scan(self, path: PathBuf) -> AsyncIterable[Entry]:
+ async def list(self, path: PathBuf, **kwargs) -> AsyncIterable[Entry]:
+ """List objects at the given path.
+
+ Args:
+ path (str | Path): The path to the directory/ prefix.
+ **kwargs (Any): Optional listing parameters matching the
+ [OpenDAL
`ListOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.ListOptions.html):
+
+ - limit (int): The limit passed to the underlying service to
specify the
+ max results that could return per-request. Users could use
this to
+ control the memory usage of list operation. If not set,
all matching
+ entries will be listed.
+ - start_after (str): Start listing after this key. Useful for
pagination
+ or resuming interrupted listings.
+ - recursive (bool): Whether to list entries recursively
through all
+ subdirectories. If False, lists only top-level entries
(entries
+ under the given path).
+ - versions (bool): Whether to include all versions of objects,
if the
+ underlying service supports versioning.
+ - deleted (bool): Whether to include deleted objects, if the
underlying
+ service supports soft-deletes or versioning.
+
+ Returns
+ -------
+ Iterable[Entry]: An iterable of entries representing the objects
in the
+ directory or prefix.
+ """
+ @deprecated("Use `list()` instead.")
+ async def scan(self, path: PathBuf, **kwargs) -> AsyncIterable[Entry]:
"""Scan the objects at the given path recursively.
+
Args:
- path (str|Path): The path to the directory.
+ path (str | Path): The path to the directory/ prefix.
+ **kwargs (Any): Optional listing parameters matching the
+ [OpenDAL
`ListOptions`](https://opendal.apache.org/docs/rust/opendal/options/struct.ListOptions.html),
+ excluding `recursive` which is always enforced as `True`
- Returns:
- An iterable of entries representing the objects in the directory.
+ Returns
+ -------
+ Iterable[Entry]: An iterable of all entries under the given path,
+ recursively traversing all subdirectories. Each entry
represents
+ an object (e.g., file or directory) discovered within the full
+ descendant hierarchy of the specified path.
"""
async def presign_stat(self, path: PathBuf, expire_second: int) ->
PresignedRequest:
"""Generate a presigned URL for stat operation.
@@ -381,7 +481,8 @@ class AsyncOperator(_Base):
path (str|Path): The path to the object.
expire_second (int): The expiration time in seconds.
- Returns:
+ Returns
+ -------
A presigned request object.
"""
async def presign_read(self, path: PathBuf, expire_second: int) ->
PresignedRequest:
@@ -391,7 +492,8 @@ class AsyncOperator(_Base):
path (str|Path): The path to the object.
expire_second (int): The expiration time in seconds.
- Returns:
+ Returns
+ -------
A presigned request object.
"""
async def presign_write(
@@ -403,7 +505,8 @@ class AsyncOperator(_Base):
path (str|Path): The path to the object.
expire_second (int): The expiration time in seconds.
- Returns:
+ Returns
+ -------
A presigned request object.
"""
async def presign_delete(
@@ -415,7 +518,8 @@ class AsyncOperator(_Base):
path (str|Path): The path to the object.
expire_second (int): The expiration time in seconds.
- Returns:
+ Returns
+ -------
A presigned request object.
"""
def capability(self) -> Capability: ...
@@ -448,13 +552,15 @@ class File:
Created by the `open` method of the `Operator` class.
"""
+
def read(self, size: int | None = None) -> bytes:
"""Read the content of the file.
Args:
size (int): The number of bytes to read. If None, read all.
- Returns:
+ Returns
+ -------
The content of the file as bytes.
"""
def readline(self, size: int | None = None) -> bytes:
@@ -463,7 +569,8 @@ class File:
Args:
size (int): The number of bytes to read. If None, read until
newline.
- Returns:
+ Returns
+ -------
The line read from the file as bytes.
"""
def write(self, bs: bytes) -> None:
@@ -479,13 +586,15 @@ class File:
pos (int): The position to set.
whence (int): The reference point for the position. Can be 0, 1,
or 2.
- Returns:
+ Returns
+ -------
The new position in the file.
"""
def tell(self) -> int:
"""Get the current position in the file.
- Returns:
+ Returns
+ -------
The current position in the file.
"""
def close(self) -> None:
@@ -512,7 +621,8 @@ class File:
Args:
buffer (bytes|bytearray): The buffer to read into.
- Returns:
+ Returns
+ -------
The number of bytes read.
"""
def seekable(self) -> bool:
@@ -527,13 +637,15 @@ class AsyncFile:
Created by the `open` method of the `AsyncOperator` class.
"""
+
async def read(self, size: int | None = None) -> bytes:
"""Read the content of the file.
Args:
size (int): The number of bytes to read. If None, read all.
- Returns:
+ Returns
+ -------
The content of the file as bytes.
"""
async def write(self, bs: bytes) -> None:
@@ -549,13 +661,15 @@ class AsyncFile:
pos (int): The position to set.
whence (int): The reference point for the position. Can be 0, 1,
or 2.
- Returns:
+ Returns
+ -------
The new position in the file.
"""
async def tell(self) -> int:
"""Get the current position in the file.
- Returns:
+ Returns
+ -------
The current position in the file.
"""
async def close(self) -> None:
@@ -582,9 +696,13 @@ class AsyncFile:
@final
class Entry:
"""An entry in the directory listing."""
+
@property
def path(self) -> str:
"""The path of the entry."""
+ @property
+ def metadata(self) -> Metadata:
+ """The metadata of the entry."""
@final
class Metadata:
@@ -601,11 +719,29 @@ class Metadata:
def content_type(self) -> str | None:
"""The mime type of the object."""
@property
+ def content_encoding(self) -> str | None:
+ """The content encoding of the object."""
+ @property
def etag(self) -> str | None:
"""The ETag of the object."""
@property
def mode(self) -> EntryMode:
"""The mode of the object."""
+ @property
+ def is_file(self) -> bool:
+ """Returns `True` if this metadata is for a file."""
+ @property
+ def is_dir(self) -> bool:
+ """Returns `True` if this metadata is for a directory."""
+ @property
+ def last_modified(self) -> datetime | None:
+ """The last modified time of the object."""
+ @property
+ def version(self) -> str | None:
+ """The version of the object, if available."""
+ @property
+ def user_metadata(self) -> str | None:
+ """The user defined metadata of the object."""
@final
class EntryMode:
diff --git a/bindings/python/python/opendal/exceptions.pyi
b/bindings/python/python/opendal/exceptions.pyi
index 89f6311d6..dd18e34a5 100644
--- a/bindings/python/python/opendal/exceptions.pyi
+++ b/bindings/python/python/opendal/exceptions.pyi
@@ -19,31 +19,31 @@ class Error(Exception):
"""Base class for exceptions in this module."""
class Unexpected(Error):
- """Unexpected errors"""
+ """Unexpected errors."""
class Unsupported(Error):
- """Unsupported operation"""
+ """Unsupported operation."""
class ConfigInvalid(Error):
- """Config is invalid"""
+ """Config is invalid."""
class NotFound(Error):
- """Not found"""
+ """Not found."""
class PermissionDenied(Error):
- """Permission denied"""
+ """Permission denied."""
class IsADirectory(Error):
- """Is a directory"""
+ """Is a directory."""
class NotADirectory(Error):
- """Not a directory"""
+ """Not a directory."""
class AlreadyExists(Error):
- """Already exists"""
+ """Already exists."""
class IsSameFile(Error):
- """Is same file"""
+ """Is same file."""
class ConditionNotMatch(Error):
- """Condition not match"""
+ """Condition not match."""
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 456c3114e..37001dfca 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -55,6 +55,8 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) ->
PyResult<()> {
m.add_class::<WriteOptions>()?;
m.add_class::<ReadOptions>()?;
+ m.add_class::<ListOptions>()?;
+ m.add_class::<StatOptions>()?;
// Layer module
let layers_module = PyModule::new(py, "layers")?;
diff --git a/bindings/python/src/metadata.rs b/bindings/python/src/metadata.rs
index bf3da4520..f7bdd8a14 100644
--- a/bindings/python/src/metadata.rs
+++ b/bindings/python/src/metadata.rs
@@ -17,6 +17,7 @@
use chrono::prelude::*;
use pyo3::prelude::*;
+use std::collections::HashMap;
use crate::*;
@@ -31,18 +32,28 @@ impl Entry {
#[pymethods]
impl Entry {
- /// Path of entry. Path is relative to operator's root.
+ /// Path of entry. Path is relative to the operator's root.
#[getter]
pub fn path(&self) -> &str {
self.0.path()
}
+ /// Metadata of entry.
+ #[getter]
+ pub fn metadata(&self) -> Metadata {
+ Metadata::new(self.0.metadata().clone())
+ }
+
fn __str__(&self) -> &str {
self.0.path()
}
fn __repr__(&self) -> String {
- format!("Entry({:?})", self.0.path())
+ format!(
+ "Entry(path={:?}, metadata={})",
+ self.path(),
+ self.metadata().__repr__()
+ )
}
}
@@ -80,37 +91,72 @@ impl Metadata {
self.0.content_type()
}
+ /// Content Type of this entry.
+ #[getter]
+ pub fn content_encoding(&self) -> Option<&str> {
+ self.0.content_encoding()
+ }
+
/// ETag of this entry.
#[getter]
pub fn etag(&self) -> Option<&str> {
self.0.etag()
}
- /// mode represent this entry's mode.
+ /// mode represents this entry's mode.
#[getter]
pub fn mode(&self) -> EntryMode {
EntryMode(self.0.mode())
}
+ /// Returns `true` if this metadata is for a file.
+ #[getter]
+ pub fn is_file(&self) -> bool {
+ self.mode().is_file()
+ }
+
+ /// Returns `true` if this metadata is for a directory.
+ #[getter]
+ pub fn is_dir(&self) -> bool {
+ self.mode().is_dir()
+ }
+
/// Last modified time
#[getter]
pub fn last_modified(&self) -> Option<DateTime<Utc>> {
self.0.last_modified()
}
- pub fn __repr__(&self) -> String {
- let last_modified_str = match self.0.last_modified() {
- Some(dt) => dt.format("%Y-%m-%dT%H:%M:%S").to_string(),
- None => "None".to_string(),
- };
- format!(
- "Metadata(mode={}, content_length={}, content_type={},
last_modified={}, etag={})",
- self.0.mode(),
- self.0.content_length(),
- self.0.content_type().unwrap_or("None"),
- last_modified_str,
- self.0.etag().unwrap_or("None"),
- )
+ /// Version of this entry, if available.
+ #[getter]
+ pub fn version(&self) -> Option<&str> {
+ self.0.version()
+ }
+
+ /// User defined metadata of this entry
+ #[getter]
+ pub fn user_metadata(&self) -> Option<&HashMap<String, String>> {
+ self.0.user_metadata()
+ }
+
+ pub fn __repr__(&self) -> String {
+ let mut parts = vec![];
+
+ parts.push(format!("mode={}", self.0.mode()));
+ parts.push(format!(
+ "content_disposition={:?}",
+ self.0.content_disposition()
+ ));
+ parts.push(format!("content_length={}", self.0.content_length()));
+ parts.push(format!("content_md5={:?}", self.0.content_md5()));
+ parts.push(format!("content_type={:?}", self.0.content_type()));
+ parts.push(format!("content_encoding={:?}",
self.0.content_encoding()));
+ parts.push(format!("etag={:?}", self.0.etag()));
+ parts.push(format!("last_modified={:?}", self.0.last_modified()));
+ parts.push(format!("version={:?}", self.0.version()));
+ parts.push(format!("user_metadata={:?}", self.0.user_metadata()));
+
+ format!("Metadata({})", parts.join(", "))
}
}
diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index 166dbe5d9..b281b28ed 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -169,11 +169,16 @@ impl Operator {
.map_err(format_pyerr)
}
- /// Get the current path's metadata **without cache** directly.
- pub fn stat(&self, path: PathBuf) -> PyResult<Metadata> {
+ /// Get metadata for the current path **without cache** directly.
+ #[pyo3(signature = (path, **kwargs))]
+ pub fn stat(&self, path: PathBuf, kwargs: Option<&Bound<PyDict>>) ->
PyResult<Metadata> {
let path = path.to_string_lossy().to_string();
+ let kwargs = kwargs
+ .map(|v| v.extract::<StatOptions>())
+ .transpose()?
+ .unwrap_or_default();
self.core
- .stat(&path)
+ .stat_options(&path, kwargs.into())
.map_err(format_pyerr)
.map(Metadata::new)
}
@@ -236,36 +241,35 @@ impl Operator {
}
/// List current dir path.
- #[pyo3(signature = (path, *, start_after=None))]
- pub fn list(&self, path: PathBuf, start_after: Option<String>) ->
PyResult<BlockingLister> {
+ #[pyo3(signature = (path, **kwargs))]
+ pub fn list(&self, path: PathBuf, kwargs: Option<&Bound<PyDict>>) ->
PyResult<BlockingLister> {
let path = path.to_string_lossy().to_string();
+
+ let kwargs = kwargs
+ .map(|v| v.extract::<ListOptions>())
+ .transpose()?
+ .unwrap_or_default();
+
let l = self
.core
- .lister_options(
- &path,
- ocore::options::ListOptions {
- start_after,
- ..Default::default()
- },
- )
+ .lister_options(&path, kwargs.into())
.map_err(format_pyerr)?;
Ok(BlockingLister::new(l))
}
/// List dir in a flat way.
- pub fn scan(&self, path: PathBuf) -> PyResult<BlockingLister> {
- let path = path.to_string_lossy().to_string();
- let l = self
- .core
- .lister_options(
- &path,
- ocore::options::ListOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .map_err(format_pyerr)?;
- Ok(BlockingLister::new(l))
+ #[pyo3(signature = (path, **kwargs))]
+ pub fn scan<'p>(
+ &self,
+ py: Python<'p>,
+ path: PathBuf,
+ kwargs: Option<&Bound<PyDict>>,
+ ) -> PyResult<BlockingLister> {
+ let d = PyDict::new(py);
+ let kwargs = kwargs.unwrap_or(&d);
+ kwargs.set_item("recursive", true)?;
+
+ self.list(path, Some(kwargs))
}
pub fn capability(&self) -> PyResult<capability::Capability> {
@@ -466,13 +470,24 @@ impl AsyncOperator {
})
}
- /// Get current path's metadata **without cache** directly.
- pub fn stat<'p>(&'p self, py: Python<'p>, path: PathBuf) ->
PyResult<Bound<'p, PyAny>> {
+ /// Get metadata for the current path **without cache** directly.
+ #[pyo3(signature = (path, **kwargs))]
+ pub fn stat<'p>(
+ &'p self,
+ py: Python<'p>,
+ path: PathBuf,
+ kwargs: Option<&Bound<PyDict>>,
+ ) -> PyResult<Bound<'p, PyAny>> {
let this = self.core.clone();
let path = path.to_string_lossy().to_string();
+ let kwargs = kwargs
+ .map(|v| v.extract::<StatOptions>())
+ .transpose()?
+ .unwrap_or_default();
+
future_into_py(py, async move {
let res: Metadata = this
- .stat(&path)
+ .stat_options(&path, kwargs.into())
.await
.map_err(format_pyerr)
.map(Metadata::new)?;
@@ -575,38 +590,44 @@ impl AsyncOperator {
}
/// List current dir path.
- #[pyo3(signature = (path, *, start_after=None))]
+ #[pyo3(signature = (path, **kwargs))]
pub fn list<'p>(
&'p self,
py: Python<'p>,
path: PathBuf,
- start_after: Option<String>,
+ kwargs: Option<&Bound<PyDict>>,
) -> PyResult<Bound<'p, PyAny>> {
let this = self.core.clone();
let path = path.to_string_lossy().to_string();
+ let kwargs = kwargs
+ .map(|v| v.extract::<ListOptions>())
+ .transpose()?
+ .unwrap_or_default();
+
future_into_py(py, async move {
- let mut builder = this.lister_with(&path);
- if let Some(start_after) = start_after {
- builder = builder.start_after(&start_after);
- }
- let lister = builder.await.map_err(format_pyerr)?;
+ let lister = this
+ .lister_options(&path, kwargs.into())
+ .await
+ .map_err(format_pyerr)?;
let pylister = Python::with_gil(|py|
AsyncLister::new(lister).into_py_any(py))?;
Ok(pylister)
})
}
- /// List dir in flat way.
- pub fn scan<'p>(&'p self, py: Python<'p>, path: PathBuf) ->
PyResult<Bound<'p, PyAny>> {
- let this = self.core.clone();
- let path = path.to_string_lossy().to_string();
- future_into_py(py, async move {
- let builder = this.lister_with(&path).recursive(true);
- let lister = builder.await.map_err(format_pyerr)?;
- let pylister: PyObject =
- Python::with_gil(|py|
AsyncLister::new(lister).into_py_any(py))?;
- Ok(pylister)
- })
+ /// List dir in a flat way.
+ #[pyo3(signature = (path, **kwargs))]
+ pub fn scan<'p>(
+ &'p self,
+ py: Python<'p>,
+ path: PathBuf,
+ kwargs: Option<&Bound<PyDict>>,
+ ) -> PyResult<Bound<'p, PyAny>> {
+ let d = PyDict::new(py);
+ let kwargs = kwargs.unwrap_or(&d);
+ kwargs.set_item("recursive", true)?;
+
+ self.list(py, path, Some(kwargs))
}
/// Presign an operation for stat(head) which expires after
`expire_second` seconds.
diff --git a/bindings/python/src/options.rs b/bindings/python/src/options.rs
index f47fdb147..6e5953c01 100644
--- a/bindings/python/src/options.rs
+++ b/bindings/python/src/options.rs
@@ -122,3 +122,53 @@ impl From<WriteOptions> for ocore::options::WriteOptions {
}
}
}
+
+#[pyclass(module = "opendal")]
+#[derive(FromPyObject, Default, Debug)]
+pub struct ListOptions {
+ pub limit: Option<usize>,
+ pub start_after: Option<String>,
+ pub recursive: Option<bool>,
+ pub versions: Option<bool>,
+ pub deleted: Option<bool>,
+}
+
+impl From<ListOptions> for ocore::options::ListOptions {
+ fn from(opts: ListOptions) -> Self {
+ Self {
+ limit: opts.limit,
+ start_after: opts.start_after,
+ recursive: opts.recursive.unwrap_or(false),
+ versions: opts.versions.unwrap_or(false),
+ deleted: opts.deleted.unwrap_or(false),
+ }
+ }
+}
+
+#[pyclass(module = "opendal")]
+#[derive(FromPyObject, Default, Debug)]
+pub struct StatOptions {
+ pub version: Option<String>,
+ pub if_match: Option<String>,
+ pub if_none_match: Option<String>,
+ pub if_modified_since: Option<DateTime<Utc>>,
+ pub if_unmodified_since: Option<DateTime<Utc>>,
+ pub content_type: Option<String>,
+ pub cache_control: Option<String>,
+ pub content_disposition: Option<String>,
+}
+
+impl From<StatOptions> for ocore::options::StatOptions {
+ fn from(opts: StatOptions) -> Self {
+ Self {
+ version: opts.version,
+ if_match: opts.if_match,
+ if_none_match: opts.if_none_match,
+ if_modified_since: opts.if_modified_since,
+ if_unmodified_since: opts.if_unmodified_since,
+ override_content_type: opts.content_type,
+ override_cache_control: opts.cache_control,
+ override_content_disposition: opts.content_disposition,
+ }
+ }
+}
diff --git a/bindings/python/tests/test_async_pickle_types.py
b/bindings/python/tests/test_async_pickle_types.py
index fca814ccf..4462e739e 100644
--- a/bindings/python/tests/test_async_pickle_types.py
+++ b/bindings/python/tests/test_async_pickle_types.py
@@ -26,10 +26,7 @@ import pytest
@pytest.mark.asyncio
@pytest.mark.need_capability("read", "write", "delete", "shared")
async def test_operator_pickle(service_name, operator, async_operator):
- """
- Test AsyncOperator's pickle serialization and deserialization.
- """
-
+ """Test AsyncOperator's pickle serialization and deserialization."""
size = randint(1, 1024)
filename = f"random_file_{str(uuid4())}"
content = os.urandom(size)
diff --git a/bindings/python/tests/test_pickle_rw.py
b/bindings/python/tests/test_pickle_rw.py
index 575a13881..3c4d538a2 100644
--- a/bindings/python/tests/test_pickle_rw.py
+++ b/bindings/python/tests/test_pickle_rw.py
@@ -24,9 +24,7 @@ import pytest
@pytest.mark.need_capability("read", "write", "delete")
def test_sync_file_pickle(service_name, operator, async_operator):
- """
- Test pickle streaming serialization and deserialization using OpenDAL
operator.
- """
+ """Test pickle streaming serialization and deserialization using
operator."""
data = {
"a": 1,
"b": "hello",
@@ -34,7 +32,7 @@ def test_sync_file_pickle(service_name, operator,
async_operator):
"d": {"e": 4},
"f": None,
"g": b"hello\nworld",
- "h": set([1, 2, 3]),
+ "h": {1, 2, 3},
"i": 1.23,
"j": True,
"k": datetime.strptime("2024-01-01", "%Y-%m-%d"),
diff --git a/bindings/python/tests/test_sync_pickle_types.py
b/bindings/python/tests/test_sync_pickle_types.py
index ef1467c62..bc360adfa 100644
--- a/bindings/python/tests/test_sync_pickle_types.py
+++ b/bindings/python/tests/test_sync_pickle_types.py
@@ -25,10 +25,7 @@ import pytest
@pytest.mark.need_capability("read", "write", "delete", "shared")
def test_operator_pickle(service_name, operator, async_operator):
- """
- Test Operator's pickle serialization and deserialization.
- """
-
+ """Test Operator's pickle serialization and deserialization."""
size = randint(1, 1024)
filename = f"random_file_{str(uuid4())}"
content = os.urandom(size)
diff --git a/bindings/python/tests/test_write.py
b/bindings/python/tests/test_write.py
index 571d06344..b13026875 100644
--- a/bindings/python/tests/test_write.py
+++ b/bindings/python/tests/test_write.py
@@ -37,19 +37,6 @@ def test_sync_write(service_name, operator, async_operator):
assert metadata.mode.is_file()
assert metadata.content_length == size
- last_modified = (
- metadata.last_modified.strftime("%Y-%m-%dT%H:%M:%S")
- if metadata.last_modified
- else None
- )
- assert repr(metadata) == (
- "Metadata(mode=file, "
- f"content_length={metadata.content_length}, "
- f"content_type={metadata.content_type}, "
- f"last_modified={last_modified}, "
- f"etag={metadata.etag})"
- )
-
operator.delete(filename)
@@ -65,19 +52,6 @@ def test_sync_write_path(service_name, operator,
async_operator):
assert metadata.mode.is_file()
assert metadata.content_length == size
- last_modified = (
- metadata.last_modified.strftime("%Y-%m-%dT%H:%M:%S")
- if metadata.last_modified
- else None
- )
- assert repr(metadata) == (
- "Metadata(mode=file, "
- f"content_length={metadata.content_length}, "
- f"content_type={metadata.content_type}, "
- f"last_modified={last_modified}, "
- f"etag={metadata.etag})"
- )
-
operator.delete(filename)
