This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b38e09397e5 Add local scheme as alternative to file for using the
ObjectStoragePath (#46670)
b38e09397e5 is described below
commit b38e09397e562b8a35dc09c8f3da4761445933ec
Author: David Blain <[email protected]>
AuthorDate: Fri Feb 28 22:52:23 2025 +0100
Add local scheme as alternative to file for using the ObjectStoragePath
(#46670)
* refactor: Added local filesystem
* refactor: Refactored test_ls with parametrized scheme
* docs: Added example of using the XComObjectStorageBackend with local
filesystem
---------
Co-authored-by: David Blain <[email protected]>
---
airflow/io/__init__.py | 1 +
providers/common/io/docs/xcom_backend.rst | 16 ++++++++++++++++
tests/io/test_path.py | 11 +++++++++--
3 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/airflow/io/__init__.py b/airflow/io/__init__.py
index 3229b831016..7ae083ab451 100644
--- a/airflow/io/__init__.py
+++ b/airflow/io/__init__.py
@@ -47,6 +47,7 @@ def _file(_: str | None, storage_options: Properties) ->
LocalFileSystem:
# builtin supported filesystems
_BUILTIN_SCHEME_TO_FS: dict[str, Callable[[str | None, Properties],
AbstractFileSystem]] = {
"file": _file,
+ "local": _file,
}
diff --git a/providers/common/io/docs/xcom_backend.rst
b/providers/common/io/docs/xcom_backend.rst
index 43bb7bb5792..ae2a8bc3673 100644
--- a/providers/common/io/docs/xcom_backend.rst
+++ b/providers/common/io/docs/xcom_backend.rst
@@ -37,6 +37,22 @@ So for example the following configuration will store
anything above 1MB in S3 a
xcom_objectstorage_threshold = 1048576
xcom_objectstorage_compression = gzip
+Another example using the local filesystem::
+
+ [core]
+ xcom_backend =
airflow.providers.common.io.xcom.backend.XComObjectStorageBackend
+
+ [common.io]
+ xcom_objectstorage_path = file://airflow/xcoms
+
+The local filesystem scheme can also be used as it's the same as file show
above, see the `fsspec
<https://github.com/fsspec/filesystem_spec/blob/f30bc759f30327dfb499f37e967648f175750fac/fsspec/implementations/local.py#L30>`_::
+
+ [core]
+ xcom_backend =
airflow.providers.common.io.xcom.backend.XComObjectStorageBackend
+
+ [common.io]
+ xcom_objectstorage_path = local://airflow/xcoms
+
.. note::
Compression requires the support for it is installed in your python
environment. For example, to use ``snappy`` compression, you need to install
``python-snappy``. Zip, gzip and bz2 work out of the box.
diff --git a/tests/io/test_path.py b/tests/io/test_path.py
index 29e67ca8464..f079357c398 100644
--- a/tests/io/test_path.py
+++ b/tests/io/test_path.py
@@ -135,11 +135,18 @@ class TestFs:
o.unlink()
- def test_ls(self):
+ @pytest.mark.parametrize(
+ "scheme",
+ [
+ pytest.param("local", id="local_scheme"),
+ pytest.param("file", id="file_scheme"),
+ ],
+ )
+ def test_ls(self, scheme):
dirname = str(uuid.uuid4())
filename = str(uuid.uuid4())
- d = ObjectStoragePath(f"file:///tmp/{dirname}")
+ d = ObjectStoragePath(f"{scheme}:///tmp/{dirname}")
d.mkdir(parents=True)
o = d / filename
o.touch()
