This is an automated email from the ASF dual-hosted git repository.
weilee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bf46f68a7c1 feat(datasets): make strict_dataset_uri_validation default
to True (#41814)
bf46f68a7c1 is described below
commit bf46f68a7c1dedb6b5abc8dbb410af7e8cd88d55
Author: Wei Lee <[email protected]>
AuthorDate: Wed Oct 9 18:52:10 2024 +0800
feat(datasets): make strict_dataset_uri_validation default to True (#41814)
* feat(datasets): make strict_dataset_uri_validation default to True
* docs(assets): add uri scheme restrict instruction for AIP-60
* feat(assets): add an error logging to URI that is not AIP-60 compliant
* Better test string
---------
Co-authored-by: Tzu-ping Chung <[email protected]>
---
airflow/assets/__init__.py | 20 ++--
airflow/config_templates/config.yml | 4 +-
.../authoring-and-scheduling/assets.rst | 109 ++++++++++++---------
newsfragments/41814.significant.rst | 1 +
tests/assets/tests_asset.py | 11 ---
tests/io/test_path.py | 2 +-
6 files changed, 77 insertions(+), 70 deletions(-)
diff --git a/airflow/assets/__init__.py b/airflow/assets/__init__.py
index deb9aa593de..e11b9c49df3 100644
--- a/airflow/assets/__init__.py
+++ b/airflow/assets/__init__.py
@@ -17,6 +17,7 @@
from __future__ import annotations
+import logging
import os
import urllib.parse
import warnings
@@ -41,6 +42,9 @@ from airflow.configuration import conf
__all__ = ["Asset", "AssetAll", "AssetAny"]
+log = logging.getLogger(__name__)
+
+
def normalize_noop(parts: SplitResult) -> SplitResult:
"""
Place-hold a :class:`~urllib.parse.SplitResult`` normalizer.
@@ -109,14 +113,16 @@ def _sanitize_uri(uri: str) -> str:
try:
parsed = normalizer(parsed)
except ValueError as exception:
- if conf.getboolean("core", "strict_asset_uri_validation",
fallback=False):
+ if conf.getboolean("core", "strict_asset_uri_validation",
fallback=True):
+ log.error(
+ (
+ "The Asset URI %s is not AIP-60 compliant: %s. "
+ "Please check
https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/assets.html";
+ ),
+ uri,
+ exception,
+ )
raise
- warnings.warn(
- f"The Asset URI {uri} is not AIP-60 compliant: {exception}. "
- f"In Airflow 3, this will raise an exception.",
- UserWarning,
- stacklevel=3,
- )
return urllib.parse.urlunsplit(parsed)
diff --git a/airflow/config_templates/config.yml
b/airflow/config_templates/config.yml
index b96c07f2373..0be77a3b682 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -486,9 +486,7 @@ core:
strict_asset_uri_validation:
description: |
Asset URI validation should raise an exception if it is not compliant
with AIP-60.
- By default this configuration is false, meaning that Airflow 2.x only
warns the user.
- In Airflow 3, this configuration will be enabled by default.
- default: "False"
+ default: "True"
example: ~
version_added: 2.9.2
type: boolean
diff --git a/docs/apache-airflow/authoring-and-scheduling/assets.rst
b/docs/apache-airflow/authoring-and-scheduling/assets.rst
index d37143367fa..7940a905167 100644
--- a/docs/apache-airflow/authoring-and-scheduling/assets.rst
+++ b/docs/apache-airflow/authoring-and-scheduling/assets.rst
@@ -23,23 +23,23 @@ Data-aware scheduling
Quickstart
----------
-In addition to scheduling DAGs based on time, you can also schedule DAGs to
run based on when a task updates a asset.
+In addition to scheduling DAGs based on time, you can also schedule DAGs to
run based on when a task updates an asset.
.. code-block:: python
- from airflow.assets import asset
+ from airflow.assets import Asset
with DAG(...):
MyOperator(
# this task updates example.csv
- outlets=[asset("s3://asset-bucket/example.csv")],
+ outlets=[Asset("s3://asset-bucket/example.csv")],
...,
)
with DAG(
# this DAG should be run when example.csv is updated (by dag1)
- schedule=[asset("s3://asset-bucket/example.csv")],
+ schedule=[Asset("s3://asset-bucket/example.csv")],
...,
):
...
@@ -48,7 +48,7 @@ In addition to scheduling DAGs based on time, you can also
schedule DAGs to run
.. image:: /img/asset-scheduled-dags.png
-What is a "asset"?
+What is an "Asset"?
--------------------
An Airflow asset is a logical grouping of data. Upstream producer tasks can
update assets, and asset updates contribute to scheduling downstream consumer
DAGs.
@@ -57,13 +57,19 @@ An Airflow asset is a logical grouping of data. Upstream
producer tasks can upda
.. code-block:: python
- from airflow.assets import asset
+ from airflow.assets import Asset
- example_asset = asset("s3://asset-bucket/example.csv")
+ example_asset = Asset("s3://asset-bucket/example.csv")
Airflow makes no assumptions about the content or location of the data
represented by the URI, and treats the URI like a string. This means that
Airflow treats any regular expressions, like ``input_\d+.csv``, or file glob
patterns, such as ``input_2022*.csv``, as an attempt to create multiple assets
from one declaration, and they will not work.
-You must create assets with a valid URI. Airflow core and providers define
various URI schemes that you can use, such as ``file`` (core), ``postgres`` (by
the Postgres provider), and ``s3`` (by the Amazon provider). Third-party
providers and plugins might also provide their own schemes. These pre-defined
schemes have individual semantics that are expected to be followed.
+You must create assets with a valid URI. Airflow core and providers define
various URI schemes that you can use, such as ``file`` (core), ``postgres`` (by
the Postgres provider), and ``s3`` (by the Amazon provider). Third-party
providers and plugins might also provide their own schemes. These pre-defined
schemes have individual semantics that are expected to be followed. You can use
the optional name argument to provide a more human-readable identifier to the
asset.
+
+.. code-block:: python
+
+ from airflow.assets import Asset
+
+ example_asset = Asset(uri="s3://asset-bucket/example.csv", name="bucket-1")
What is valid URI?
------------------
@@ -72,6 +78,13 @@ Technically, the URI must conform to the valid character set
in RFC 3986, which
The URI is also case sensitive, so ``s3://example/asset`` and
``s3://Example/asset`` are considered different. Note that the *host* part of
the URI is also case sensitive, which differs from RFC 3986.
+For pre-defined schemes (e.g., ``file``, ``postgres``, and ``s3``), you must
provide a meaning URI. If you can't provide one, use another scheme altogether
that don't have the semantic restrictions. Airflow will never require a
semantic for user-defined URI schemes (with a prefix x-), so that can be a
good alternative. If you have a URI that can only be obtained later (e.g.,
during task execution), consider using ``AssetAlias`` instead and update the
URI later.
+
+.. code-block:: python
+
+ # invalid asset:
+ must_contain_bucket_name = Asset("s3://")
+
Do not use the ``airflow`` scheme, which is is reserved for Airflow's
internals.
Airflow always prefers using lower cases in schemes, and case sensitivity is
needed in the host part of the URI to correctly distinguish between resources.
@@ -79,65 +92,65 @@ Airflow always prefers using lower cases in schemes, and
case sensitivity is nee
.. code-block:: python
# invalid assets:
- reserved = asset("airflow://example_asset")
- not_ascii = asset("èxample_datašet")
+ reserved = Asset("airflow://example_asset")
+ not_ascii = Asset("èxample_datašet")
If you want to define assets with a scheme that doesn't include additional
semantic constraints, use a scheme with the prefix ``x-``. Airflow skips any
semantic validation on URIs with these schemes.
.. code-block:: python
# valid asset, treated as a plain string
- my_ds = asset("x-my-thing://foobarbaz")
+ my_ds = Asset("x-my-thing://foobarbaz")
The identifier does not have to be absolute; it can be a scheme-less, relative
URI, or even just a simple path or string:
.. code-block:: python
# valid assets:
- schemeless = asset("//example/asset")
- csv_file = asset("example_asset")
+ schemeless = Asset("//example/asset")
+ csv_file = Asset("example_asset")
Non-absolute identifiers are considered plain strings that do not carry any
semantic meanings to Airflow.
Extra information on asset
----------------------------
-If needed, you can include an extra dictionary in a asset:
+If needed, you can include an extra dictionary in an asset:
.. code-block:: python
- example_asset = asset(
+ example_asset = Asset(
"s3://asset/example.csv",
extra={"team": "trainees"},
)
-This can be used to supply custom description to the asset, such as who has
ownership to the target file, or what the file is for. The extra information
does not affect a asset's identity. This means a DAG will be triggered by a
asset with an identical URI, even if the extra dict is different:
+This can be used to supply custom description to the asset, such as who has
ownership to the target file, or what the file is for. The extra information
does not affect an asset's identity. This means a DAG will be triggered by an
asset with an identical URI, even if the extra dict is different:
.. code-block:: python
with DAG(
dag_id="consumer",
- schedule=[asset("s3://asset/example.csv", extra={"different":
"extras"})],
+ schedule=[Asset("s3://asset/example.csv", extra={"different":
"extras"})],
):
...
with DAG(dag_id="producer", ...):
MyOperator(
# triggers "consumer" with the given extra!
- outlets=[asset("s3://asset/example.csv", extra={"team":
"trainees"})],
+ outlets=[Asset("s3://asset/example.csv", extra={"team":
"trainees"})],
...,
)
-.. note:: **Security Note:** asset URI and extra fields are not encrypted,
they are stored in cleartext in Airflow's metadata database. Do NOT store any
sensitive values, especially credentials, in either asset URIs or extra key
values!
+.. note:: **Security Note:** Asset URI and extra fields are not encrypted,
they are stored in cleartext in Airflow's metadata database. Do NOT store any
sensitive values, especially credentials, in either asset URIs or extra key
values!
How to use assets in your DAGs
--------------------------------
-You can use assets to specify data dependencies in your DAGs. The following
example shows how after the ``producer`` task in the ``producer`` DAG
successfully completes, Airflow schedules the ``consumer`` DAG. Airflow marks a
asset as ``updated`` only if the task completes successfully. If the task fails
or if it is skipped, no update occurs, and Airflow doesn't schedule the
``consumer`` DAG.
+You can use assets to specify data dependencies in your DAGs. The following
example shows how after the ``producer`` task in the ``producer`` DAG
successfully completes, Airflow schedules the ``consumer`` DAG. Airflow marks
an asset as ``updated`` only if the task completes successfully. If the task
fails or if it is skipped, no update occurs, and Airflow doesn't schedule the
``consumer`` DAG.
.. code-block:: python
- example_asset = asset("s3://asset/example.csv")
+ example_asset = Asset("s3://asset/example.csv")
with DAG(dag_id="producer", ...):
BashOperator(task_id="producer", outlets=[example_asset], ...)
@@ -147,7 +160,7 @@ You can use assets to specify data dependencies in your
DAGs. The following exam
You can find a listing of the relationships between assets and DAGs in the
-:ref:`assets View<ui:assets-view>`
+:ref:`Assets View<ui:assets-view>`
Multiple assets
-----------------
@@ -228,17 +241,17 @@ Attaching extra information to an emitting asset event
.. versionadded:: 2.10.0
-A task with a asset outlet can optionally attach extra information before it
emits a asset event. This is different
-from `Extra information on asset`_. Extra information on a asset statically
describes the entity pointed to by the asset URI; extra information on the
*asset event* instead should be used to annotate the triggering data change,
such as how many rows in the database are changed by the update, or the date
range covered by it.
+A task with an asset outlet can optionally attach extra information before it
emits an asset event. This is different
+from `Extra information on asset`_. Extra information on an asset statically
describes the entity pointed to by the asset URI; extra information on the
*asset event* instead should be used to annotate the triggering data change,
such as how many rows in the database are changed by the update, or the date
range covered by it.
The easiest way to attach extra information to the asset event is by
``yield``-ing a ``Metadata`` object from a task:
.. code-block:: python
- from airflow.assets import asset
+ from airflow.assets import Asset
from airflow.assets.metadata import Metadata
- example_s3_asset = asset("s3://asset/example.csv")
+ example_s3_asset = Asset("s3://asset/example.csv")
@task(outlets=[example_s3_asset])
@@ -268,7 +281,7 @@ Fetching information from previously emitted asset events
.. versionadded:: 2.10.0
-Events of a asset defined in a task's ``outlets``, as described in the
previous section, can be read by a task that declares the same asset in its
``inlets``. A asset event entry contains ``extra`` (see previous section for
details), ``timestamp`` indicating when the event was emitted from a task, and
``source_task_instance`` linking the event back to its source.
+Events of an asset defined in a task's ``outlets``, as described in the
previous section, can be read by a task that declares the same asset in its
``inlets``. A asset event entry contains ``extra`` (see previous section for
details), ``timestamp`` indicating when the event was emitted from a task, and
``source_task_instance`` linking the event back to its source.
Inlet asset events can be read with the ``inlet_events`` accessor in the
execution context. Continuing from the ``write_to_s3`` task in the previous
section:
@@ -291,7 +304,7 @@ Example:
.. code-block:: python
- example_snowflake_asset = asset("snowflake://my_db/my_schema/my_table")
+ example_snowflake_asset = Asset("snowflake://my_db/my_schema/my_table")
with DAG(dag_id="load_snowflake_data", schedule="@hourly", ...):
SQLExecuteQueryOperator(
@@ -332,7 +345,7 @@ In this example, the DAG ``waiting_for_asset_1_and_2`` will
be triggered when ta
with DAG(
dag_id="waiting_for_asset_1_and_2",
- schedule=[asset("asset-1"), asset("asset-2")],
+ schedule=[Asset("asset-1"), Asset("asset-2")],
...,
):
...
@@ -344,8 +357,8 @@ In this example, the DAG ``waiting_for_asset_1_and_2`` will
be triggered when ta
* Get queued asset events for a DAG: ``/dags/{dag_id}/assets/queuedEvent``
* Delete a queued asset event for a DAG: ``/assets/queuedEvent/{uri}``
* Delete queued asset events for a DAG: ``/dags/{dag_id}/assets/queuedEvent``
-* Get queued asset events for a asset:
``/dags/{dag_id}/assets/queuedEvent/{uri}``
-* Delete queued asset events for a asset: ``DELETE
/dags/{dag_id}/assets/queuedEvent/{uri}``
+* Get queued asset events for an asset:
``/dags/{dag_id}/assets/queuedEvent/{uri}``
+* Delete queued asset events for an asset: ``DELETE
/dags/{dag_id}/assets/queuedEvent/{uri}``
For how to use REST API and the parameters needed for these endpoints, please
refer to :doc:`Airflow API </stable-rest-api-ref>`.
@@ -373,8 +386,8 @@ To schedule a DAG to run only when two specific assets have
both been updated, u
.. code-block:: python
- dag1_asset = asset("s3://dag1/output_1.txt")
- dag2_asset = asset("s3://dag2/output_1.txt")
+ dag1_asset = Asset("s3://dag1/output_1.txt")
+ dag2_asset = Asset("s3://dag2/output_1.txt")
with DAG(
# Consume asset 1 and 2 with asset expressions
@@ -402,7 +415,7 @@ For scenarios requiring more intricate conditions, such as
triggering a DAG when
.. code-block:: python
- dag3_asset = asset("s3://dag3/output_3.txt")
+ dag3_asset = Asset("s3://dag3/output_3.txt")
with DAG(
# Consume asset 1 or both 2 and 3 with asset expressions
@@ -421,9 +434,9 @@ How to use AssetAlias
``AssetAlias`` has one single argument ``name`` that uniquely identifies the
asset. The task must first declare the alias as an outlet, and use
``outlet_events`` or yield ``Metadata`` to add events to it.
-The following example creates a asset event against the S3 URI
``f"s3://bucket/my-task"`` with optional extra information ``extra``. If the
asset does not exist, Airflow will dynamically create it and log a warning
message.
+The following example creates an asset event against the S3 URI
``f"s3://bucket/my-task"`` with optional extra information ``extra``. If the
asset does not exist, Airflow will dynamically create it and log a warning
message.
-**Emit a asset event during task execution through outlet_events**
+**Emit an asset event during task execution through outlet_events**
.. code-block:: python
@@ -432,10 +445,10 @@ The following example creates a asset event against the
S3 URI ``f"s3://bucket/m
@task(outlets=[AssetAlias("my-task-outputs")])
def my_task_with_outlet_events(*, outlet_events):
- outlet_events["my-task-outputs"].add(asset("s3://bucket/my-task"),
extra={"k": "v"})
+ outlet_events["my-task-outputs"].add(Asset("s3://bucket/my-task"),
extra={"k": "v"})
-**Emit a asset event during task execution through yielding Metadata**
+**Emit an asset event during task execution through yielding Metadata**
.. code-block:: python
@@ -444,7 +457,7 @@ The following example creates a asset event against the S3
URI ``f"s3://bucket/m
@task(outlets=[AssetAlias("my-task-outputs")])
def my_task_with_metadata():
- s3_asset = asset("s3://bucket/my-task")
+ s3_asset = Asset("s3://bucket/my-task")
yield Metadata(s3_asset, extra={"k": "v"}, alias="my-task-outputs")
Only one asset event is emitted for an added asset, even if it is added to the
alias multiple times, or added to multiple aliases. However, if different
``extra`` values are passed, it can emit multiple asset events. In the
following example, two asset events will be emitted.
@@ -462,15 +475,15 @@ Only one asset event is emitted for an added asset, even
if it is added to the a
]
)
def my_task_with_outlet_events(*, outlet_events):
- outlet_events["my-task-outputs-1"].add(asset("s3://bucket/my-task"),
extra={"k": "v"})
+ outlet_events["my-task-outputs-1"].add(Asset("s3://bucket/my-task"),
extra={"k": "v"})
# This line won't emit an additional asset event as the asset and
extra are the same as the previous line.
- outlet_events["my-task-outputs-2"].add(asset("s3://bucket/my-task"),
extra={"k": "v"})
+ outlet_events["my-task-outputs-2"].add(Asset("s3://bucket/my-task"),
extra={"k": "v"})
# This line will emit an additional asset event as the extra is
different.
- outlet_events["my-task-outputs-3"].add(asset("s3://bucket/my-task"),
extra={"k2": "v2"})
+ outlet_events["my-task-outputs-3"].add(Asset("s3://bucket/my-task"),
extra={"k2": "v2"})
Scheduling based on asset aliases
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since asset events added to an alias are just simple asset events, a
downstream DAG depending on the actual asset can read asset events of it
normally, without considering the associated aliases. A downstream DAG can also
depend on an asset alias. The authoring syntax is referencing the
``AssetAlias`` by name, and the associated asset events are picked up for
scheduling. Note that a DAG can be triggered by a task with
``outlets=AssetAlias("xxx")`` if and only if the alias is resolved int [...]
+Since asset events added to an alias are just simple asset events, a
downstream DAG depending on the actual asset can read asset events of it
normally, without considering the associated aliases. A downstream DAG can also
depend on an asset alias. The authoring syntax is referencing the
``AssetAlias`` by name, and the associated asset events are picked up for
scheduling. Note that a DAG can be triggered by a task with
``outlets=AssetAlias("xxx")`` if and only if the alias is resolved int [...]
The asset alias is resolved to the assets during DAG parsing. Thus, if the
"min_file_process_interval" configuration is set to a high value, there is a
possibility that the asset alias may not be resolved. To resolve this issue,
you can trigger DAG parsing.
@@ -478,7 +491,7 @@ The asset alias is resolved to the assets during DAG
parsing. Thus, if the "min_
with DAG(dag_id="asset-producer"):
- @task(outlets=[asset("example-alias")])
+ @task(outlets=[Asset("example-alias")])
def produce_asset_events():
pass
@@ -487,17 +500,17 @@ The asset alias is resolved to the assets during DAG
parsing. Thus, if the "min_
@task(outlets=[AssetAlias("example-alias")])
def produce_asset_events(*, outlet_events):
- outlet_events["example-alias"].add(asset("s3://bucket/my-task"))
+ outlet_events["example-alias"].add(Asset("s3://bucket/my-task"))
- with DAG(dag_id="asset-consumer", schedule=asset("s3://bucket/my-task")):
+ with DAG(dag_id="asset-consumer", schedule=Asset("s3://bucket/my-task")):
...
with DAG(dag_id="asset-alias-consumer",
schedule=AssetAlias("example-alias")):
...
-In the example provided, once the DAG ``asset-alias-producer`` is executed,
the asset alias ``AssetAlias("example-alias")`` will be resolved to
``asset("s3://bucket/my-task")``. However, the DAG ``asset-alias-consumer``
will have to wait for the next DAG re-parsing to update its schedule. To
address this, Airflow will re-parse the DAGs relying on the asset alias
``AssetAlias("example-alias")`` when it's resolved into assets that these DAGs
did not previously depend on. As a result, both [...]
+In the example provided, once the DAG ``asset-alias-producer`` is executed,
the asset alias ``AssetAlias("example-alias")`` will be resolved to
``Asset("s3://bucket/my-task")``. However, the DAG ``asset-alias-consumer``
will have to wait for the next DAG re-parsing to update its schedule. To
address this, Airflow will re-parse the DAGs relying on the asset alias
``AssetAlias("example-alias")`` when it's resolved into assets that these DAGs
did not previously depend on. As a result, both [...]
Fetching information from previously emitted asset events through resolved
asset aliases
@@ -511,7 +524,7 @@ As mentioned in :ref:`Fetching information from previously
emitted asset events<
@task(outlets=[AssetAlias("example-alias")])
def produce_asset_events(*, outlet_events):
- outlet_events["example-alias"].add(asset("s3://bucket/my-task"),
extra={"row_count": 1})
+ outlet_events["example-alias"].add(Asset("s3://bucket/my-task"),
extra={"row_count": 1})
with DAG(dag_id="asset-alias-consumer", schedule=None):
diff --git a/newsfragments/41814.significant.rst
b/newsfragments/41814.significant.rst
new file mode 100644
index 00000000000..f3b6003a5c3
--- /dev/null
+++ b/newsfragments/41814.significant.rst
@@ -0,0 +1 @@
+Change the default value of ``strict_dataset_uri_validation`` to True.
diff --git a/tests/assets/tests_asset.py b/tests/assets/tests_asset.py
index da6ef8ee79e..afbb46827f3 100644
--- a/tests/assets/tests_asset.py
+++ b/tests/assets/tests_asset.py
@@ -39,7 +39,6 @@ from airflow.models.asset import AssetAliasModel,
AssetDagRunQueue, AssetModel
from airflow.models.serialized_dag import SerializedDagModel
from airflow.operators.empty import EmptyOperator
from airflow.serialization.serialized_objects import BaseSerialization,
SerializedDAG
-from tests.test_utils.config import conf_vars
@pytest.fixture
@@ -492,16 +491,6 @@ def _mock_get_uri_normalizer_noop(normalized_scheme):
@patch("airflow.assets._get_uri_normalizer",
_mock_get_uri_normalizer_raising_error)
-@patch("airflow.assets.warnings.warn")
-def test_sanitize_uri_raises_warning(mock_warn):
- _sanitize_uri("postgres://localhost:5432/database.schema.table")
- msg = mock_warn.call_args.args[0]
- assert "The Asset URI postgres://localhost:5432/database.schema.table is
not AIP-60 compliant" in msg
- assert "In Airflow 3, this will raise an exception." in msg
-
-
-@patch("airflow.assets._get_uri_normalizer",
_mock_get_uri_normalizer_raising_error)
-@conf_vars({("core", "strict_asset_uri_validation"): "True"})
def test_sanitize_uri_raises_exception():
with pytest.raises(ValueError) as e_info:
_sanitize_uri("postgres://localhost:5432/database.schema.table")
diff --git a/tests/io/test_path.py b/tests/io/test_path.py
index 0e504b586b3..9d944c5f51c 100644
--- a/tests/io/test_path.py
+++ b/tests/io/test_path.py
@@ -406,7 +406,7 @@ class TestFs:
attach("s3", fs=FakeRemoteFileSystem())
p = "s3"
- f = "/tmp/foo"
+ f = "bucket/object"
i = Asset(uri=f"{p}://{f}", extra={"foo": "bar"})
o = ObjectStoragePath(i)
assert o.protocol == p
