This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new dcbf21690e8 Pass AssetStateStoreAccessors into response_check_path
Callable in HttpEventTrigger (#72287)
dcbf21690e8 is described below
commit dcbf21690e8b444f0dc76d47f7188006b47d5454
Author: sundeep8967 <[email protected]>
AuthorDate: Thu Nov 5 21:08:43 2026 +0530
Pass AssetStateStoreAccessors into response_check_path Callable in
HttpEventTrigger (#72287)
* Pass AssetStateStoreAccessors into response_check_path Callable in
HttpEventTrigger
HttpEventTrigger inherits from BaseEventTrigger which provides
asset_state_store to read and persist state for an Asset.
This update inspects the response_check callable signature and passes
asset_state_store as a keyword argument if the callable accepts it
(or **kwargs), avoiding positional argument collisions. It also safely
guards against AttributeError on Airflow versions earlier than 3.3.0
via getattr(self, "asset_state_store", None), updates docs and examples
with non-blocking aget/aset calls and downstream task consumption,
and expands tests with mock AssetStateStoreAccessors.
Fixes #70074
Signed-off-by: sundeep8967 <[email protected]>
* test(providers/http): convert response_check test shapes to async def
helpers
Use module-level async def functions instead of non-coroutine lambdas so
that _run_response_check coroutine validation passes for all callable
shapes.
Also declare inlets=[asset] on the downstream task in the docs example
so context['asset_state_store'] is populated, and note the Airflow 3.3+
requirement.
Signed-off-by: sundeep8967 <[email protected]>
* Fix formatting and grammar in HttpEventTrigger tests and docs
Generated-by: Claude Opus 5
---------
Signed-off-by: sundeep8967 <[email protected]>
Co-authored-by: Jarek Potiuk <[email protected]>
---
providers/http/docs/triggers.rst | 65 ++++++++++--------
.../src/airflow/providers/http/triggers/http.py | 14 +++-
.../http/tests/unit/http/triggers/test_http.py | 79 ++++++++++++++++++++++
3 files changed, 129 insertions(+), 29 deletions(-)
diff --git a/providers/http/docs/triggers.rst b/providers/http/docs/triggers.rst
index 84294811287..903b86c31ab 100644
--- a/providers/http/docs/triggers.rst
+++ b/providers/http/docs/triggers.rst
@@ -39,6 +39,7 @@ How It Works
.. note::
This trigger requires **Airflow >= 3.0** due to dependencies on
``AssetWatcher`` and event-driven scheduling infrastructure.
+ The ``asset_state_store`` deduplication and downstream task context access
shown in the example below require **Airflow >= 3.3**.
Usage Example with AssetWatcher
-------------------------------
@@ -48,13 +49,12 @@ Here's an example of using the ``HttpEventTrigger`` in an
``AssetWatcher`` to mo
.. code-block:: python
+ import asyncio
import datetime
import os
- from asgiref.sync import sync_to_async
-
from airflow.providers.http.triggers.http import HttpEventTrigger
- from airflow.sdk import Asset, AssetWatcher, Variable, dag, task
+ from airflow.sdk import Asset, AssetWatcher, dag, task
# This token must be generated through GitHub and added as an environment
variable
token = os.getenv("GITHUB_TOKEN")
@@ -66,27 +66,33 @@ Here's an example of using the ``HttpEventTrigger`` in an
``AssetWatcher`` to mo
}
- async def check_github_api_response(response):
+ async def check_github_api_response(response, asset_state_store=None):
"""Determine if a new version of Airflow has been released."""
- # Convert Variable.get to be asynchronous to be used on the Triggerer
- get_variable_async = sync_to_async(Variable.get)
-
- # Retrieve the previous and current release IDs (one from Variables,
one from response)
- previous_release_id = await get_variable_async(key="release_id_var",
default=None)
data = response.json()
release_id = str(data["id"])
- if release_id == previous_release_id:
- return False
-
- # Parse and persist Airflow release data to be used in the downstream
Task
- release_name = data["name"]
- release_html_url = data["html_url"]
- set_variable_async = sync_to_async(Variable.set)
-
- await set_variable_async(key="release_id_var", value=str(release_id))
- await set_variable_async(key="release_name_var", value=release_name)
- await set_variable_async(key="release_html_url_var",
value=release_html_url)
+ if asset_state_store is not None:
+ # aget/aset landed in Airflow 3.3.2; 3.3.0 and 3.3.1 expose the
blocking API.
+ # Use aget/aset when available, or fall back to asyncio.to_thread
to avoid blocking
+ # the triggerer's shared event loop.
+ if hasattr(asset_state_store, "aget"):
+ previous_release_id = await
asset_state_store.aget("release_id", None)
+ else:
+ previous_release_id = await
asyncio.to_thread(asset_state_store.get, "release_id", None)
+
+ if release_id == previous_release_id:
+ return False
+
+ release_name = data.get("name", "Unknown")
+ release_html_url = data.get("html_url", "Unknown")
+ if hasattr(asset_state_store, "aset"):
+ await asset_state_store.aset("release_id", release_id)
+ await asset_state_store.aset("release_name", release_name)
+ await asset_state_store.aset("release_html_url",
release_html_url)
+ else:
+ await asyncio.to_thread(asset_state_store.set, "release_id",
release_id)
+ await asyncio.to_thread(asset_state_store.set, "release_name",
release_name)
+ await asyncio.to_thread(asset_state_store.set,
"release_html_url", release_html_url)
return True
@@ -107,11 +113,11 @@ Here's an example of using the ``HttpEventTrigger`` in an
``AssetWatcher`` to mo
@dag(start_date=datetime.datetime(2024, 10, 1), schedule=asset,
catchup=False)
def check_airflow_releases():
- @task()
- def print_airflow_release_info():
- # Retrieve and output values persisted from the
``response_check_path`` function
- release_name = Variable.get("release_name_var")
- release_html_url = Variable.get("release_html_url_var")
+ @task(inlets=[asset])
+ def print_airflow_release_info(**context):
+ store = context.get("asset_state_store")
+ release_name = store.get("release_name") if store else "Unknown"
+ release_html_url = store.get("release_html_url") if store else
"Unknown"
print(f"{release_name} has been released. Check it out at
{release_html_url}")
print_airflow_release_info()
@@ -144,7 +150,10 @@ Parameters
Additional keyword arguments to pass when creating a request
``response_check_path``
- Path to callable that evaluates whether the API response passes the
conditions set by the user to trigger DAGs
+ Path to callable that evaluates whether the API response passes the
conditions set by the user to trigger DAGs.
+ If the callable accepts an ``asset_state_store`` keyword argument (or
``**kwargs``), the trigger provides the
+ ``AssetStateStoreAccessors`` instance (on Airflow >= 3.3.0) or ``None``
(on older Airflow versions or when the trigger
+ is not running in an asset watcher context).
``poll_interval``
How often, in seconds, the trigger should send a request to the API
@@ -162,5 +171,5 @@ Important Notes
1. A ``response_check_path`` value is required.
2. The ``response_check_path`` must contain the path to an asynchronous
callable. Synchronous callables will raise an exception.
3. The ``poll_interval`` defaults to 60 seconds. This may be changed to avoid
hitting API rate limits.
-4. This trigger does not automatically record the previous API response.
-5. The previous response may have to be persisted manually though
``Variable.set()`` in the ``response_check_path`` callable to prevent the
trigger from emitting events repeatedly for the same API response.
+4. On Airflow >= 3.3.0, the ``response_check_path`` callable can accept
``asset_state_store`` (e.g. ``async def check(response,
asset_state_store=None)``) to read and persist state (like cursors or seen IDs)
across polls without needing external variables.
+5. On Airflow versions earlier than 3.3.0, or when not running in an asset
watcher context, ``asset_state_store`` is ``None``.
diff --git a/providers/http/src/airflow/providers/http/triggers/http.py
b/providers/http/src/airflow/providers/http/triggers/http.py
index d91559bd689..165692b62c3 100644
--- a/providers/http/src/airflow/providers/http/triggers/http.py
+++ b/providers/http/src/airflow/providers/http/triggers/http.py
@@ -288,6 +288,9 @@ class HttpEventTrigger(HttpTrigger, BaseEventTrigger):
:param response_check_path: Path to the function that evaluates whether
the API response
passes the conditions set by the user to fire the trigger. The method
must be asynchronous.
+ Optionally, it may accept an ``asset_state_store`` keyword argument
(or ``**kwargs``)
+ to inspect or persist state across evaluations. Note that
``asset_state_store`` can be
+ ``None`` when running on Airflow < 3.3.0 or when the trigger is not
associated with an asset watcher.
:param http_conn_id: http connection id that has the base
API url i.e https://www.google.com/ and optional authentication
credentials. Default
headers can also be specified in the Extra field in json format.
@@ -393,5 +396,14 @@ class HttpEventTrigger(HttpTrigger, BaseEventTrigger):
response_check = await self._import_from_response_check_path()
if not inspect.iscoroutinefunction(response_check):
raise AirflowException("The response_check callable is not
asynchronous.")
- check = await response_check(response)
+ sig = inspect.signature(response_check)
+ param = sig.parameters.get("asset_state_store")
+ accepts_store = (param is not None and param.kind is not
inspect.Parameter.POSITIONAL_ONLY) or any(
+ p.kind is inspect.Parameter.VAR_KEYWORD for p in
sig.parameters.values()
+ )
+ store = getattr(self, "asset_state_store", None)
+ if accepts_store:
+ check = await response_check(response, asset_state_store=store)
+ else:
+ check = await response_check(response)
return check
diff --git a/providers/http/tests/unit/http/triggers/test_http.py
b/providers/http/tests/unit/http/triggers/test_http.py
index a1542a41508..83367e8edb6 100644
--- a/providers/http/tests/unit/http/triggers/test_http.py
+++ b/providers/http/tests/unit/http/triggers/test_http.py
@@ -38,6 +38,8 @@ from airflow.providers.http.triggers.http import (
)
from airflow.triggers.base import TriggerEvent
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_3_PLUS
+
HTTP_PATH = "airflow.providers.http.triggers.http.{}"
TEST_CONN_ID = "http_default"
TEST_AUTH_TYPE = None
@@ -388,3 +390,80 @@ class TestHttpEventTrigger:
assert kwargs["data"] == TEST_DATA
assert kwargs["json"] is None
assert kwargs["params"] is None
+
+
+async def _single_positional_arg(resp):
+ return resp == "ok"
+
+
+async def _default_second_arg_not_overwritten(resp, threshold=5):
+ return resp == "ok" and threshold == 5
+
+
+async def _keyword_only_arg_not_passed_positionally(resp, *, strict=True):
+ return resp == "ok" and strict is True
+
+
+async def _explicit_asset_state_store_kwarg(resp, asset_state_store=None):
+ return resp == "ok" and asset_state_store == "sentinel_store"
+
+
+async def _var_keyword_kwargs(resp, **kwargs):
+ return resp == "ok" and kwargs.get("asset_state_store") == "sentinel_store"
+
+
+async def _positional_only_asset_state_store(resp,
asset_state_store="default_val", /):
+ return resp == "ok" and asset_state_store == "default_val"
+
+
+class TestHttpEventTriggerResponseCheckCallableShapes:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "check",
+ [
+ pytest.param(_single_positional_arg, id="single_positional_arg"),
+ pytest.param(_default_second_arg_not_overwritten,
id="default_second_arg_not_overwritten"),
+ pytest.param(
+ _keyword_only_arg_not_passed_positionally,
+ id="keyword_only_arg_not_passed_positionally",
+ ),
+ pytest.param(_explicit_asset_state_store_kwarg,
id="explicit_asset_state_store_kwarg"),
+ pytest.param(_var_keyword_kwargs, id="var_keyword_kwargs"),
+ pytest.param(_positional_only_asset_state_store,
id="positional_only_asset_state_store"),
+ ],
+ )
+ async def test_run_response_check_callable_shapes(self, event_trigger,
check):
+ event_trigger.asset_state_store = "sentinel_store"
+ event_trigger._import_from_response_check_path =
mock.AsyncMock(return_value=check)
+ result = await event_trigger._run_response_check("ok")
+ assert result is True
+
+ @pytest.mark.asyncio
+ @pytest.mark.skipif(not AIRFLOW_V_3_3_PLUS, reason="asset_state_store
arrived in Airflow 3.3.0")
+ async def
test_run_response_check_with_real_asset_state_store_autospec(self,
event_trigger):
+ """Verify compatibility with AssetStateStoreAccessors when available
on Airflow >= 3.3."""
+ from airflow.sdk.execution_time.context import AssetStateStoreAccessors
+
+ mock_store = mock.create_autospec(AssetStateStoreAccessors,
instance=True)
+ event_trigger.asset_state_store = mock_store
+
+ async def check(resp, asset_state_store=None):
+ return resp == "ok" and asset_state_store is mock_store
+
+ event_trigger._import_from_response_check_path =
mock.AsyncMock(return_value=check)
+ result = await event_trigger._run_response_check("ok")
+ assert result is True
+
+ @pytest.mark.asyncio
+ async def
test_run_response_check_without_asset_state_store_attribute(self,
event_trigger):
+ """Verify behavior on Airflow versions where asset_state_store is not
set on BaseEventTrigger."""
+ if hasattr(event_trigger, "asset_state_store"):
+ del event_trigger.asset_state_store
+ assert not hasattr(event_trigger, "asset_state_store")
+
+ async def mock_check_with_store(response, asset_state_store=None):
+ return response == "ok" and asset_state_store is None
+
+ event_trigger._import_from_response_check_path =
mock.AsyncMock(return_value=mock_check_with_store)
+ result = await event_trigger._run_response_check("ok")
+ assert result is True