This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5a846fd1a8e Support durable execution for dbt cloud job runs (#76918)
5a846fd1a8e is described below
commit 5a846fd1a8e29dc09ba3ef50c6fab1072a609451
Author: Amogh Desai <[email protected]>
AuthorDate: Tue Nov 17 12:47:59 2026 +0530
Support durable execution for dbt cloud job runs (#76918)
---
providers/dbt/cloud/docs/operators.rst | 38 ++++
providers/dbt/cloud/pyproject.toml | 2 +-
.../airflow/providers/dbt/cloud/operators/dbt.py | 180 +++++++++++-----
.../tests/unit/dbt/cloud/operators/test_dbt.py | 235 +++++++++++++++++++--
4 files changed, 385 insertions(+), 70 deletions(-)
diff --git a/providers/dbt/cloud/docs/operators.rst
b/providers/dbt/cloud/docs/operators.rst
index 80ed3a1b1a7..b6f54f4ec98 100644
--- a/providers/dbt/cloud/docs/operators.rst
+++ b/providers/dbt/cloud/docs/operators.rst
@@ -87,6 +87,44 @@ by providing the ``project_name``, ``environment_name``, and
``job_name``.
Please note that it will only work if the above three parameters uniquely
identify a job in your account
(i.e. you cannot have two jobs with the same name in the same project and
environment).
+Durable execution
+"""""""""""""""""
+
+On Airflow 3.3 and later, when ``wait_for_termination`` is True, the operator
stores the dbt Cloud run
+id in the :doc:`task state store
<apache-airflow:core-concepts/task-state-store>` before it waits. If
+the worker dies and the task retries, the operator checks that run instead of
triggering a new one:
+
+* If the run is still queued, starting or running, the operator waits on it
again, in deferrable mode
+ too.
+* If the run succeeded, the operator returns its run id without triggering
anything.
+* If the run errored, was cancelled or no longer exists, the operator triggers
a new run as before,
+ which respects ``retry_from_failure``.
+
+The run id is saved right after the run is triggered. If the worker dies
between the two, the retry
+finds nothing and triggers a second run. On reconnect, ``timeout`` starts
counting again from zero.
+
+Clearing the task instance from the UI or the REST API discards the stored run
id by default, so the
+next try triggers a new run. This is what you want after fixing upstream data.
To keep the stored run
+id instead, tick "keep task state" in the clear dialog or pass
``keep_task_state=true`` to the clear
+endpoint. The next try then treats the stored run like a retry does.
+
+Before Airflow 3.5, clearing the whole Dag run, marking a task as success or
failed, and
+``airflow tasks clear`` / ``airflow dags clear`` keep the stored run id. So if
you clear the whole Dag
+run after fixing upstream data, a task whose run succeeded returns without
running dbt again. Clear
+the task instance itself instead. From Airflow 3.5, every way of clearing
discards the stored run id
+by default.
+
+Clearing a task that is waiting on the worker cancels its run first, so the
next try triggers a new
+run either way. Only a deferred task's run keeps going. See
+:ref:`apache-airflow:concepts-resumable-tasks` for details.
+
+Below Airflow 3.3 the operator always triggers a new run on retry. Set
``durable=False`` to get that
+behavior on any version:
+
+.. code-block:: python
+
+ DbtCloudRunJobOperator(task_id="run_job", job_id=48617, durable=False)
+
Linking dbt Cloud runs to the triggering Airflow task
""""""""""""""""""""""""""""""""""""""""""""""""""""""
diff --git a/providers/dbt/cloud/pyproject.toml
b/providers/dbt/cloud/pyproject.toml
index 08a12958428..ab2fc114df6 100644
--- a/providers/dbt/cloud/pyproject.toml
+++ b/providers/dbt/cloud/pyproject.toml
@@ -59,7 +59,7 @@ requires-python = ">=3.11"
# After you modify the dependencies, and rebuild your Breeze CI image with
``breeze ci-image build``
dependencies = [
"apache-airflow>=2.11.0",
- "apache-airflow-providers-common-compat>=1.12.0",
+ "apache-airflow-providers-common-compat>=1.12.0", # use next version
"apache-airflow-providers-http",
"asgiref>=2.3.0; python_version < '3.14'",
"asgiref>=3.11.1; python_version >= '3.14'",
diff --git
a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py
b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py
index 0420ef1208c..314ac90d001 100644
--- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py
+++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py
@@ -21,9 +21,17 @@ import time
import warnings
from functools import cached_property
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
-from airflow.providers.common.compat.sdk import BaseOperator,
BaseOperatorLink, XCom, conf
+from requests import HTTPError
+
+from airflow.providers.common.compat.sdk import (
+ BaseOperator,
+ BaseOperatorLink,
+ ResumableJobMixin,
+ XCom,
+ conf,
+)
from airflow.providers.dbt.cloud.hooks.dbt import (
DbtCloudHook,
DbtCloudJobRunException,
@@ -38,6 +46,8 @@ from airflow.providers.dbt.cloud.utils.openlineage import (
)
if TYPE_CHECKING:
+ from pydantic import JsonValue
+
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.sdk import Context
@@ -51,7 +61,17 @@ class DbtCloudRunJobOperatorLink(BaseOperatorLink):
return XCom.get_value(key="job_run_url", ti_key=ti_key)
-class DbtCloudRunJobOperator(BaseOperator):
+def _is_not_found(error: Exception) -> bool:
+ # HttpHook.check_response re-raises without the status code; it stays on
the chained HTTPError.
+ http_error = error if isinstance(error, HTTPError) else error.__context__
+ return (
+ isinstance(http_error, HTTPError)
+ and http_error.response is not None
+ and http_error.response.status_code == 404
+ )
+
+
+class DbtCloudRunJobOperator(ResumableJobMixin, BaseOperator):
"""
Executes a dbt Cloud job.
@@ -94,6 +114,9 @@ class DbtCloudRunJobOperator(BaseOperator):
reading dbt Cloud runs can parse this to link the dbt Cloud run back
to the Airflow task that
triggered it. Note this overwrites the ``trigger_reason``. Defaults to
the
``openlineage.spark_inject_parent_job_info`` config value.
+ :param durable: When ``True`` (the default), the dbt Cloud run id is
persisted to task state before
+ waiting, so a retry after a worker crash reconnects to that run
instead of triggering a new one.
+ Applies only when ``wait_for_termination`` is True. Has no effect
below Airflow 3.3.
:param execution_timeout: Maximum time allowed for the task to run. If
exceeded, the dbt Cloud
job will be cancelled and the task will fail. When both
``execution_timeout`` and
``timeout`` are set, the earlier deadline takes precedence.
@@ -115,6 +138,8 @@ class DbtCloudRunJobOperator(BaseOperator):
operator_extra_links = (DbtCloudRunJobOperatorLink(),)
+ external_id_key = "dbt_cloud_job_run_id"
+
def __init__(
self,
*,
@@ -140,8 +165,12 @@ class DbtCloudRunJobOperator(BaseOperator):
openlineage_inject_parent_job_info: bool = conf.getboolean(
"openlineage", "spark_inject_parent_job_info", fallback=False
),
+ durable: bool | None = None,
**kwargs,
) -> None:
+ # Named param so default_args reaches it below Airflow 3.3 too.
+ if durable is not None:
+ kwargs["durable"] = durable
super().__init__(**kwargs)
self.dbt_cloud_conn_id = dbt_cloud_conn_id
self.account_id = account_id
@@ -157,6 +186,8 @@ class DbtCloudRunJobOperator(BaseOperator):
self.check_interval = check_interval
self.additional_run_config = additional_run_config or {}
self.run_id: int | None = None
+ self._job_run_url: str | None = None
+ self._run_xcoms_pushed = False
self.reuse_existing_run = reuse_existing_run
self.retry_from_failure = retry_from_failure
self.deferrable = deferrable
@@ -243,68 +274,103 @@ class DbtCloudRunJobOperator(BaseOperator):
self.job_id = self._resolve_job_id()
- self.run_id, job_run_url = self._get_or_trigger_run(context)
+ if self.wait_for_termination:
+ self.execute_resumable(context)
+ return self.run_id
+ if self.deferrable is True:
+ warnings.warn(
+ "Argument `wait_for_termination` is False and `deferrable` is
True , hence "
+ "`deferrable` parameter doesn't have any effect",
+ UserWarning,
+ stacklevel=2,
+ )
+ self.run_id, self._job_run_url = self._get_or_trigger_run(context)
# Push the ``job_run_url`` and ``job_run_id`` value to XCom regardless
of what happens during execution.
# This enables job monitoring via the operator link and provides
direct access
# to the job run ID without requiring users to parse the URL manually
- context["ti"].xcom_push(key="job_run_url", value=job_run_url)
+ self._push_run_xcoms(context)
+ return self.run_id
+
+ def _defer_until_complete(self, run_id: int) -> None:
+ # Derive absolute deadlines for deferrable execution.
+ # execution_timeout is a hard task-level limit (cancels the job),
+ # while timeout only limits how long we wait for the job to finish.
+ # If both are set, the earliest deadline wins.
+ end_time = time.time() + self.timeout
+ execution_deadline = None
+ if self.execution_timeout is not None:
+ execution_deadline = time.time() +
self.execution_timeout.total_seconds()
+
+ job_run_info = JobRunInfo(account_id=self.account_id, run_id=run_id)
+ job_run_status = self.hook.get_job_run_status(**job_run_info)
+ if DbtCloudJobRunStatus.is_terminal(job_run_status):
+ self._handle_terminal_status(job_run_status)
+ return
+ self.defer(
+ timeout=None,
+ trigger=DbtCloudRunJobTrigger(
+ conn_id=self.dbt_cloud_conn_id,
+ run_id=run_id,
+ end_time=end_time,
+ execution_deadline=execution_deadline,
+ account_id=self.account_id,
+ poll_interval=self.check_interval,
+ hook_params=self.hook_params,
+ ),
+ method_name="execute_complete",
+ )
+
+ def _push_run_xcoms(self, context: Context) -> None:
+ context["ti"].xcom_push(key="job_run_url", value=self._job_run_url)
context["ti"].xcom_push(key="job_run_id", value=self.run_id)
+ self._run_xcoms_pushed = True
- if self.wait_for_termination and isinstance(self.run_id, int):
- if self.deferrable is False:
- self.log.info("Waiting for job run %s to terminate.",
self.run_id)
-
- if self.hook.wait_for_job_run_status(
- run_id=self.run_id,
- account_id=self.account_id,
- expected_statuses=DbtCloudJobRunStatus.SUCCESS.value,
- check_interval=self.check_interval,
- timeout=self.timeout,
- ):
- self.log.info("Job run %s has completed successfully.",
self.run_id)
- else:
- raise DbtCloudJobRunException(f"Job run {self.run_id} has
failed or has been cancelled.")
-
- return self.run_id
-
- # Derive absolute deadlines for deferrable execution.
- # execution_timeout is a hard task-level limit (cancels the job),
- # while timeout only limits how long we wait for the job to finish.
- # If both are set, the earliest deadline wins.
- end_time = time.time() + self.timeout
- execution_deadline = None
- if self.execution_timeout is not None:
- execution_deadline = time.time() +
self.execution_timeout.total_seconds()
-
- job_run_info = JobRunInfo(account_id=self.account_id,
run_id=self.run_id)
- job_run_status = self.hook.get_job_run_status(**job_run_info)
- if not DbtCloudJobRunStatus.is_terminal(job_run_status):
- self.defer(
- timeout=None,
- trigger=DbtCloudRunJobTrigger(
- conn_id=self.dbt_cloud_conn_id,
- run_id=self.run_id,
- end_time=end_time,
- execution_deadline=execution_deadline,
- account_id=self.account_id,
- poll_interval=self.check_interval,
- hook_params=self.hook_params,
- ),
- method_name="execute_complete",
- )
- else:
- return self._handle_terminal_status(job_run_status)
+ def submit_job(self, context: Context) -> int:
+ self.run_id, self._job_run_url = self._get_or_trigger_run(context)
+ return self.run_id
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ try:
+ job_run = self.hook.get_job_run(run_id=cast("int", external_id),
account_id=self.account_id)
+ except Exception as e:
+ if _is_not_found(e):
+ return "NOT_FOUND"
+ raise
+ data = job_run.json()["data"]
+ self._job_run_url = data["href"]
+ return DbtCloudJobRunStatus(data["status"]).name
+
+ def is_job_active(self, status: str) -> bool:
+ return status in ("QUEUED", "STARTING", "RUNNING")
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return status == "SUCCESS"
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ self.run_id = cast("int", external_id)
+ self._push_run_xcoms(context)
+ if self.deferrable:
+ self._defer_until_complete(self.run_id)
+ return
+ self.log.info("Waiting for job run %s to terminate.", self.run_id)
+ if self.hook.wait_for_job_run_status(
+ run_id=self.run_id,
+ account_id=self.account_id,
+ expected_statuses=DbtCloudJobRunStatus.SUCCESS.value,
+ check_interval=self.check_interval,
+ timeout=self.timeout,
+ ):
+ self.log.info("Job run %s has completed successfully.",
self.run_id)
else:
- if self.deferrable is True:
- warnings.warn(
- "Argument `wait_for_termination` is False and `deferrable`
is True , hence "
- "`deferrable` parameter doesn't have any effect",
- UserWarning,
- stacklevel=2,
- )
- return self.run_id
+ raise DbtCloudJobRunException(f"Job run {self.run_id} has failed
or has been cancelled.")
+
+ def get_job_result(self, external_id: JsonValue, context: Context) -> int:
+ self.run_id = cast("int", external_id)
+ if not self._run_xcoms_pushed:
+ self._push_run_xcoms(context)
+ return self.run_id
def execute_complete(self, context: Context, event: dict[str, Any] | None
= None) -> int:
"""Execute when the trigger fires - returns immediately."""
diff --git a/providers/dbt/cloud/tests/unit/dbt/cloud/operators/test_dbt.py
b/providers/dbt/cloud/tests/unit/dbt/cloud/operators/test_dbt.py
index 91c0d3863cf..0e3c13f14bd 100644
--- a/providers/dbt/cloud/tests/unit/dbt/cloud/operators/test_dbt.py
+++ b/providers/dbt/cloud/tests/unit/dbt/cloud/operators/test_dbt.py
@@ -18,12 +18,12 @@ from __future__ import annotations
import os
from datetime import timedelta
-from unittest.mock import ANY, MagicMock, patch
+from unittest.mock import ANY, MagicMock, call, patch
import pytest
from airflow.models import DAG, Connection
-from airflow.providers.common.compat.sdk import TaskDeferred, timezone
+from airflow.providers.common.compat.sdk import AirflowException,
TaskDeferred, timezone
from airflow.providers.dbt.cloud.hooks.dbt import (
DbtCloudHook,
DbtCloudJobRunException,
@@ -38,7 +38,7 @@ from airflow.providers.dbt.cloud.operators.dbt import (
)
from airflow.providers.dbt.cloud.triggers.dbt import DbtCloudRunJobTrigger
-from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS,
AIRFLOW_V_3_3_PLUS
if AIRFLOW_V_3_0_PLUS:
from airflow.sdk.execution_time.comms import XComResult
@@ -156,7 +156,7 @@ class TestDbtCloudRunJobOperator:
dag=self.dag,
deferrable=True,
)
- dbt_op.execute(MagicMock())
+ dbt_op.execute({"ti": MagicMock()})
assert not mock_defer.called
@patch(
@@ -183,7 +183,7 @@ class TestDbtCloudRunJobOperator:
deferrable=True,
)
with pytest.raises(DbtCloudJobRunException):
- dbt_op.execute(MagicMock())
+ dbt_op.execute({"ti": MagicMock()})
assert not mock_defer.called
@patch(
@@ -216,7 +216,7 @@ class TestDbtCloudRunJobOperator:
execution_timeout=timedelta(seconds=3),
)
- dbt_op.execute(MagicMock())
+ dbt_op.execute({"ti": MagicMock()})
# Explicitly pass timeout=None to defer() so Airflow's framework-level
# deferred timeout handling does not raise TaskDeferredTimeout before
@@ -270,7 +270,7 @@ class TestDbtCloudRunJobOperator:
hook_params=hook_params,
)
- dbt_op.execute(MagicMock())
+ dbt_op.execute({"ti": MagicMock()})
assert mock_dbt_trigger.call_args.kwargs["hook_params"] == hook_params
@@ -306,7 +306,7 @@ class TestDbtCloudRunJobOperator:
deferrable=True,
)
with pytest.raises(TaskDeferred) as exc:
- dbt_op.execute(MagicMock())
+ dbt_op.execute({"ti": MagicMock()})
assert isinstance(exc.value.trigger, DbtCloudRunJobTrigger), "Trigger
is not a DbtCloudRunJobTrigger"
def test_execute_complete_timeout_without_run_id(self):
@@ -467,7 +467,7 @@ class TestDbtCloudRunJobOperator:
timeout=3,
dag=self.dag,
)
- dbt_op.execute(MagicMock())
+ dbt_op.execute({"ti": MagicMock()})
mock_trigger_job_run.assert_called_once()
@pytest.mark.parametrize(
@@ -814,13 +814,14 @@ class TestDbtCloudRunJobOperator:
},
)
+ @patch.object(DbtCloudHook, "wait_for_job_run_status", autospec=True,
return_value=True)
@patch.object(DbtCloudHook, "trigger_job_run")
@pytest.mark.parametrize(
("conn_id", "account_id"),
[(ACCOUNT_ID_CONN, None), (NO_ACCOUNT_ID_CONN, ACCOUNT_ID)],
ids=["default_account", "explicit_account"],
)
- def test_execute_retry_from_failure(self, mock_run_job, conn_id,
account_id):
+ def test_execute_retry_from_failure(self, mock_run_job, mock_wait,
conn_id, account_id):
operator = DbtCloudRunJobOperator(
task_id=TASK_ID,
dbt_cloud_conn_id=conn_id,
@@ -853,13 +854,14 @@ class TestDbtCloudRunJobOperator:
additional_run_config=self.config["additional_run_config"],
)
+ @patch.object(DbtCloudHook, "wait_for_job_run_status", autospec=True,
return_value=True)
@patch.object(DbtCloudHook, "_run_and_get_response")
@pytest.mark.parametrize(
("conn_id", "account_id"),
[(ACCOUNT_ID_CONN, None), (NO_ACCOUNT_ID_CONN, ACCOUNT_ID)],
ids=["default_account", "explicit_account"],
)
- def test_execute_retry_from_failure_run(self, mock_run_req, conn_id,
account_id):
+ def test_execute_retry_from_failure_run(self, mock_run_req, mock_wait,
conn_id, account_id):
operator = DbtCloudRunJobOperator(
task_id=TASK_ID,
dbt_cloud_conn_id=conn_id,
@@ -885,6 +887,7 @@ class TestDbtCloudRunJobOperator:
mock_run_req.assert_called()
+ @patch.object(DbtCloudHook, "wait_for_job_run_status", autospec=True,
return_value=True)
@patch.object(
DbtCloudHook, "_run_and_get_response",
return_value=mock_response_json(JOB_RUN_ERROR_RESPONSE)
)
@@ -894,7 +897,9 @@ class TestDbtCloudRunJobOperator:
[(ACCOUNT_ID_CONN, None), (NO_ACCOUNT_ID_CONN, ACCOUNT_ID)],
ids=["default_account", "explicit_account"],
)
- def test_execute_retry_from_failure_rerun(self, mock_run_req,
mock_rerun_req, conn_id, account_id):
+ def test_execute_retry_from_failure_rerun(
+ self, mock_run_req, mock_rerun_req, mock_wait, conn_id, account_id
+ ):
operator = DbtCloudRunJobOperator(
task_id=TASK_ID,
dbt_cloud_conn_id=conn_id,
@@ -1058,6 +1063,212 @@ class TestDbtCloudRunJobOperator:
)
+STORED_RUN_ID = 9999
+STORED_RUN_URL = EXPECTED_JOB_RUN_OP_EXTRA_LINK.format(
+ account_id=DEFAULT_ACCOUNT_ID, project_id=PROJECT_ID, run_id=STORED_RUN_ID
+)
+STORED_RUN_API_URL =
f"https://cloud.getdbt.com/api/v2/accounts/{DEFAULT_ACCOUNT_ID}/runs/{STORED_RUN_ID}/"
+
+
[email protected](
+ not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution)
requires Airflow 3.3+"
+)
[email protected](DbtCloudHook, "wait_for_job_run_status", autospec=True,
return_value=True)
[email protected](
+ DbtCloudHook,
+ "trigger_job_run",
+ autospec=True,
+ return_value=mock_response_json(DEFAULT_ACCOUNT_JOB_RUN_RESPONSE),
+)
+class TestDbtCloudRunJobOperatorDurable:
+ def setup_method(self):
+ self.dag = DAG("test_dbt_cloud_job_run_op_durable", schedule=None,
start_date=DEFAULT_DATE)
+ self.ti = MagicMock(stats_tags={}, try_number=1)
+ self.task_store = MagicMock(spec_set=["get", "set"])
+ self.task_store.get.return_value = None
+ self.context = {"ti": self.ti, "task_state_store": self.task_store}
+
+ def _operator(self, **kwargs):
+ return DbtCloudRunJobOperator(
+ task_id=TASK_ID, dbt_cloud_conn_id=ACCOUNT_ID_CONN, job_id=JOB_ID,
dag=self.dag, **kwargs
+ )
+
+ def _assert_run_xcoms(self, run_id, run_url):
+ assert self.ti.xcom_push.call_args_list == [
+ call(key="job_run_url", value=run_url),
+ call(key="job_run_id", value=run_id),
+ ]
+
+ def test_fresh_submit_persists_run_id(self, mock_trigger, mock_wait):
+ assert self._operator().execute(self.context) == RUN_ID
+
+ mock_trigger.assert_called_once()
+ mock_wait.assert_called_once()
+ self.task_store.set.assert_called_once_with("dbt_cloud_job_run_id",
RUN_ID)
+ self._assert_run_xcoms(RUN_ID,
DEFAULT_ACCOUNT_JOB_RUN_RESPONSE["data"]["href"])
+
+ @pytest.mark.parametrize(
+ ("status", "waits"),
+ [
+ (DbtCloudJobRunStatus.QUEUED, True),
+ (DbtCloudJobRunStatus.STARTING, True),
+ (DbtCloudJobRunStatus.RUNNING, True),
+ (DbtCloudJobRunStatus.SUCCESS, False),
+ ],
+ )
+ def test_stored_run_is_reused(self, mock_trigger, mock_wait,
requests_mock, status, waits):
+ self.task_store.get.return_value = STORED_RUN_ID
+ requests_mock.get(
+ STORED_RUN_API_URL,
+ json={"data": {"id": STORED_RUN_ID, "status": status.value,
"href": STORED_RUN_URL}},
+ )
+ operator = self._operator()
+
+ assert operator.execute(self.context) == STORED_RUN_ID
+
+ mock_trigger.assert_not_called()
+ assert mock_wait.called is waits
+ self.task_store.set.assert_not_called()
+ assert operator.run_id == STORED_RUN_ID
+ self._assert_run_xcoms(STORED_RUN_ID, STORED_RUN_URL)
+
+ @pytest.mark.parametrize(
+ "response",
+ [
+ {
+ "json": {
+ "data": {
+ "id": STORED_RUN_ID,
+ "status": DbtCloudJobRunStatus.ERROR.value,
+ "href": STORED_RUN_URL,
+ }
+ }
+ },
+ {
+ "json": {
+ "data": {
+ "id": STORED_RUN_ID,
+ "status": DbtCloudJobRunStatus.CANCELLED.value,
+ "href": STORED_RUN_URL,
+ }
+ }
+ },
+ {"status_code": 404, "reason": "Not Found"},
+ ],
+ ids=["error", "cancelled", "not_found"],
+ )
+ def test_stored_run_not_reusable_submits_fresh(self, mock_trigger,
mock_wait, requests_mock, response):
+ self.task_store.get.return_value = STORED_RUN_ID
+ requests_mock.get(STORED_RUN_API_URL, **response)
+
+ assert self._operator().execute(self.context) == RUN_ID
+
+ mock_trigger.assert_called_once()
+ self.task_store.set.assert_called_once_with("dbt_cloud_job_run_id",
RUN_ID)
+ self._assert_run_xcoms(RUN_ID,
DEFAULT_ACCOUNT_JOB_RUN_RESPONSE["data"]["href"])
+
+ def test_stored_errored_run_retries_from_failure(self, mock_trigger,
mock_wait, requests_mock):
+ self.ti.try_number = 2
+ self.task_store.get.return_value = STORED_RUN_ID
+ requests_mock.get(
+ STORED_RUN_API_URL,
+ json={
+ "data": {
+ "id": STORED_RUN_ID,
+ "status": DbtCloudJobRunStatus.ERROR.value,
+ "href": STORED_RUN_URL,
+ }
+ },
+ )
+
+ assert self._operator(retry_from_failure=True).execute(self.context)
== RUN_ID
+
+ mock_trigger.assert_called_once()
+ assert mock_trigger.call_args.kwargs["retry_from_failure"] is True
+ self.task_store.set.assert_called_once_with("dbt_cloud_job_run_id",
RUN_ID)
+
+ @patch.object(DbtCloudHook, "get_job_runs", autospec=True)
+ def test_stored_run_takes_precedence_over_reuse_existing_run(
+ self, mock_get_job_runs, mock_trigger, mock_wait, requests_mock
+ ):
+ self.ti.try_number = 2
+ self.task_store.get.return_value = STORED_RUN_ID
+ requests_mock.get(
+ STORED_RUN_API_URL,
+ json={
+ "data": {
+ "id": STORED_RUN_ID,
+ "status": DbtCloudJobRunStatus.RUNNING.value,
+ "href": STORED_RUN_URL,
+ }
+ },
+ )
+
+ assert self._operator(reuse_existing_run=True).execute(self.context)
== STORED_RUN_ID
+
+ mock_get_job_runs.assert_not_called()
+ mock_trigger.assert_not_called()
+ mock_wait.assert_called_once()
+ assert mock_wait.call_args.kwargs["run_id"] == STORED_RUN_ID
+
+ def test_stored_run_lookup_error_fails_without_triggering(self,
mock_trigger, mock_wait, requests_mock):
+ self.task_store.get.return_value = STORED_RUN_ID
+ requests_mock.get(STORED_RUN_API_URL, status_code=500,
reason="Internal Server Error")
+
+ with pytest.raises(AirflowException, match="^500:"):
+ self._operator().execute(self.context)
+
+ mock_trigger.assert_not_called()
+ self.task_store.set.assert_not_called()
+
+ @patch.object(DbtCloudRunJobOperator, "defer", autospec=True)
+ @patch.object(
+ DbtCloudHook, "get_job_run_status", autospec=True,
return_value=DbtCloudJobRunStatus.RUNNING.value
+ )
+ def test_deferrable_fresh_run_defers_and_persists_run_id(
+ self, mock_status, mock_defer, mock_trigger, mock_wait
+ ):
+ self._operator(deferrable=True).execute(self.context)
+
+ mock_trigger.assert_called_once()
+ mock_wait.assert_not_called()
+ self.task_store.set.assert_called_once_with("dbt_cloud_job_run_id",
RUN_ID)
+ assert mock_defer.call_args.kwargs["trigger"].run_id == RUN_ID
+
+ @patch.object(DbtCloudRunJobOperator, "defer", autospec=True)
+ @patch.object(
+ DbtCloudHook, "get_job_run_status", autospec=True,
return_value=DbtCloudJobRunStatus.RUNNING.value
+ )
+ def test_deferrable_reconnect_defers_on_stored_run(
+ self, mock_status, mock_defer, mock_trigger, mock_wait, requests_mock
+ ):
+ self.task_store.get.return_value = STORED_RUN_ID
+ requests_mock.get(
+ STORED_RUN_API_URL,
+ json={
+ "data": {
+ "id": STORED_RUN_ID,
+ "status": DbtCloudJobRunStatus.RUNNING.value,
+ "href": STORED_RUN_URL,
+ }
+ },
+ )
+
+ self._operator(deferrable=True).execute(self.context)
+
+ mock_trigger.assert_not_called()
+ mock_wait.assert_not_called()
+ self.task_store.set.assert_not_called()
+ assert mock_defer.call_args.kwargs["trigger"].run_id == STORED_RUN_ID
+
+ def test_durable_false_never_touches_task_state_store(self, mock_trigger,
mock_wait):
+ assert self._operator(durable=False).execute(self.context) == RUN_ID
+
+ mock_trigger.assert_called_once()
+ self.task_store.get.assert_not_called()
+ self.task_store.set.assert_not_called()
+
+
class TestDbtCloudGetJobRunArtifactOperator:
def setup_method(self):
self.dag = DAG("test_dbt_cloud_get_artifact_op", schedule=None,
start_date=DEFAULT_DATE)