amoghrajesh commented on code in PR #76918:
URL: https://github.com/apache/airflow/pull/76918#discussion_r4585634316


##########
providers/dbt/cloud/docs/operators.rst:
##########
@@ -87,6 +87,41 @@ by providing the ``project_name``, ``environment_name``, and 
``job_name``.
 Please note that it will only work if the above three parameters uniquely 
identify a job in your account
 (i.e. you cannot have two jobs with the same name in the same project and 
environment).
 
+Durable execution
+"""""""""""""""""
+
+On Airflow 3.3 and later, when ``wait_for_termination`` is True, the operator 
stores the dbt Cloud run

Review Comment:
   Handled in c37e6a7388e



##########
providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py:
##########
@@ -243,68 +274,103 @@ def execute(self, context: Context):
 
         self.job_id = self._resolve_job_id()
 
-        self.run_id, job_run_url = self._get_or_trigger_run(context)
+        if self.wait_for_termination:
+            self.execute_resumable(context)
+            return self.run_id
 
+        if self.deferrable is True:
+            warnings.warn(
+                "Argument `wait_for_termination` is False and `deferrable` is 
True , hence "
+                "`deferrable` parameter doesn't have any effect",
+                UserWarning,
+                stacklevel=2,
+            )
+        self.run_id, self._job_run_url = self._get_or_trigger_run(context)
         # Push the ``job_run_url`` and ``job_run_id`` value to XCom regardless 
of what happens during execution.
         # This enables job monitoring via the operator link and provides 
direct access
         # to the job run ID without requiring users to parse the URL manually
-        context["ti"].xcom_push(key="job_run_url", value=job_run_url)
+        self._push_run_xcoms(context)
+        return self.run_id
+
+    def _defer_until_complete(self, run_id: int) -> None:
+        # Derive absolute deadlines for deferrable execution.
+        # execution_timeout is a hard task-level limit (cancels the job),
+        # while timeout only limits how long we wait for the job to finish.
+        # If both are set, the earliest deadline wins.
+        end_time = time.time() + self.timeout
+        execution_deadline = None
+        if self.execution_timeout is not None:
+            execution_deadline = time.time() + 
self.execution_timeout.total_seconds()
+
+        job_run_info = JobRunInfo(account_id=self.account_id, run_id=run_id)
+        job_run_status = self.hook.get_job_run_status(**job_run_info)
+        if DbtCloudJobRunStatus.is_terminal(job_run_status):
+            self._handle_terminal_status(job_run_status)
+            return
+        self.defer(
+            timeout=None,
+            trigger=DbtCloudRunJobTrigger(
+                conn_id=self.dbt_cloud_conn_id,
+                run_id=run_id,
+                end_time=end_time,
+                execution_deadline=execution_deadline,
+                account_id=self.account_id,
+                poll_interval=self.check_interval,
+                hook_params=self.hook_params,
+            ),
+            method_name="execute_complete",
+        )
+
+    def _push_run_xcoms(self, context: Context) -> None:
+        context["ti"].xcom_push(key="job_run_url", value=self._job_run_url)
         context["ti"].xcom_push(key="job_run_id", value=self.run_id)
+        self._run_xcoms_pushed = True
 
-        if self.wait_for_termination and isinstance(self.run_id, int):
-            if self.deferrable is False:
-                self.log.info("Waiting for job run %s to terminate.", 
self.run_id)
-
-                if self.hook.wait_for_job_run_status(
-                    run_id=self.run_id,
-                    account_id=self.account_id,
-                    expected_statuses=DbtCloudJobRunStatus.SUCCESS.value,
-                    check_interval=self.check_interval,
-                    timeout=self.timeout,
-                ):
-                    self.log.info("Job run %s has completed successfully.", 
self.run_id)
-                else:
-                    raise DbtCloudJobRunException(f"Job run {self.run_id} has 
failed or has been cancelled.")
-
-                return self.run_id
-
-            # Derive absolute deadlines for deferrable execution.
-            # execution_timeout is a hard task-level limit (cancels the job),
-            # while timeout only limits how long we wait for the job to finish.
-            # If both are set, the earliest deadline wins.
-            end_time = time.time() + self.timeout
-            execution_deadline = None
-            if self.execution_timeout is not None:
-                execution_deadline = time.time() + 
self.execution_timeout.total_seconds()
-
-            job_run_info = JobRunInfo(account_id=self.account_id, 
run_id=self.run_id)
-            job_run_status = self.hook.get_job_run_status(**job_run_info)
-            if not DbtCloudJobRunStatus.is_terminal(job_run_status):
-                self.defer(
-                    timeout=None,
-                    trigger=DbtCloudRunJobTrigger(
-                        conn_id=self.dbt_cloud_conn_id,
-                        run_id=self.run_id,
-                        end_time=end_time,
-                        execution_deadline=execution_deadline,
-                        account_id=self.account_id,
-                        poll_interval=self.check_interval,
-                        hook_params=self.hook_params,
-                    ),
-                    method_name="execute_complete",
-                )
-            else:
-                return self._handle_terminal_status(job_run_status)
+    def submit_job(self, context: Context) -> int:
+        self.run_id, self._job_run_url = self._get_or_trigger_run(context)

Review Comment:
   Handled in c37e6a7388e



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to