o-nikolas commented on code in PR #73701:
URL: https://github.com/apache/airflow/pull/73701#discussion_r4486663712


##########
providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py:
##########
@@ -1500,6 +1501,104 @@ def execute(self, context: Context, event: dict[str, 
Any] | None = None) -> str
 
         return self.job_id
 
+    def _start_job_run(self, context: Context) -> str:
+        """Submit a new EMR Serverless job run and return its id."""
+        configuration_overrides = self.configuration_overrides
+        if self.openlineage_inject_parent_job_info:
+            self.log.info("Injecting OpenLineage parent job information into 
EMR Serverless configuration.")
+            configuration_overrides = 
inject_parent_job_information_into_emr_serverless_properties(
+                configuration_overrides, context
+            )
+        if self.openlineage_inject_transport_info:
+            self.log.info("Injecting OpenLineage transport information into 
EMR Serverless configuration.")
+            configuration_overrides = 
inject_transport_information_into_emr_serverless_properties(
+                configuration_overrides, context
+            )
+
+        args = {
+            "clientToken": self.client_request_token,
+            "applicationId": self.application_id,
+            "executionRoleArn": self.execution_role_arn,
+            "jobDriver": self.job_driver,
+            "name": self.name,
+            **self.config,
+        }
+        if configuration_overrides is not None:
+            args["configurationOverrides"] = configuration_overrides
+        response = self.hook.conn.start_job_run(
+            **args,
+        )
+
+        if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
+            raise AirflowException(f"EMR serverless job failed to start: 
{response}")
+
+        job_id = response["jobRunId"]
+        self.log.info("EMR serverless job started: %s", job_id)
+        return job_id
+
+    def _find_job_run_to_reconnect(self, context: Context) -> tuple[str, str] 
| None:
+        """
+        Return the job run id and state to reconnect to instead of submitting 
a new run, or ``None``.
+
+        With ``durable=True`` the submitted run id is persisted to 
``task_state_store`` (Airflow 3.3+) and
+        read back on a retry or a manual clear/rerun, so a still-running Spark 
job is reattached to rather
+        than duplicated -- matching ``GlueJobOperator``'s ``durable`` 
behaviour. A prior run that already
+        succeeded is reattached to (its poll returns immediately); a 
terminal-failed run falls through to a

Review Comment:
   nit: "its poll returns immediately" is out of date. A succeeded run now 
returns early in `execute` without polling.



##########
providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py:
##########
@@ -1500,6 +1501,104 @@ def execute(self, context: Context, event: dict[str, 
Any] | None = None) -> str
 
         return self.job_id
 
+    def _start_job_run(self, context: Context) -> str:
+        """Submit a new EMR Serverless job run and return its id."""
+        configuration_overrides = self.configuration_overrides
+        if self.openlineage_inject_parent_job_info:
+            self.log.info("Injecting OpenLineage parent job information into 
EMR Serverless configuration.")
+            configuration_overrides = 
inject_parent_job_information_into_emr_serverless_properties(
+                configuration_overrides, context
+            )
+        if self.openlineage_inject_transport_info:
+            self.log.info("Injecting OpenLineage transport information into 
EMR Serverless configuration.")
+            configuration_overrides = 
inject_transport_information_into_emr_serverless_properties(
+                configuration_overrides, context
+            )
+
+        args = {
+            "clientToken": self.client_request_token,
+            "applicationId": self.application_id,
+            "executionRoleArn": self.execution_role_arn,
+            "jobDriver": self.job_driver,
+            "name": self.name,
+            **self.config,
+        }
+        if configuration_overrides is not None:
+            args["configurationOverrides"] = configuration_overrides
+        response = self.hook.conn.start_job_run(
+            **args,
+        )
+
+        if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
+            raise AirflowException(f"EMR serverless job failed to start: 
{response}")
+
+        job_id = response["jobRunId"]
+        self.log.info("EMR serverless job started: %s", job_id)
+        return job_id
+
+    def _find_job_run_to_reconnect(self, context: Context) -> tuple[str, str] 
| None:
+        """
+        Return the job run id and state to reconnect to instead of submitting 
a new run, or ``None``.
+
+        With ``durable=True`` the submitted run id is persisted to 
``task_state_store`` (Airflow 3.3+) and
+        read back on a retry or a manual clear/rerun, so a still-running Spark 
job is reattached to rather
+        than duplicated -- matching ``GlueJobOperator``'s ``durable`` 
behaviour. A prior run that already
+        succeeded is reattached to (its poll returns immediately); a 
terminal-failed run falls through to a
+        fresh submission. Returns ``None`` when the store is unavailable 
(Airflow < 3.3) or holds no id,
+        and also when ``cancel_on_kill`` is on in deferrable mode, after 
cancelling a still-active run.
+
+        The deferrable path cannot reuse ``ResumableJobMixin`` for this today; 
tracked at
+        https://github.com/apache/airflow/issues/71485.
+        """
+        task_state_store = context.get("task_state_store")
+        if task_state_store is None:
+            return None
+        stored_job_id = task_state_store.get(self.external_id_key)
+        if not stored_job_id:
+            return None
+        job_run_id = cast("str", stored_job_id)
+        state = self.hook.conn.get_job_run(applicationId=self.application_id, 
jobRunId=job_run_id)["jobRun"][
+            "state"
+        ]
+        if self.deferrable and self.cancel_on_kill:

Review Comment:
   With `cancel_on_kill=True`, a stored run that already succeeded gets re-run. 
If the job succeeded but the worker died before the task was marked success, 
the retry runs the whole Spark job again, which is what `durable` is meant to 
stop. The cancel reason here only covers runs that are still going. Could a 
`SUCCESS` run return early here too? That matches the docstring at line 1319, 
and the `succeeded` test case would flip.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to