amoghrajesh commented on code in PR #67118:
URL: https://github.com/apache/airflow/pull/67118#discussion_r3272670921
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -626,28 +626,15 @@ def submit(self, application: str = "", **kwargs: Any) ->
None:
f"Cannot execute: {self._mask_cmd(spark_submit_cmd)}.
Error code is: {returncode}."
)
- self.log.debug("Should track driver: %s",
self._should_track_driver_status)
-
- # We want the Airflow job to wait until the Spark driver is
finished
- if self._should_track_driver_status:
- if self._driver_id is None:
- raise AirflowException(
- "No driver id is known: something went wrong when
executing the spark submit command"
- )
-
- # We start with the SUBMITTED status as initial status
- self._driver_status = "SUBMITTED"
-
- # Start tracking the driver status (blocking function)
- self._start_driver_status_tracking()
-
- if self._driver_status != "FINISHED":
- raise AirflowException(
- f"ERROR : Driver {self._driver_id} badly exited with
status {self._driver_status}"
- )
+ if self._should_track_driver_status and self._driver_id is None:
+ raise AirflowException(
+ "No driver id is known: something went wrong when
executing the spark submit command"
+ )
finally:
self._run_post_submit_commands()
Review Comment:
Nice catch, made the change to move that call to operator
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
