nailo2c commented on code in PR #65991:
URL: https://github.com/apache/airflow/pull/65991#discussion_r3263581316
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -670,6 +710,16 @@ def _process_spark_submit_log(self, itr: Iterator[Any]) ->
None:
if match:
self._yarn_application_id = match.group(0)
self.log.info("Identified spark application id: %s",
self._yarn_application_id)
+ # Once the YARN application id is known and the caller
opted in to
+ # `yarn application -status` polling, stop consuming the
JVM stdout
+ # so submit() can terminate the spark-submit process and
switch to
+ # polling ResourceManager.
+ if self._yarn_track_via_application_status and
self._is_yarn_application_submitted(
+ line, self._yarn_application_id
+ ):
+ self._yarn_application_submitted = True
Review Comment:
Same as #5, The comment I added at L633 covered this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
