amoghrajesh commented on code in PR #70513:
URL: https://github.com/apache/airflow/pull/70513#discussion_r3657368999
##########
providers/apache/spark/tests/unit/apache/spark/hooks/test_spark_submit.py:
##########
@@ -391,19 +391,22 @@ def test_submit_failure_includes_captured_log_tail(self,
mock_popen, sdk_connect
with pytest.raises(AirflowException, match="Last spark-submit
output:") as exc_info:
hook.submit()
- assert "Exception in thread main: SparkException: bad jar" in
str(exc_info.value)
+ assert 'Exception in thread "main" org.apache.spark.SparkException:
bad jar' in str(exc_info.value)
@pytest.mark.db_test
@patch("airflow.providers.apache.spark.hooks.spark_submit.subprocess.Popen")
- def test_submit_no_driver_id_includes_captured_log_tail(self, mock_popen,
sdk_connection_not_found):
+ def test_submit_no_driver_id_omits_log_tail_without_exception_marker(
+ self, mock_popen, sdk_connection_not_found
+ ):
+ """spark-submit output with no uncaught-exception marker contributes
no log tail."""
mock_popen.return_value.stdout = StringIO("some unrelated spark-submit
output")
mock_popen.return_value.stderr = StringIO("")
mock_popen.return_value.wait.return_value = 0
hook = SparkSubmitHook(conn_id="spark_standalone_cluster")
with pytest.raises(AirflowException, match="No driver id is known") as
exc_info:
hook.submit()
- assert "Last spark-submit output:\nsome unrelated spark-submit output"
in str(exc_info.value)
+ assert "Last spark-submit output:" not in str(exc_info.value)
Review Comment:
Reverted, matches the original assertion now.
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -55,6 +55,11 @@
_K8S_WAIT_APP_COMPLETION_CONF = "spark.kubernetes.submission.waitAppCompletion"
+# JVM's default uncaught exception handler always prints this exact shape,
regardless of
+# which library raised the exception and that makes it a reliable anchor for
where the real error starts,
+# unlike guessing at stack frame formatting.
+_EXCEPTION_START_RE = re.compile(r'Exception in thread "[^"]*"')
Review Comment:
Trimmed to one line
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
