This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f860141cfd respect "soft_fail" argument when running BatchSensor in
deferrable mode (#33405)
f860141cfd is described below
commit f860141cfd1df91efaca0a1c904762778d91cd77
Author: Wei Lee <[email protected]>
AuthorDate: Thu Aug 31 23:45:45 2023 +0800
respect "soft_fail" argument when running BatchSensor in deferrable mode
(#33405)
---
airflow/providers/amazon/aws/sensors/batch.py | 9 +++++++--
tests/providers/amazon/aws/sensors/test_batch.py | 8 +++++++-
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/airflow/providers/amazon/aws/sensors/batch.py
b/airflow/providers/amazon/aws/sensors/batch.py
index c193a747d1..3f31a87bed 100644
--- a/airflow/providers/amazon/aws/sensors/batch.py
+++ b/airflow/providers/amazon/aws/sensors/batch.py
@@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Any, Sequence
from deprecated import deprecated
from airflow.configuration import conf
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowSkipException
from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
from airflow.providers.amazon.aws.triggers.batch import BatchJobTrigger
from airflow.sensors.base import BaseSensorOperator
@@ -115,7 +115,12 @@ class BatchSensor(BaseSensorOperator):
Relies on trigger to throw an exception, otherwise it assumes
execution was successful.
"""
if event["status"] != "success":
- raise AirflowException(f"Error while running job: {event}")
+ message = f"Error while running job: {event}"
+ # TODO: remove this if-else block when min_airflow_version is set
to higher than the version that
+ # changed in https://github.com/apache/airflow/pull/33424 is
released
+ if self.soft_fail:
+ raise AirflowSkipException(message)
+ raise AirflowException(message)
job_id = event["job_id"]
self.log.info("Batch Job %s complete", job_id)
diff --git a/tests/providers/amazon/aws/sensors/test_batch.py
b/tests/providers/amazon/aws/sensors/test_batch.py
index 353db3b812..a7d0cb6974 100644
--- a/tests/providers/amazon/aws/sensors/test_batch.py
+++ b/tests/providers/amazon/aws/sensors/test_batch.py
@@ -20,7 +20,7 @@ from unittest import mock
import pytest
-from airflow.exceptions import AirflowException, TaskDeferred
+from airflow.exceptions import AirflowException, AirflowSkipException,
TaskDeferred
from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
from airflow.providers.amazon.aws.sensors.batch import (
BatchComputeEnvironmentSensor,
@@ -100,6 +100,12 @@ class TestBatchSensor:
with pytest.raises(AirflowException):
deferrable_batch_sensor.execute_complete(context={},
event={"status": "failure"})
+ def test_execute_failure_in_deferrable_mode_with_soft_fail(self,
deferrable_batch_sensor: BatchSensor):
+ """Tests that an AirflowSkipException is raised in case of error event
and soft_fail is set to True"""
+ deferrable_batch_sensor.soft_fail = True
+ with pytest.raises(AirflowSkipException):
+ deferrable_batch_sensor.execute_complete(context={},
event={"status": "failure"})
+
@pytest.fixture(scope="module")
def batch_compute_environment_sensor() -> BatchComputeEnvironmentSensor:
