This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 53ca1605d23 Add deferrable support to AthenaSparkOperator (#75915)
53ca1605d23 is described below
commit 53ca1605d2351dcb8c7953188ffd7989312d7ce6
Author: SameerMesiah97 <[email protected]>
AuthorDate: Sat Nov 14 16:33:33 2026 +0000
Add deferrable support to AthenaSparkOperator (#75915)
* Add deferrable support to AthenaSparkOperator
Monitor calculations asynchronously with AthenaSparkTrigger, releasing
worker slots while waiting. Preserve metadata and failure handling,
with tests, documentation, and a system test example.
* Add waiter tests for terminal states, continued polling, and
exhaustion.Simplify trigger event validation, remove the redundant hook
override, and document the remaining AWS configuration parameters.
---
.../amazon/docs/operators/athena/athena_spark.rst | 24 +++
.../providers/amazon/aws/operators/athena_spark.py | 49 +++++
.../providers/amazon/aws/triggers/athena.py | 49 +++++
.../providers/amazon/aws/waiters/athena.json | 25 +++
.../system/amazon/aws/example_athena_spark.py | 13 ++
.../unit/amazon/aws/operators/test_athena_spark.py | 220 +++++++++++++++++++++
.../tests/unit/amazon/aws/triggers/test_athena.py | 55 +++++-
.../tests/unit/amazon/aws/waiters/test_athena.py | 91 +++++++++
8 files changed, 525 insertions(+), 1 deletion(-)
diff --git a/providers/amazon/docs/operators/athena/athena_spark.rst
b/providers/amazon/docs/operators/athena/athena_spark.rst
index 260675ba40b..2bbf43f1d7d 100644
--- a/providers/amazon/docs/operators/athena/athena_spark.rst
+++ b/providers/amazon/docs/operators/athena/athena_spark.rst
@@ -53,6 +53,30 @@ this operator, please see the `Sample Dag
<https://github.com/apache/airflow/blo
:start-after: [START howto_operator_athena_spark]
:end-before: [END howto_operator_athena_spark]
+Deferrable mode
+~~~~~~~~~~~~~~~
+
+Set ``deferrable=True`` to wait for the calculation asynchronously using the
+Airflow triggerer, releasing the worker slot while the calculation runs.
+This requires a running triggerer and the ``aiobotocore`` package.
+
+The ``waiter_delay`` and ``waiter_max_attempts`` parameters control polling
+in both synchronous and deferrable modes. Both modes return the same execution
+metadata when the calculation completes successfully and raise an exception
+if it fails or is canceled.
+
+.. exampleinclude::
/../../amazon/tests/system/amazon/aws/example_athena_spark.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_athena_spark_deferrable]
+ :end-before: [END howto_operator_athena_spark_deferrable]
+
+.. note::
+
+ When the trigger reports that polling attempts have been exhausted, the
+ operator requests cancellation of the calculation before failing.
+ Killing the task while it is deferred does not invoke ``on_kill()``,
+ so the calculation may continue running in Athena.
Sensors
-------
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
b/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
index b79c5749107..5a38731af95 100644
---
a/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
+++
b/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
@@ -24,7 +24,9 @@ from typing import TYPE_CHECKING, Any
from airflow.providers.amazon.aws.exceptions import WaiterMaxAttemptsError
from airflow.providers.amazon.aws.hooks.athena import AthenaHook
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
+from airflow.providers.amazon.aws.triggers.athena import AthenaSparkTrigger
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
+from airflow.providers.common.compat.sdk import conf
if TYPE_CHECKING:
from airflow.sdk import Context
@@ -49,6 +51,8 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
:param waiter_delay: Seconds to wait between status checks. Defaults to 30.
:param waiter_max_attempts: Maximum number of polling attempts before
timing out. Defaults to 120.
To limit total task time, use execution_timeout on the task as well.
+ :param deferrable: Whether to wait asynchronously using the triggerer.
+ Defaults to the ``operators.default_deferrable`` configuration value
(False).
:param log_query: Whether to log submission details. Defaults to True.
:param aws_conn_id: The Airflow connection used for AWS credentials.
Defaults to ``aws_default``.
:param region_name: AWS region. If not set, default boto3 behavior is
used. Defaults to None.
@@ -71,6 +75,7 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
client_request_token: str | None = None,
waiter_delay: int = 30,
waiter_max_attempts: int = 120,
+ deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
log_query: bool = True,
aws_conn_id: str | None = "aws_default",
region_name: str | None = None,
@@ -91,6 +96,7 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
self.client_request_token = client_request_token
self.waiter_delay = waiter_delay
self.waiter_max_attempts = waiter_max_attempts
+ self.deferrable = deferrable
self.log_query = log_query
self._calculation_execution_id: str | None = None
@@ -112,6 +118,21 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
self.log.info("Calculation submitted. CalculationExecutionId: %s",
calculation_execution_id)
+ if self.deferrable:
+ self.defer(
+ trigger=AthenaSparkTrigger(
+ calculation_execution_id=calculation_execution_id,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ region_name=self.region_name,
+ verify=self.verify,
+ botocore_config=self.botocore_config,
+ ),
+ method_name="execute_complete",
+ kwargs={"calculation_execution_id": calculation_execution_id},
+ )
+
try:
final_state = self.hook.poll_spark_calculation_status(
calculation_execution_id,
@@ -135,6 +156,34 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
return self._handle_terminal_state(calculation_execution_id,
final_state)
+ def execute_complete(
+ self,
+ context: Context,
+ calculation_execution_id: str,
+ event: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
+ """Handle the trigger result and return calculation metadata."""
+ self._calculation_execution_id = calculation_execution_id
+
+ if event is None or event.get("status") != "success":
+ self._stop_calculation(calculation_execution_id)
+ message = "event is None" if event is None else
event.get("message", "Unknown trigger error.")
+ raise RuntimeError(
+ f"Error waiting for Athena Spark calculation
{calculation_execution_id}: {message}"
+ )
+
+ execution_info =
self.hook.get_spark_calculation_info(calculation_execution_id)
+ final_state = execution_info.get("Status", {}).get("State")
+
+ if final_state not in AthenaHook.SPARK_TERMINAL_STATES:
+ self._stop_calculation(calculation_execution_id)
+ raise RuntimeError(
+ f"Expected a terminal state for calculation "
+ f"{calculation_execution_id}, received {final_state!r}."
+ )
+
+ return self._handle_terminal_state(calculation_execution_id,
final_state)
+
def _stop_calculation(self, calculation_execution_id: str) -> None:
self.log.info("Stopping Athena Spark calculation %s",
calculation_execution_id)
try:
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
b/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
index 6468431127c..d8a3c9301a6 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
@@ -66,3 +66,52 @@ class AthenaTrigger(AwsBaseWaiterTrigger):
verify=self.verify,
config=self.botocore_config,
)
+
+
+class AthenaSparkTrigger(AwsBaseWaiterTrigger):
+ """
+ Wait asynchronously for an Athena Spark calculation to reach a terminal
state.
+
+ :param calculation_execution_id: Calculation execution ID to monitor.
+ :param waiter_delay: Seconds between status checks. Defaults to 30.
+ :param waiter_max_attempts: Maximum number of polling attempts. Defaults
to 120.
+ :param aws_conn_id: Airflow connection used for AWS credentials. Defaults
to ``aws_default``.
+ :param region_name: AWS region. If not set, default boto3 behavior is used.
+ Defaults to None.
+ :param verify: Whether to verify SSL certificates, or the path to a CA
bundle.
+ Defaults to None.
+ :param botocore_config: Optional botocore configuration dict. Defaults to
None.
+ """
+
+ aws_hook_class = AthenaHook
+
+ def __init__(
+ self,
+ *,
+ calculation_execution_id: str,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 120,
+ aws_conn_id: str | None = "aws_default",
+ region_name: str | None = None,
+ verify: bool | str | None = None,
+ botocore_config: dict | None = None,
+ ) -> None:
+ super().__init__(
+ serialized_fields={
+ "calculation_execution_id": calculation_execution_id,
+ },
+ waiter_name="spark_calculation_terminal",
+ waiter_args={
+ "CalculationExecutionId": calculation_execution_id,
+ },
+ failure_message=(f"Error waiting for Athena Spark calculation
{calculation_execution_id}"),
+ status_message=(f"Athena Spark calculation execution ID:
{calculation_execution_id}"),
+ status_queries=["Status.State", "Status.StateChangeReason"],
+ return_value=calculation_execution_id,
+ waiter_delay=waiter_delay,
+ waiter_max_attempts=waiter_max_attempts,
+ aws_conn_id=aws_conn_id,
+ region_name=region_name,
+ verify=verify,
+ botocore_config=botocore_config,
+ )
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
b/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
index 00cf7142f8c..535db97350f 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
+++ b/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
@@ -81,6 +81,31 @@
"argument": "Status.State"
}
]
+ },
+ "spark_calculation_terminal": {
+ "operation": "GetCalculationExecution",
+ "delay": 30,
+ "maxAttempts": 120,
+ "acceptors": [
+ {
+ "matcher": "path",
+ "argument": "Status.State",
+ "expected": "COMPLETED",
+ "state": "success"
+ },
+ {
+ "matcher": "path",
+ "argument": "Status.State",
+ "expected": "FAILED",
+ "state": "success"
+ },
+ {
+ "matcher": "path",
+ "argument": "Status.State",
+ "expected": "CANCELED",
+ "state": "success"
+ }
+ ]
}
}
}
diff --git a/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
b/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
index f479b45347a..32f44502770 100644
--- a/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
+++ b/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
@@ -132,9 +132,21 @@ with DAG(
code_block="print('hello from athena spark')",
waiter_delay=30,
waiter_max_attempts=120,
+ deferrable=False,
)
# [END howto_operator_athena_spark]
+ # [START howto_operator_athena_spark_deferrable]
+ run_spark_calculation_deferrable = AthenaSparkOperator(
+ task_id="run_spark_calculation_deferrable",
+ session_id=idle_session_id,
+ code_block="print('hello from athena spark in deferrable mode')",
+ waiter_delay=30,
+ waiter_max_attempts=120,
+ deferrable=True,
+ )
+ # [END howto_operator_athena_spark_deferrable]
+
calculation_execution_id = start_athena_spark_calculation(idle_session_id)
# [START howto_sensor_athena_spark]
@@ -164,6 +176,7 @@ with DAG(
idle_session_id,
# TEST BODY
run_spark_calculation,
+ run_spark_calculation_deferrable,
calculation_execution_id,
await_spark_calculation,
# TEST TEARDOWN
diff --git
a/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
b/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
index 97d14ba15c0..d097ae169d7 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
@@ -22,10 +22,12 @@ from unittest import mock
import pytest
from moto import mock_aws
+from airflow.exceptions import TaskDeferred
from airflow.models import DAG
from airflow.providers.amazon.aws.exceptions import WaiterMaxAttemptsError
from airflow.providers.amazon.aws.hooks.athena import AthenaHook
from airflow.providers.amazon.aws.operators.athena_spark import
AthenaSparkOperator
+from airflow.providers.amazon.aws.triggers.athena import AthenaSparkTrigger
from tests_common.test_utils.compat import timezone
from unit.amazon.aws.utils.test_template_fields import validate_template_fields
@@ -79,6 +81,7 @@ class TestAthenaSparkOperator:
client_request_token=MOCK_DATA["client_request_token"],
waiter_delay=0,
waiter_max_attempts=3,
+ deferrable=False,
)
self.athena = AthenaSparkOperator(**self.default_op_kwargs,
aws_conn_id=None, dag=self.dag)
@@ -112,6 +115,7 @@ class TestAthenaSparkOperator:
assert self.athena.client_request_token ==
MOCK_DATA["client_request_token"]
assert self.athena.waiter_delay == 0
assert self.athena.waiter_max_attempts == 3
+ assert self.athena.deferrable is False
assert self.athena._calculation_execution_id is None
@mock.patch.object(AthenaHook, "get_spark_calculation_info")
@@ -273,6 +277,222 @@ class TestAthenaSparkOperator:
client_request_token=MOCK_DATA["client_request_token"],
)
+ @mock.patch.object(AthenaHook, "stop_spark_calculation")
+ @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+ @mock.patch.object(AthenaHook, "poll_spark_calculation_status")
+ @mock.patch.object(AthenaHook, "start_spark_calculation",
return_value=ATHENA_CALCULATION_ID)
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_execute_deferrable(
+ self,
+ mock_conn,
+ mock_start_spark_calculation,
+ mock_poll_spark_calculation_status,
+ mock_get_spark_calculation_info,
+ mock_stop_spark_calculation,
+ ):
+ self.athena.deferrable = True
+ self.athena.aws_conn_id = "aws-test-custom-conn"
+ self.athena.region_name = "eu-west-1"
+ self.athena.verify = False
+ self.athena.botocore_config = {"read_timeout": 42}
+
+ with pytest.raises(TaskDeferred) as exc:
+ self.athena.execute({})
+
+ mock_start_spark_calculation.assert_called_once_with(
+ session_id=MOCK_DATA["session_id"],
+ code_block=MOCK_DATA["code_block"],
+ description=None,
+ client_request_token=MOCK_DATA["client_request_token"],
+ )
+ mock_poll_spark_calculation_status.assert_not_called()
+ mock_get_spark_calculation_info.assert_not_called()
+ mock_stop_spark_calculation.assert_not_called()
+
+ deferred = exc.value
+ assert deferred.method_name == "execute_complete"
+ assert deferred.kwargs == {
+ "calculation_execution_id": ATHENA_CALCULATION_ID,
+ }
+ assert isinstance(deferred.trigger, AthenaSparkTrigger)
+ assert deferred.trigger.serialize() == (
+ "airflow.providers.amazon.aws.triggers.athena.AthenaSparkTrigger",
+ {
+ "calculation_execution_id": ATHENA_CALCULATION_ID,
+ "waiter_delay": 0,
+ "waiter_max_attempts": 3,
+ "aws_conn_id": "aws-test-custom-conn",
+ "region_name": "eu-west-1",
+ "verify": False,
+ "botocore_config": {"read_timeout": 42},
+ },
+ )
+
+ @mock.patch.object(AthenaHook, "stop_spark_calculation")
+ @mock.patch.object(AthenaHook, "poll_spark_calculation_status")
+ @mock.patch.object(AthenaHook, "start_spark_calculation")
+ @mock.patch.object(AthenaHook,
"get_spark_calculation_state_change_reason", return_value=None)
+ @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_execute_complete_success(
+ self,
+ mock_conn,
+ mock_get_spark_calculation_info,
+ mock_get_spark_calculation_state_change_reason,
+ mock_start_spark_calculation,
+ mock_poll_spark_calculation_status,
+ mock_stop_spark_calculation,
+ ):
+ submission_time = timezone.datetime(2026, 1, 1)
+ completion_time = timezone.datetime(2026, 1, 1, 0, 1)
+ mock_get_spark_calculation_info.return_value = _calculation_info(
+ "COMPLETED",
+ submission_time=submission_time,
+ completion_time=completion_time,
+ )
+
+ # A resumed operator starts without the ID assigned during execute().
+ assert self.athena._calculation_execution_id is None
+
+ result = self.athena.execute_complete(
+ context={},
+ calculation_execution_id=ATHENA_CALCULATION_ID,
+ event={"status": "success", "value": ATHENA_CALCULATION_ID},
+ )
+
+ assert result == {
+ "calculation_execution_id": ATHENA_CALCULATION_ID,
+ "state": "COMPLETED",
+ "state_change_reason": None,
+ "submission_time": str(submission_time),
+ "completion_time": str(completion_time),
+ "session_id": MOCK_DATA["session_id"],
+ "working_directory": "s3://test-bucket/spark/",
+ "stdout_s3_uri": "s3://test-bucket/spark/stdout",
+ "stderr_s3_uri": "s3://test-bucket/spark/stderr",
+ "result_s3_uri": "s3://test-bucket/spark/results",
+ "result_type": "application/vnd.aws.athena.v1+json",
+ }
+ assert self.athena._calculation_execution_id == ATHENA_CALCULATION_ID
+
mock_get_spark_calculation_state_change_reason.assert_called_once_with(ATHENA_CALCULATION_ID)
+ mock_start_spark_calculation.assert_not_called()
+ mock_poll_spark_calculation_status.assert_not_called()
+ mock_stop_spark_calculation.assert_not_called()
+
+ @pytest.mark.parametrize("state", ["FAILED", "CANCELED"])
+ @mock.patch.object(AthenaHook, "stop_spark_calculation")
+ @mock.patch.object(
+ AthenaHook,
+ "get_spark_calculation_state_change_reason",
+ return_value="Test reason",
+ )
+ @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_execute_complete_terminal_failure(
+ self,
+ mock_conn,
+ mock_get_spark_calculation_info,
+ mock_get_spark_calculation_state_change_reason,
+ mock_stop_spark_calculation,
+ state,
+ ):
+ mock_get_spark_calculation_info.return_value = _calculation_info(state)
+
+ with pytest.raises(
+ RuntimeError,
+ match=(
+ rf"Athena Spark calculation ended in {state}\. "
+ rf"CalculationExecutionId: {ATHENA_CALCULATION_ID}\. "
+ r"Reason: Test reason"
+ ),
+ ):
+ self.athena.execute_complete(
+ context={},
+ calculation_execution_id=ATHENA_CALCULATION_ID,
+ event={"status": "success", "value": ATHENA_CALCULATION_ID},
+ )
+
+
mock_get_spark_calculation_state_change_reason.assert_called_once_with(ATHENA_CALCULATION_ID)
+ mock_stop_spark_calculation.assert_not_called()
+
+ @pytest.mark.parametrize(
+ ("event", "expected_message"),
+ [
+ pytest.param(
+ {"status": "error", "message": "Waiter exhausted"},
+ "Waiter exhausted",
+ id="waiter-error",
+ ),
+ pytest.param(None, "event is None", id="missing-event"),
+ pytest.param({}, "Unknown trigger error", id="missing-status"),
+ ],
+ )
+ @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+ @mock.patch.object(AthenaHook, "stop_spark_calculation")
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_execute_complete_trigger_error(
+ self,
+ mock_conn,
+ mock_stop_spark_calculation,
+ mock_get_spark_calculation_info,
+ event,
+ expected_message,
+ ):
+ with pytest.raises(RuntimeError, match=expected_message):
+ self.athena.execute_complete(
+ context={},
+ calculation_execution_id=ATHENA_CALCULATION_ID,
+ event=event,
+ )
+
+
mock_stop_spark_calculation.assert_called_once_with(ATHENA_CALCULATION_ID)
+ mock_get_spark_calculation_info.assert_not_called()
+
+ @pytest.mark.parametrize("state", ["RUNNING", "UNKNOWN", None])
+ @mock.patch.object(AthenaHook, "get_spark_calculation_state_change_reason")
+ @mock.patch.object(AthenaHook, "stop_spark_calculation")
+ @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_execute_complete_invalid_state(
+ self,
+ mock_conn,
+ mock_get_spark_calculation_info,
+ mock_stop_spark_calculation,
+ mock_get_spark_calculation_state_change_reason,
+ state,
+ ):
+ mock_get_spark_calculation_info.return_value = _calculation_info(state)
+
+ with pytest.raises(RuntimeError, match="Expected a terminal state"):
+ self.athena.execute_complete(
+ context={},
+ calculation_execution_id=ATHENA_CALCULATION_ID,
+ event={"status": "success", "value": ATHENA_CALCULATION_ID},
+ )
+
+
mock_stop_spark_calculation.assert_called_once_with(ATHENA_CALCULATION_ID)
+ mock_get_spark_calculation_state_change_reason.assert_not_called()
+
+ @mock.patch.object(
+ AthenaHook,
+ "stop_spark_calculation",
+ side_effect=RuntimeError("Stop failed"),
+ )
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_execute_complete_stop_failure_preserves_trigger_error(
+ self,
+ mock_conn,
+ mock_stop_spark_calculation,
+ ):
+ with pytest.raises(RuntimeError, match="Waiter exhausted"):
+ self.athena.execute_complete(
+ context={},
+ calculation_execution_id=ATHENA_CALCULATION_ID,
+ event={"status": "error", "message": "Waiter exhausted"},
+ )
+
+
mock_stop_spark_calculation.assert_called_once_with(ATHENA_CALCULATION_ID)
+
@mock.patch.object(AthenaHook, "stop_spark_calculation")
@mock.patch.object(AthenaHook, "get_conn")
def test_on_kill_calls_stop_spark_calculation(self, mock_conn,
mock_stop_spark_calculation):
diff --git a/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
b/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
index 02e0ef92370..1125f2d16bb 100644
--- a/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
+++ b/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
@@ -16,7 +16,10 @@
# under the License.
from __future__ import annotations
-from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger
+import pytest
+
+from airflow.providers.amazon.aws.hooks.athena import AthenaHook
+from airflow.providers.amazon.aws.triggers.athena import AthenaSparkTrigger,
AthenaTrigger
class TestAthenaTrigger:
@@ -40,3 +43,53 @@ class TestAthenaTrigger:
"waiter_max_attempts": 60,
"aws_conn_id": "aws_default",
}
+
+
+class TestAthenaSparkTrigger:
+ @pytest.mark.parametrize(
+ "aws_kwargs",
+ [
+ pytest.param({}, id="defaults"),
+ pytest.param(
+ {
+ "region_name": "eu-west-1",
+ "verify": False,
+ "botocore_config": {"read_timeout": 42},
+ },
+ id="custom-aws-settings",
+ ),
+ ],
+ )
+ def test_serialization(self, aws_kwargs):
+ expected_kwargs = {
+ "calculation_execution_id": "test_calculation_execution_id",
+ "waiter_delay": 30,
+ "waiter_max_attempts": 60,
+ "aws_conn_id": "aws_default",
+ **aws_kwargs,
+ }
+ trigger = AthenaSparkTrigger(**expected_kwargs)
+
+ classpath, kwargs = trigger.serialize()
+
+ assert classpath ==
"airflow.providers.amazon.aws.triggers.athena.AthenaSparkTrigger"
+ assert kwargs == expected_kwargs
+ assert AthenaSparkTrigger(**kwargs).serialize() == (classpath, kwargs)
+
+ def test_waiter_configuration(self):
+ calculation_execution_id = "test_calculation_execution_id"
+
+ trigger = AthenaSparkTrigger(
+ calculation_execution_id=calculation_execution_id,
+ waiter_delay=30,
+ waiter_max_attempts=60,
+ aws_conn_id="aws_default",
+ )
+
+ assert trigger.aws_hook_class is AthenaHook
+ assert trigger.waiter_name == "spark_calculation_terminal"
+ assert trigger.waiter_args == {
+ "CalculationExecutionId": calculation_execution_id,
+ }
+ assert trigger.status_queries == ["Status.State",
"Status.StateChangeReason"]
+ assert trigger.return_value == calculation_execution_id
diff --git a/providers/amazon/tests/unit/amazon/aws/waiters/test_athena.py
b/providers/amazon/tests/unit/amazon/aws/waiters/test_athena.py
new file mode 100644
index 00000000000..776dfc5a0a2
--- /dev/null
+++ b/providers/amazon/tests/unit/amazon/aws/waiters/test_athena.py
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from unittest import mock
+
+import boto3
+import botocore
+import pytest
+
+from airflow.providers.amazon.aws.hooks.athena import AthenaHook
+
+
+class TestAthenaSparkCustomWaiters:
+ def test_calculation_terminal_waiter(self):
+ assert "spark_calculation_terminal" in AthenaHook().list_waiters()
+
+
+class TestAthenaSparkCalculationTerminalWaiter:
+ WAITER_NAME = "spark_calculation_terminal"
+ CALCULATION_EXECUTION_ID = "calc-exec-123"
+
+ @pytest.fixture(autouse=True)
+ def mock_conn(self, monkeypatch):
+ self.client = boto3.client("athena")
+ monkeypatch.setattr(AthenaHook, "conn", self.client)
+
+ @pytest.fixture
+ def mock_get_calculation(self):
+ with mock.patch.object(self.client, "get_calculation_execution") as
mock_getter:
+ yield mock_getter
+
+ @pytest.mark.parametrize("state", ["COMPLETED", "FAILED", "CANCELED"])
+ def test_terminal_state_ends_wait(self, state, mock_get_calculation):
+ mock_get_calculation.return_value = {"Status": {"State": state}}
+
+ AthenaHook().get_waiter(self.WAITER_NAME).wait(
+ CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+ WaiterConfig={"Delay": 0, "MaxAttempts": 1},
+ )
+
+ mock_get_calculation.assert_called_once_with(
+ CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+ )
+
+ @pytest.mark.parametrize(
+ "intermediate",
+ ["CREATING", "CREATED", "QUEUED", "RUNNING", "CANCELING"],
+ )
+ def test_nonterminal_state_retries_then_completes(self, intermediate,
mock_get_calculation):
+ mock_get_calculation.side_effect = [
+ {"Status": {"State": intermediate}},
+ {"Status": {"State": intermediate}},
+ {"Status": {"State": "COMPLETED"}},
+ ]
+
+ AthenaHook().get_waiter(self.WAITER_NAME).wait(
+ CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+ WaiterConfig={"Delay": 0, "MaxAttempts": 3},
+ )
+
+ assert mock_get_calculation.call_args_list == [
+ mock.call(CalculationExecutionId=self.CALCULATION_EXECUTION_ID),
+ mock.call(CalculationExecutionId=self.CALCULATION_EXECUTION_ID),
+ mock.call(CalculationExecutionId=self.CALCULATION_EXECUTION_ID),
+ ]
+
+ def test_running_state_exhausts_attempts(self, mock_get_calculation):
+ mock_get_calculation.return_value = {"Status": {"State": "RUNNING"}}
+
+ with pytest.raises(botocore.exceptions.WaiterError, match="Max
attempts exceeded"):
+ AthenaHook().get_waiter(self.WAITER_NAME).wait(
+ CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+ WaiterConfig={"Delay": 0, "MaxAttempts": 3},
+ )
+
+ assert mock_get_calculation.call_count == 3