This is an automated email from the ASF dual-hosted git repository.

shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 53ca1605d23 Add deferrable support to AthenaSparkOperator (#75915)
53ca1605d23 is described below

commit 53ca1605d2351dcb8c7953188ffd7989312d7ce6
Author: SameerMesiah97 <[email protected]>
AuthorDate: Sat Nov 14 16:33:33 2026 +0000

    Add deferrable support to AthenaSparkOperator (#75915)
    
    * Add deferrable support to AthenaSparkOperator
    
    Monitor calculations asynchronously with AthenaSparkTrigger, releasing
    worker slots while waiting. Preserve metadata and failure handling,
    with tests, documentation, and a system test example.
    
    * Add waiter tests for terminal states, continued polling, and 
exhaustion.Simplify trigger event validation, remove the redundant hook 
override, and document the remaining AWS configuration parameters.
---
 .../amazon/docs/operators/athena/athena_spark.rst  |  24 +++
 .../providers/amazon/aws/operators/athena_spark.py |  49 +++++
 .../providers/amazon/aws/triggers/athena.py        |  49 +++++
 .../providers/amazon/aws/waiters/athena.json       |  25 +++
 .../system/amazon/aws/example_athena_spark.py      |  13 ++
 .../unit/amazon/aws/operators/test_athena_spark.py | 220 +++++++++++++++++++++
 .../tests/unit/amazon/aws/triggers/test_athena.py  |  55 +++++-
 .../tests/unit/amazon/aws/waiters/test_athena.py   |  91 +++++++++
 8 files changed, 525 insertions(+), 1 deletion(-)

diff --git a/providers/amazon/docs/operators/athena/athena_spark.rst 
b/providers/amazon/docs/operators/athena/athena_spark.rst
index 260675ba40b..2bbf43f1d7d 100644
--- a/providers/amazon/docs/operators/athena/athena_spark.rst
+++ b/providers/amazon/docs/operators/athena/athena_spark.rst
@@ -53,6 +53,30 @@ this operator, please see the `Sample Dag 
<https://github.com/apache/airflow/blo
     :start-after: [START howto_operator_athena_spark]
     :end-before: [END howto_operator_athena_spark]
 
+Deferrable mode
+~~~~~~~~~~~~~~~
+
+Set ``deferrable=True`` to wait for the calculation asynchronously using the
+Airflow triggerer, releasing the worker slot while the calculation runs.
+This requires a running triggerer and the ``aiobotocore`` package.
+
+The ``waiter_delay`` and ``waiter_max_attempts`` parameters control polling
+in both synchronous and deferrable modes. Both modes return the same execution
+metadata when the calculation completes successfully and raise an exception
+if it fails or is canceled.
+
+.. exampleinclude:: 
/../../amazon/tests/system/amazon/aws/example_athena_spark.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_athena_spark_deferrable]
+    :end-before: [END howto_operator_athena_spark_deferrable]
+
+.. note::
+
+    When the trigger reports that polling attempts have been exhausted, the
+    operator requests cancellation of the calculation before failing.
+    Killing the task while it is deferred does not invoke ``on_kill()``,
+    so the calculation may continue running in Athena.
 
 Sensors
 -------
diff --git 
a/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py 
b/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
index b79c5749107..5a38731af95 100644
--- 
a/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
+++ 
b/providers/amazon/src/airflow/providers/amazon/aws/operators/athena_spark.py
@@ -24,7 +24,9 @@ from typing import TYPE_CHECKING, Any
 from airflow.providers.amazon.aws.exceptions import WaiterMaxAttemptsError
 from airflow.providers.amazon.aws.hooks.athena import AthenaHook
 from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
+from airflow.providers.amazon.aws.triggers.athena import AthenaSparkTrigger
 from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
+from airflow.providers.common.compat.sdk import conf
 
 if TYPE_CHECKING:
     from airflow.sdk import Context
@@ -49,6 +51,8 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
     :param waiter_delay: Seconds to wait between status checks. Defaults to 30.
     :param waiter_max_attempts: Maximum number of polling attempts before 
timing out. Defaults to 120.
         To limit total task time, use execution_timeout on the task as well.
+    :param deferrable: Whether to wait asynchronously using the triggerer.
+        Defaults to the ``operators.default_deferrable`` configuration value 
(False).
     :param log_query: Whether to log submission details. Defaults to True.
     :param aws_conn_id: The Airflow connection used for AWS credentials. 
Defaults to ``aws_default``.
     :param region_name: AWS region. If not set, default boto3 behavior is 
used. Defaults to None.
@@ -71,6 +75,7 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
         client_request_token: str | None = None,
         waiter_delay: int = 30,
         waiter_max_attempts: int = 120,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", 
fallback=False),
         log_query: bool = True,
         aws_conn_id: str | None = "aws_default",
         region_name: str | None = None,
@@ -91,6 +96,7 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
         self.client_request_token = client_request_token
         self.waiter_delay = waiter_delay
         self.waiter_max_attempts = waiter_max_attempts
+        self.deferrable = deferrable
         self.log_query = log_query
         self._calculation_execution_id: str | None = None
 
@@ -112,6 +118,21 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
 
         self.log.info("Calculation submitted. CalculationExecutionId: %s", 
calculation_execution_id)
 
+        if self.deferrable:
+            self.defer(
+                trigger=AthenaSparkTrigger(
+                    calculation_execution_id=calculation_execution_id,
+                    waiter_delay=self.waiter_delay,
+                    waiter_max_attempts=self.waiter_max_attempts,
+                    aws_conn_id=self.aws_conn_id,
+                    region_name=self.region_name,
+                    verify=self.verify,
+                    botocore_config=self.botocore_config,
+                ),
+                method_name="execute_complete",
+                kwargs={"calculation_execution_id": calculation_execution_id},
+            )
+
         try:
             final_state = self.hook.poll_spark_calculation_status(
                 calculation_execution_id,
@@ -135,6 +156,34 @@ class AthenaSparkOperator(AwsBaseOperator[AthenaHook]):
 
         return self._handle_terminal_state(calculation_execution_id, 
final_state)
 
+    def execute_complete(
+        self,
+        context: Context,
+        calculation_execution_id: str,
+        event: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Handle the trigger result and return calculation metadata."""
+        self._calculation_execution_id = calculation_execution_id
+
+        if event is None or event.get("status") != "success":
+            self._stop_calculation(calculation_execution_id)
+            message = "event is None" if event is None else 
event.get("message", "Unknown trigger error.")
+            raise RuntimeError(
+                f"Error waiting for Athena Spark calculation 
{calculation_execution_id}: {message}"
+            )
+
+        execution_info = 
self.hook.get_spark_calculation_info(calculation_execution_id)
+        final_state = execution_info.get("Status", {}).get("State")
+
+        if final_state not in AthenaHook.SPARK_TERMINAL_STATES:
+            self._stop_calculation(calculation_execution_id)
+            raise RuntimeError(
+                f"Expected a terminal state for calculation "
+                f"{calculation_execution_id}, received {final_state!r}."
+            )
+
+        return self._handle_terminal_state(calculation_execution_id, 
final_state)
+
     def _stop_calculation(self, calculation_execution_id: str) -> None:
         self.log.info("Stopping Athena Spark calculation %s", 
calculation_execution_id)
         try:
diff --git 
a/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py 
b/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
index 6468431127c..d8a3c9301a6 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/triggers/athena.py
@@ -66,3 +66,52 @@ class AthenaTrigger(AwsBaseWaiterTrigger):
             verify=self.verify,
             config=self.botocore_config,
         )
+
+
+class AthenaSparkTrigger(AwsBaseWaiterTrigger):
+    """
+    Wait asynchronously for an Athena Spark calculation to reach a terminal 
state.
+
+    :param calculation_execution_id: Calculation execution ID to monitor.
+    :param waiter_delay: Seconds between status checks. Defaults to 30.
+    :param waiter_max_attempts: Maximum number of polling attempts. Defaults 
to 120.
+    :param aws_conn_id: Airflow connection used for AWS credentials. Defaults 
to ``aws_default``.
+    :param region_name: AWS region. If not set, default boto3 behavior is used.
+        Defaults to None.
+    :param verify: Whether to verify SSL certificates, or the path to a CA 
bundle.
+        Defaults to None.
+    :param botocore_config: Optional botocore configuration dict. Defaults to 
None.
+    """
+
+    aws_hook_class = AthenaHook
+
+    def __init__(
+        self,
+        *,
+        calculation_execution_id: str,
+        waiter_delay: int = 30,
+        waiter_max_attempts: int = 120,
+        aws_conn_id: str | None = "aws_default",
+        region_name: str | None = None,
+        verify: bool | str | None = None,
+        botocore_config: dict | None = None,
+    ) -> None:
+        super().__init__(
+            serialized_fields={
+                "calculation_execution_id": calculation_execution_id,
+            },
+            waiter_name="spark_calculation_terminal",
+            waiter_args={
+                "CalculationExecutionId": calculation_execution_id,
+            },
+            failure_message=(f"Error waiting for Athena Spark calculation 
{calculation_execution_id}"),
+            status_message=(f"Athena Spark calculation execution ID: 
{calculation_execution_id}"),
+            status_queries=["Status.State", "Status.StateChangeReason"],
+            return_value=calculation_execution_id,
+            waiter_delay=waiter_delay,
+            waiter_max_attempts=waiter_max_attempts,
+            aws_conn_id=aws_conn_id,
+            region_name=region_name,
+            verify=verify,
+            botocore_config=botocore_config,
+        )
diff --git 
a/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json 
b/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
index 00cf7142f8c..535db97350f 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
+++ b/providers/amazon/src/airflow/providers/amazon/aws/waiters/athena.json
@@ -81,6 +81,31 @@
                 "argument": "Status.State"
                 }
             ]
+        },
+        "spark_calculation_terminal": {
+        "operation": "GetCalculationExecution",
+        "delay": 30,
+        "maxAttempts": 120,
+        "acceptors": [
+            {
+            "matcher": "path",
+            "argument": "Status.State",
+            "expected": "COMPLETED",
+            "state": "success"
+            },
+            {
+            "matcher": "path",
+            "argument": "Status.State",
+            "expected": "FAILED",
+            "state": "success"
+            },
+            {
+            "matcher": "path",
+            "argument": "Status.State",
+            "expected": "CANCELED",
+            "state": "success"
+            }
+            ]
         }
     }
 }
diff --git a/providers/amazon/tests/system/amazon/aws/example_athena_spark.py 
b/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
index f479b45347a..32f44502770 100644
--- a/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
+++ b/providers/amazon/tests/system/amazon/aws/example_athena_spark.py
@@ -132,9 +132,21 @@ with DAG(
         code_block="print('hello from athena spark')",
         waiter_delay=30,
         waiter_max_attempts=120,
+        deferrable=False,
     )
     # [END howto_operator_athena_spark]
 
+    # [START howto_operator_athena_spark_deferrable]
+    run_spark_calculation_deferrable = AthenaSparkOperator(
+        task_id="run_spark_calculation_deferrable",
+        session_id=idle_session_id,
+        code_block="print('hello from athena spark in deferrable mode')",
+        waiter_delay=30,
+        waiter_max_attempts=120,
+        deferrable=True,
+    )
+    # [END howto_operator_athena_spark_deferrable]
+
     calculation_execution_id = start_athena_spark_calculation(idle_session_id)
 
     # [START howto_sensor_athena_spark]
@@ -164,6 +176,7 @@ with DAG(
         idle_session_id,
         # TEST BODY
         run_spark_calculation,
+        run_spark_calculation_deferrable,
         calculation_execution_id,
         await_spark_calculation,
         # TEST TEARDOWN
diff --git 
a/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py 
b/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
index 97d14ba15c0..d097ae169d7 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_athena_spark.py
@@ -22,10 +22,12 @@ from unittest import mock
 import pytest
 from moto import mock_aws
 
+from airflow.exceptions import TaskDeferred
 from airflow.models import DAG
 from airflow.providers.amazon.aws.exceptions import WaiterMaxAttemptsError
 from airflow.providers.amazon.aws.hooks.athena import AthenaHook
 from airflow.providers.amazon.aws.operators.athena_spark import 
AthenaSparkOperator
+from airflow.providers.amazon.aws.triggers.athena import AthenaSparkTrigger
 
 from tests_common.test_utils.compat import timezone
 from unit.amazon.aws.utils.test_template_fields import validate_template_fields
@@ -79,6 +81,7 @@ class TestAthenaSparkOperator:
             client_request_token=MOCK_DATA["client_request_token"],
             waiter_delay=0,
             waiter_max_attempts=3,
+            deferrable=False,
         )
         self.athena = AthenaSparkOperator(**self.default_op_kwargs, 
aws_conn_id=None, dag=self.dag)
 
@@ -112,6 +115,7 @@ class TestAthenaSparkOperator:
         assert self.athena.client_request_token == 
MOCK_DATA["client_request_token"]
         assert self.athena.waiter_delay == 0
         assert self.athena.waiter_max_attempts == 3
+        assert self.athena.deferrable is False
         assert self.athena._calculation_execution_id is None
 
     @mock.patch.object(AthenaHook, "get_spark_calculation_info")
@@ -273,6 +277,222 @@ class TestAthenaSparkOperator:
             client_request_token=MOCK_DATA["client_request_token"],
         )
 
+    @mock.patch.object(AthenaHook, "stop_spark_calculation")
+    @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+    @mock.patch.object(AthenaHook, "poll_spark_calculation_status")
+    @mock.patch.object(AthenaHook, "start_spark_calculation", 
return_value=ATHENA_CALCULATION_ID)
+    @mock.patch.object(AthenaHook, "get_conn")
+    def test_execute_deferrable(
+        self,
+        mock_conn,
+        mock_start_spark_calculation,
+        mock_poll_spark_calculation_status,
+        mock_get_spark_calculation_info,
+        mock_stop_spark_calculation,
+    ):
+        self.athena.deferrable = True
+        self.athena.aws_conn_id = "aws-test-custom-conn"
+        self.athena.region_name = "eu-west-1"
+        self.athena.verify = False
+        self.athena.botocore_config = {"read_timeout": 42}
+
+        with pytest.raises(TaskDeferred) as exc:
+            self.athena.execute({})
+
+        mock_start_spark_calculation.assert_called_once_with(
+            session_id=MOCK_DATA["session_id"],
+            code_block=MOCK_DATA["code_block"],
+            description=None,
+            client_request_token=MOCK_DATA["client_request_token"],
+        )
+        mock_poll_spark_calculation_status.assert_not_called()
+        mock_get_spark_calculation_info.assert_not_called()
+        mock_stop_spark_calculation.assert_not_called()
+
+        deferred = exc.value
+        assert deferred.method_name == "execute_complete"
+        assert deferred.kwargs == {
+            "calculation_execution_id": ATHENA_CALCULATION_ID,
+        }
+        assert isinstance(deferred.trigger, AthenaSparkTrigger)
+        assert deferred.trigger.serialize() == (
+            "airflow.providers.amazon.aws.triggers.athena.AthenaSparkTrigger",
+            {
+                "calculation_execution_id": ATHENA_CALCULATION_ID,
+                "waiter_delay": 0,
+                "waiter_max_attempts": 3,
+                "aws_conn_id": "aws-test-custom-conn",
+                "region_name": "eu-west-1",
+                "verify": False,
+                "botocore_config": {"read_timeout": 42},
+            },
+        )
+
+    @mock.patch.object(AthenaHook, "stop_spark_calculation")
+    @mock.patch.object(AthenaHook, "poll_spark_calculation_status")
+    @mock.patch.object(AthenaHook, "start_spark_calculation")
+    @mock.patch.object(AthenaHook, 
"get_spark_calculation_state_change_reason", return_value=None)
+    @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+    @mock.patch.object(AthenaHook, "get_conn")
+    def test_execute_complete_success(
+        self,
+        mock_conn,
+        mock_get_spark_calculation_info,
+        mock_get_spark_calculation_state_change_reason,
+        mock_start_spark_calculation,
+        mock_poll_spark_calculation_status,
+        mock_stop_spark_calculation,
+    ):
+        submission_time = timezone.datetime(2026, 1, 1)
+        completion_time = timezone.datetime(2026, 1, 1, 0, 1)
+        mock_get_spark_calculation_info.return_value = _calculation_info(
+            "COMPLETED",
+            submission_time=submission_time,
+            completion_time=completion_time,
+        )
+
+        # A resumed operator starts without the ID assigned during execute().
+        assert self.athena._calculation_execution_id is None
+
+        result = self.athena.execute_complete(
+            context={},
+            calculation_execution_id=ATHENA_CALCULATION_ID,
+            event={"status": "success", "value": ATHENA_CALCULATION_ID},
+        )
+
+        assert result == {
+            "calculation_execution_id": ATHENA_CALCULATION_ID,
+            "state": "COMPLETED",
+            "state_change_reason": None,
+            "submission_time": str(submission_time),
+            "completion_time": str(completion_time),
+            "session_id": MOCK_DATA["session_id"],
+            "working_directory": "s3://test-bucket/spark/",
+            "stdout_s3_uri": "s3://test-bucket/spark/stdout",
+            "stderr_s3_uri": "s3://test-bucket/spark/stderr",
+            "result_s3_uri": "s3://test-bucket/spark/results",
+            "result_type": "application/vnd.aws.athena.v1+json",
+        }
+        assert self.athena._calculation_execution_id == ATHENA_CALCULATION_ID
+        
mock_get_spark_calculation_state_change_reason.assert_called_once_with(ATHENA_CALCULATION_ID)
+        mock_start_spark_calculation.assert_not_called()
+        mock_poll_spark_calculation_status.assert_not_called()
+        mock_stop_spark_calculation.assert_not_called()
+
+    @pytest.mark.parametrize("state", ["FAILED", "CANCELED"])
+    @mock.patch.object(AthenaHook, "stop_spark_calculation")
+    @mock.patch.object(
+        AthenaHook,
+        "get_spark_calculation_state_change_reason",
+        return_value="Test reason",
+    )
+    @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+    @mock.patch.object(AthenaHook, "get_conn")
+    def test_execute_complete_terminal_failure(
+        self,
+        mock_conn,
+        mock_get_spark_calculation_info,
+        mock_get_spark_calculation_state_change_reason,
+        mock_stop_spark_calculation,
+        state,
+    ):
+        mock_get_spark_calculation_info.return_value = _calculation_info(state)
+
+        with pytest.raises(
+            RuntimeError,
+            match=(
+                rf"Athena Spark calculation ended in {state}\. "
+                rf"CalculationExecutionId: {ATHENA_CALCULATION_ID}\. "
+                r"Reason: Test reason"
+            ),
+        ):
+            self.athena.execute_complete(
+                context={},
+                calculation_execution_id=ATHENA_CALCULATION_ID,
+                event={"status": "success", "value": ATHENA_CALCULATION_ID},
+            )
+
+        
mock_get_spark_calculation_state_change_reason.assert_called_once_with(ATHENA_CALCULATION_ID)
+        mock_stop_spark_calculation.assert_not_called()
+
+    @pytest.mark.parametrize(
+        ("event", "expected_message"),
+        [
+            pytest.param(
+                {"status": "error", "message": "Waiter exhausted"},
+                "Waiter exhausted",
+                id="waiter-error",
+            ),
+            pytest.param(None, "event is None", id="missing-event"),
+            pytest.param({}, "Unknown trigger error", id="missing-status"),
+        ],
+    )
+    @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+    @mock.patch.object(AthenaHook, "stop_spark_calculation")
+    @mock.patch.object(AthenaHook, "get_conn")
+    def test_execute_complete_trigger_error(
+        self,
+        mock_conn,
+        mock_stop_spark_calculation,
+        mock_get_spark_calculation_info,
+        event,
+        expected_message,
+    ):
+        with pytest.raises(RuntimeError, match=expected_message):
+            self.athena.execute_complete(
+                context={},
+                calculation_execution_id=ATHENA_CALCULATION_ID,
+                event=event,
+            )
+
+        
mock_stop_spark_calculation.assert_called_once_with(ATHENA_CALCULATION_ID)
+        mock_get_spark_calculation_info.assert_not_called()
+
+    @pytest.mark.parametrize("state", ["RUNNING", "UNKNOWN", None])
+    @mock.patch.object(AthenaHook, "get_spark_calculation_state_change_reason")
+    @mock.patch.object(AthenaHook, "stop_spark_calculation")
+    @mock.patch.object(AthenaHook, "get_spark_calculation_info")
+    @mock.patch.object(AthenaHook, "get_conn")
+    def test_execute_complete_invalid_state(
+        self,
+        mock_conn,
+        mock_get_spark_calculation_info,
+        mock_stop_spark_calculation,
+        mock_get_spark_calculation_state_change_reason,
+        state,
+    ):
+        mock_get_spark_calculation_info.return_value = _calculation_info(state)
+
+        with pytest.raises(RuntimeError, match="Expected a terminal state"):
+            self.athena.execute_complete(
+                context={},
+                calculation_execution_id=ATHENA_CALCULATION_ID,
+                event={"status": "success", "value": ATHENA_CALCULATION_ID},
+            )
+
+        
mock_stop_spark_calculation.assert_called_once_with(ATHENA_CALCULATION_ID)
+        mock_get_spark_calculation_state_change_reason.assert_not_called()
+
+    @mock.patch.object(
+        AthenaHook,
+        "stop_spark_calculation",
+        side_effect=RuntimeError("Stop failed"),
+    )
+    @mock.patch.object(AthenaHook, "get_conn")
+    def test_execute_complete_stop_failure_preserves_trigger_error(
+        self,
+        mock_conn,
+        mock_stop_spark_calculation,
+    ):
+        with pytest.raises(RuntimeError, match="Waiter exhausted"):
+            self.athena.execute_complete(
+                context={},
+                calculation_execution_id=ATHENA_CALCULATION_ID,
+                event={"status": "error", "message": "Waiter exhausted"},
+            )
+
+        
mock_stop_spark_calculation.assert_called_once_with(ATHENA_CALCULATION_ID)
+
     @mock.patch.object(AthenaHook, "stop_spark_calculation")
     @mock.patch.object(AthenaHook, "get_conn")
     def test_on_kill_calls_stop_spark_calculation(self, mock_conn, 
mock_stop_spark_calculation):
diff --git a/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py 
b/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
index 02e0ef92370..1125f2d16bb 100644
--- a/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
+++ b/providers/amazon/tests/unit/amazon/aws/triggers/test_athena.py
@@ -16,7 +16,10 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger
+import pytest
+
+from airflow.providers.amazon.aws.hooks.athena import AthenaHook
+from airflow.providers.amazon.aws.triggers.athena import AthenaSparkTrigger, 
AthenaTrigger
 
 
 class TestAthenaTrigger:
@@ -40,3 +43,53 @@ class TestAthenaTrigger:
             "waiter_max_attempts": 60,
             "aws_conn_id": "aws_default",
         }
+
+
+class TestAthenaSparkTrigger:
+    @pytest.mark.parametrize(
+        "aws_kwargs",
+        [
+            pytest.param({}, id="defaults"),
+            pytest.param(
+                {
+                    "region_name": "eu-west-1",
+                    "verify": False,
+                    "botocore_config": {"read_timeout": 42},
+                },
+                id="custom-aws-settings",
+            ),
+        ],
+    )
+    def test_serialization(self, aws_kwargs):
+        expected_kwargs = {
+            "calculation_execution_id": "test_calculation_execution_id",
+            "waiter_delay": 30,
+            "waiter_max_attempts": 60,
+            "aws_conn_id": "aws_default",
+            **aws_kwargs,
+        }
+        trigger = AthenaSparkTrigger(**expected_kwargs)
+
+        classpath, kwargs = trigger.serialize()
+
+        assert classpath == 
"airflow.providers.amazon.aws.triggers.athena.AthenaSparkTrigger"
+        assert kwargs == expected_kwargs
+        assert AthenaSparkTrigger(**kwargs).serialize() == (classpath, kwargs)
+
+    def test_waiter_configuration(self):
+        calculation_execution_id = "test_calculation_execution_id"
+
+        trigger = AthenaSparkTrigger(
+            calculation_execution_id=calculation_execution_id,
+            waiter_delay=30,
+            waiter_max_attempts=60,
+            aws_conn_id="aws_default",
+        )
+
+        assert trigger.aws_hook_class is AthenaHook
+        assert trigger.waiter_name == "spark_calculation_terminal"
+        assert trigger.waiter_args == {
+            "CalculationExecutionId": calculation_execution_id,
+        }
+        assert trigger.status_queries == ["Status.State", 
"Status.StateChangeReason"]
+        assert trigger.return_value == calculation_execution_id
diff --git a/providers/amazon/tests/unit/amazon/aws/waiters/test_athena.py 
b/providers/amazon/tests/unit/amazon/aws/waiters/test_athena.py
new file mode 100644
index 00000000000..776dfc5a0a2
--- /dev/null
+++ b/providers/amazon/tests/unit/amazon/aws/waiters/test_athena.py
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from unittest import mock
+
+import boto3
+import botocore
+import pytest
+
+from airflow.providers.amazon.aws.hooks.athena import AthenaHook
+
+
+class TestAthenaSparkCustomWaiters:
+    def test_calculation_terminal_waiter(self):
+        assert "spark_calculation_terminal" in AthenaHook().list_waiters()
+
+
+class TestAthenaSparkCalculationTerminalWaiter:
+    WAITER_NAME = "spark_calculation_terminal"
+    CALCULATION_EXECUTION_ID = "calc-exec-123"
+
+    @pytest.fixture(autouse=True)
+    def mock_conn(self, monkeypatch):
+        self.client = boto3.client("athena")
+        monkeypatch.setattr(AthenaHook, "conn", self.client)
+
+    @pytest.fixture
+    def mock_get_calculation(self):
+        with mock.patch.object(self.client, "get_calculation_execution") as 
mock_getter:
+            yield mock_getter
+
+    @pytest.mark.parametrize("state", ["COMPLETED", "FAILED", "CANCELED"])
+    def test_terminal_state_ends_wait(self, state, mock_get_calculation):
+        mock_get_calculation.return_value = {"Status": {"State": state}}
+
+        AthenaHook().get_waiter(self.WAITER_NAME).wait(
+            CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+            WaiterConfig={"Delay": 0, "MaxAttempts": 1},
+        )
+
+        mock_get_calculation.assert_called_once_with(
+            CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+        )
+
+    @pytest.mark.parametrize(
+        "intermediate",
+        ["CREATING", "CREATED", "QUEUED", "RUNNING", "CANCELING"],
+    )
+    def test_nonterminal_state_retries_then_completes(self, intermediate, 
mock_get_calculation):
+        mock_get_calculation.side_effect = [
+            {"Status": {"State": intermediate}},
+            {"Status": {"State": intermediate}},
+            {"Status": {"State": "COMPLETED"}},
+        ]
+
+        AthenaHook().get_waiter(self.WAITER_NAME).wait(
+            CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+            WaiterConfig={"Delay": 0, "MaxAttempts": 3},
+        )
+
+        assert mock_get_calculation.call_args_list == [
+            mock.call(CalculationExecutionId=self.CALCULATION_EXECUTION_ID),
+            mock.call(CalculationExecutionId=self.CALCULATION_EXECUTION_ID),
+            mock.call(CalculationExecutionId=self.CALCULATION_EXECUTION_ID),
+        ]
+
+    def test_running_state_exhausts_attempts(self, mock_get_calculation):
+        mock_get_calculation.return_value = {"Status": {"State": "RUNNING"}}
+
+        with pytest.raises(botocore.exceptions.WaiterError, match="Max 
attempts exceeded"):
+            AthenaHook().get_waiter(self.WAITER_NAME).wait(
+                CalculationExecutionId=self.CALCULATION_EXECUTION_ID,
+                WaiterConfig={"Delay": 0, "MaxAttempts": 3},
+            )
+
+        assert mock_get_calculation.call_count == 3

Reply via email to