This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 57c921193b Fix failed tasks are not detected in `AzureBatchHook` 
(#36785)
57c921193b is described below

commit 57c921193b86569f42bb521a356d28d1c4c619f3
Author: ArvidMartensRenson 
<[email protected]>
AuthorDate: Tue Jan 16 10:41:05 2024 +0100

    Fix failed tasks are not detected in `AzureBatchHook` (#36785)
    
    * fix wait_for_job_tasks_to_complete in microsoft-azure provider
    
    * add unit tests for wait_for_all_tasks_to_complete
    
    * fix azure batch hook unit tests
    
    * fix azure batch hook tests
---
 airflow/providers/microsoft/azure/hooks/batch.py   |  4 +-
 .../providers/microsoft/azure/hooks/test_batch.py  | 58 ++++++++++++++++++++--
 2 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/airflow/providers/microsoft/azure/hooks/batch.py 
b/airflow/providers/microsoft/azure/hooks/batch.py
index 7b681b3a9d..2d26cc9f03 100644
--- a/airflow/providers/microsoft/azure/hooks/batch.py
+++ b/airflow/providers/microsoft/azure/hooks/batch.py
@@ -378,7 +378,7 @@ class AzureBatchHook(BaseHook):
         """
         timeout_time = timezone.utcnow() + timedelta(minutes=timeout)
         while timezone.utcnow() < timeout_time:
-            tasks = self.connection.task.list(job_id)
+            tasks = list(self.connection.task.list(job_id))
 
             incomplete_tasks = [task for task in tasks if task.state != 
batch_models.TaskState.completed]
             if not incomplete_tasks:
@@ -386,7 +386,7 @@ class AzureBatchHook(BaseHook):
                 fail_tasks = [
                     task
                     for task in tasks
-                    if task.executionInfo.result == 
batch_models.TaskExecutionResult.failure
+                    if task.execution_info.result == 
batch_models.TaskExecutionResult.failure
                 ]
                 return fail_tasks
             for task in incomplete_tasks:
diff --git a/tests/providers/microsoft/azure/hooks/test_batch.py 
b/tests/providers/microsoft/azure/hooks/test_batch.py
index d9daa2f13d..dd696911b2 100644
--- a/tests/providers/microsoft/azure/hooks/test_batch.py
+++ b/tests/providers/microsoft/azure/hooks/test_batch.py
@@ -190,9 +190,61 @@ class TestAzureBatchHook:
         mock_instance.assert_called_once_with(job_id="myjob", task=task)
 
     @mock.patch(f"{MODULE}.BatchServiceClient")
-    def test_wait_for_all_task_to_complete(self, mock_batch):
-        # TODO: Add test
-        pass
+    def test_wait_for_all_task_to_complete_timeout(self, mock_batch):
+        hook = AzureBatchHook(azure_batch_conn_id=self.test_cloud_conn_id)
+        with pytest.raises(TimeoutError):
+            hook.wait_for_job_tasks_to_complete("myjob", -1)
+
+    @mock.patch(f"{MODULE}.BatchServiceClient")
+    def test_wait_for_all_task_to_complete_all_success(self, mock_batch):
+        hook = AzureBatchHook(azure_batch_conn_id=self.test_cloud_conn_id)
+        hook.connection.task.list.return_value = iter(
+            [
+                batch_models.CloudTask(
+                    id="mytask_1",
+                    execution_info=batch_models.TaskExecutionInformation(
+                        retry_count=0, requeue_count=0, 
result=batch_models.TaskExecutionResult.success
+                    ),
+                    state=batch_models.TaskState.completed,
+                ),
+                batch_models.CloudTask(
+                    id="mytask_2",
+                    execution_info=batch_models.TaskExecutionInformation(
+                        retry_count=0, requeue_count=0, 
result=batch_models.TaskExecutionResult.success
+                    ),
+                    state=batch_models.TaskState.completed,
+                ),
+            ]
+        )
+
+        results = hook.wait_for_job_tasks_to_complete("myjob", 60)
+        assert results == []
+        hook.connection.task.list.assert_called_once_with("myjob")
+
+    @mock.patch(f"{MODULE}.BatchServiceClient")
+    def test_wait_for_all_task_to_complete_failures(self, mock_batch):
+        hook = AzureBatchHook(azure_batch_conn_id=self.test_cloud_conn_id)
+        tasks = [
+            batch_models.CloudTask(
+                id="mytask_1",
+                execution_info=batch_models.TaskExecutionInformation(
+                    retry_count=0, requeue_count=0, 
result=batch_models.TaskExecutionResult.success
+                ),
+                state=batch_models.TaskState.completed,
+            ),
+            batch_models.CloudTask(
+                id="mytask_2",
+                execution_info=batch_models.TaskExecutionInformation(
+                    retry_count=0, requeue_count=0, 
result=batch_models.TaskExecutionResult.failure
+                ),
+                state=batch_models.TaskState.completed,
+            ),
+        ]
+        hook.connection.task.list.return_value = iter(tasks)
+
+        results = hook.wait_for_job_tasks_to_complete("myjob", 60)
+        assert results == [tasks[1]]
+        hook.connection.task.list.assert_called_once_with("myjob")
 
     @mock.patch(f"{MODULE}.BatchServiceClient")
     def test_connection_success(self, mock_batch):

Reply via email to