This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 970851e2262 Remove real sleeps from slow provider unit tests (#73478)
970851e2262 is described below
commit 970851e2262d8e97899c45c65875fe519cd36a0e
Author: Kaxil Naik <[email protected]>
AuthorDate: Mon Sep 21 20:41:58 2026 +0100
Remove real sleeps from slow provider unit tests (#73478)
These tests exercised retry and polling code paths with the production
backoff left in place, so each one spent 5 to 30 seconds doing nothing.
Together they were about three minutes of wall clock in every provider
test run. Disable the waits where the hook exposes a retry config, patch
the sleep otherwise, and fix two dataprep tests that patched the sleep of
the wrong method.
---
providers/amazon/tests/unit/amazon/aws/hooks/test_batch_client.py | 7 +++----
.../amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py | 7 +++----
providers/amazon/tests/unit/amazon/aws/hooks/test_glue.py | 5 +++--
providers/amazon/tests/unit/amazon/aws/operators/test_ec2.py | 6 ++++++
providers/amazon/tests/unit/amazon/aws/operators/test_ecs.py | 8 ++++++--
.../spark/tests/unit/apache/spark/hooks/test_spark_submit.py | 2 +-
.../tests/unit/cncf/kubernetes/utils/test_pod_manager.py | 4 ++--
providers/dbt/cloud/tests/unit/dbt/cloud/hooks/test_dbt.py | 3 +++
providers/google/tests/unit/google/cloud/hooks/test_dataprep.py | 4 ++--
.../keycloak/tests/unit/keycloak/auth_manager/routes/conftest.py | 2 +-
.../azure/tests/unit/microsoft/azure/triggers/test_powerbi.py | 6 +++++-
.../azure/tests/unit/microsoft/azure/triggers/test_wasb.py | 2 +-
.../tests/unit/snowflake/hooks/test_snowflake_sql_api.py | 8 ++++++--
13 files changed, 42 insertions(+), 22 deletions(-)
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_client.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_client.py
index 80659f07a72..07ff8cfd397 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_client.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_client.py
@@ -18,7 +18,6 @@
from __future__ import annotations
import logging
-import time
from unittest import mock
import botocore.exceptions
@@ -122,9 +121,9 @@ class TestBatchClient:
batch_log_fetcher = mock.Mock(spec=AwsTaskLogFetcher)
mock_get_batch_log_fetcher = mock.Mock(return_value=batch_log_fetcher)
- thread_start = mock.Mock(side_effect=lambda: time.sleep(2))
- thread_stop = mock.Mock(side_effect=lambda: time.sleep(2))
- thread_join = mock.Mock(side_effect=lambda: time.sleep(2))
+ thread_start = mock.Mock()
+ thread_stop = mock.Mock()
+ thread_join = mock.Mock()
with (
mock.patch.object(batch_log_fetcher, "start", thread_start) as
mock_fetcher_start,
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py
index 79053c865b9..a491abcf653 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py
@@ -19,7 +19,6 @@ from __future__ import annotations
import inspect
import itertools
-import time
from unittest import mock
import boto3
@@ -152,9 +151,9 @@ class TestBatchWaiters:
batch_log_fetcher = mock.Mock(spec=AwsTaskLogFetcher)
mock_get_batch_log_fetcher = mock.Mock(return_value=batch_log_fetcher)
- thread_start = mock.Mock(side_effect=lambda: time.sleep(2))
- thread_stop = mock.Mock(side_effect=lambda: time.sleep(2))
- thread_join = mock.Mock(side_effect=lambda: time.sleep(2))
+ thread_start = mock.Mock()
+ thread_stop = mock.Mock()
+ thread_join = mock.Mock()
with (
mock.patch.object(self.batch_waiters, "get_waiter") as
mock_get_waiter,
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_glue.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_glue.py
index 1b3dda0c881..273aab87c09 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_glue.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_glue.py
@@ -27,6 +27,7 @@ import pytest
from botocore.exceptions import ClientError
from moto import mock_aws
from moto.core import DEFAULT_ACCOUNT_ID
+from tenacity import wait_none
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook,
GlueJobHook
@@ -586,7 +587,7 @@ class TestGlueJobHook:
@mock.patch.object(GlueJobHook, "conn")
def test_get_job_state_fails_after_all_retries(self, mock_conn):
"""Test get_job_state raises exception when all retries are
exhausted."""
- hook = GlueJobHook()
+ hook = GlueJobHook(api_retry_args={"wait": wait_none()})
job_name = "test_job"
run_id = "test_run_id"
@@ -649,7 +650,7 @@ class TestGlueJobHook:
@pytest.mark.asyncio
@mock.patch.object(GlueJobHook, "get_async_conn")
async def test_async_get_job_state_fails_after_all_retries(self,
mock_get_async_conn):
- hook = GlueJobHook()
+ hook = GlueJobHook(api_retry_args={"wait": wait_none()})
job_name = "test_job"
run_id = "test_run_id"
diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_ec2.py
b/providers/amazon/tests/unit/amazon/aws/operators/test_ec2.py
index 8faa74c565a..d28dc4e9f83 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_ec2.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_ec2.py
@@ -38,6 +38,12 @@ from unit.amazon.aws.utils.test_template_fields import
validate_template_fields
class BaseEc2TestClass:
+ @pytest.fixture(autouse=True)
+ def _skip_state_poll_sleep(self):
+ # moto flips instance state on the next describe call, so the wait
between polls is pure delay.
+ with mock.patch("airflow.providers.amazon.aws.hooks.ec2.time.sleep"):
+ yield
+
@classmethod
def _get_image_id(cls, hook):
"""Get a valid image id to create an instance."""
diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_ecs.py
b/providers/amazon/tests/unit/amazon/aws/operators/test_ecs.py
index 137ab9c64c1..1b0d710512b 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_ecs.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_ecs.py
@@ -428,7 +428,8 @@ class TestEcsRunTaskOperator(EcsBaseTestCase):
mock_ti = mock.MagicMock()
mock_context = {"ti": mock_ti, "task_instance": mock_ti}
- result = self.ecs.execute(mock_context)
+ with mock.patch("airflow.providers.amazon.aws.operators.ecs.sleep"):
+ result = self.ecs.execute(mock_context)
assert result is None
assert (
@@ -860,7 +861,10 @@ class TestEcsRunTaskOperator(EcsBaseTestCase):
mock_ti = mock.MagicMock()
mock_context = {"ti": mock_ti, "task_instance": mock_ti}
- with pytest.raises(TaskDeferred) as deferred:
+ with (
+ pytest.raises(TaskDeferred) as deferred,
+ mock.patch("airflow.providers.amazon.aws.operators.ecs.sleep"),
+ ):
self.ecs.execute(mock_context)
assert deferred.value.trigger.region_name == "task-region"
diff --git
a/providers/apache/spark/tests/unit/apache/spark/hooks/test_spark_submit.py
b/providers/apache/spark/tests/unit/apache/spark/hooks/test_spark_submit.py
index d25ddbf7f76..0347c473160 100644
--- a/providers/apache/spark/tests/unit/apache/spark/hooks/test_spark_submit.py
+++ b/providers/apache/spark/tests/unit/apache/spark/hooks/test_spark_submit.py
@@ -1631,7 +1631,7 @@ class TestSparkSubmitHook:
succeeded_pod = V1Pod(status=V1PodStatus(phase="Succeeded"))
mock_client.read_namespaced_pod.side_effect = [running_pod,
succeeded_pod]
- with patch.object(hook, "_run_post_submit_commands"):
+ with patch.object(hook, "_run_post_submit_commands"),
patch("time.sleep"):
hook._poll_k8s_driver_via_api()
assert mock_client.delete_namespaced_pod.call_args.args[:2] ==
("spark-app-abc-driver", "mynamespace")
diff --git
a/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/utils/test_pod_manager.py
b/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/utils/test_pod_manager.py
index 45e4b20a84f..5473ca6c291 100644
---
a/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/utils/test_pod_manager.py
+++
b/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/utils/test_pod_manager.py
@@ -1342,8 +1342,8 @@ class TestPodManager:
mock_pod = MagicMock()
mock_container_is_running.return_value = False
mock_container_is_terminated.return_value = False
- with pytest.raises(AirflowException):
- self.pod_manager.await_xcom_sidecar_container_start(pod=mock_pod,
timeout=10, log_interval=5)
+ with pytest.raises(AirflowException), mock.patch("time.sleep"):
+ self.pod_manager.await_xcom_sidecar_container_start(pod=mock_pod,
timeout=0, log_interval=5)
mock_container_is_running.assert_any_call(mock_pod,
"airflow-xcom-sidecar")
mock_container_is_terminated.assert_any_call(mock_pod,
"airflow-xcom-sidecar")
diff --git a/providers/dbt/cloud/tests/unit/dbt/cloud/hooks/test_dbt.py
b/providers/dbt/cloud/tests/unit/dbt/cloud/hooks/test_dbt.py
index 1c573d8a36a..dc2e501e8a0 100644
--- a/providers/dbt/cloud/tests/unit/dbt/cloud/hooks/test_dbt.py
+++ b/providers/dbt/cloud/tests/unit/dbt/cloud/hooks/test_dbt.py
@@ -26,6 +26,7 @@ import aiohttp
import pytest
from requests import exceptions as requests_exceptions
from requests.models import Response
+from tenacity import wait_none
from airflow.models.connection import Connection
from airflow.providers.common.compat.sdk import AirflowException
@@ -1462,6 +1463,8 @@ class TestDbtCloudHook:
self, get_mock, error_factory, retry_qty, retry_delay
):
hook = DbtCloudHook(ACCOUNT_ID_CONN, retry_limit=retry_qty,
retry_delay=retry_delay)
+ # The exponential backoff is not what is under test here; skip the
real waits.
+ hook.retry_args["wait"] = wait_none()
def fail_cm():
cm = AsyncMock()
diff --git a/providers/google/tests/unit/google/cloud/hooks/test_dataprep.py
b/providers/google/tests/unit/google/cloud/hooks/test_dataprep.py
index 2a260b937b6..29a5049f4fd 100644
--- a/providers/google/tests/unit/google/cloud/hooks/test_dataprep.py
+++ b/providers/google/tests/unit/google/cloud/hooks/test_dataprep.py
@@ -270,7 +270,7 @@ class TestGoogleDataprepHook:
side_effect=[mock.MagicMock(), HTTPError()],
)
def test_get_job_group_status_retry_after_success(self, mock_get_request):
- self.hook.run_job_group.retry.sleep = mock.Mock()
+ self.hook.get_job_group_status.retry.sleep = mock.Mock()
self.hook.get_job_group_status(job_group_id=JOB_ID)
assert mock_get_request.call_count == 1
@@ -285,7 +285,7 @@ class TestGoogleDataprepHook:
],
)
def test_get_job_group_status_four_errors(self, mock_get_request):
- self.hook.run_job_group.retry.sleep = mock.Mock()
+ self.hook.get_job_group_status.retry.sleep = mock.Mock()
self.hook.get_job_group_status(job_group_id=JOB_ID)
assert mock_get_request.call_count == 5
diff --git
a/providers/keycloak/tests/unit/keycloak/auth_manager/routes/conftest.py
b/providers/keycloak/tests/unit/keycloak/auth_manager/routes/conftest.py
index e5ea6131155..1e23a9ea53d 100644
--- a/providers/keycloak/tests/unit/keycloak/auth_manager/routes/conftest.py
+++ b/providers/keycloak/tests/unit/keycloak/auth_manager/routes/conftest.py
@@ -69,4 +69,4 @@ def client():
)
),
)
- yield TestClient(create_app(), headers={"Authorization": f"Bearer
{token}"})
+ yield TestClient(app, headers={"Authorization": f"Bearer {token}"})
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_powerbi.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_powerbi.py
index 4b6b4f9c988..a648f1c2f9b 100644
---
a/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_powerbi.py
+++
b/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_powerbi.py
@@ -262,7 +262,11 @@ class TestPowerBITrigger:
mock_get_refresh_details_by_refresh_id.side_effect =
PowerBIDatasetRefreshException("Test exception")
mock_trigger_dataset_refresh.return_value = DATASET_REFRESH_ID
- task = [i async for i in powerbi_trigger.run()]
+ # Skip the exponential backoff between the retried status fetches.
+ with mock.patch(
+
"airflow.providers.microsoft.azure.triggers.powerbi.asyncio.sleep",
new=mock.AsyncMock()
+ ):
+ task = [i async for i in powerbi_trigger.run()]
response = TriggerEvent(
{
"status": "error",
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_wasb.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_wasb.py
index a884d23225e..8186391cc3b 100644
--- a/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_wasb.py
+++ b/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_wasb.py
@@ -34,7 +34,7 @@ TEST_DATA_STORAGE_BLOB_NAME = "test_blob_providers_team.txt"
TEST_DATA_STORAGE_CONTAINER_NAME = "test-container-providers-team"
TEST_DATA_STORAGE_BLOB_PREFIX = TEST_DATA_STORAGE_BLOB_NAME[:10]
TEST_WASB_CONN_ID = "wasb_default"
-POKE_INTERVAL = 5.0
+POKE_INTERVAL = 0.5
@pytest.fixture(autouse=True)
diff --git
a/providers/snowflake/tests/unit/snowflake/hooks/test_snowflake_sql_api.py
b/providers/snowflake/tests/unit/snowflake/hooks/test_snowflake_sql_api.py
index 6f1b7b03904..eb4fc930c6e 100644
--- a/providers/snowflake/tests/unit/snowflake/hooks/test_snowflake_sql_api.py
+++ b/providers/snowflake/tests/unit/snowflake/hooks/test_snowflake_sql_api.py
@@ -1112,7 +1112,9 @@ class TestSnowflakeSqlApiHook:
"""
Test that _make_api_call_with_retries method respects max retry
attempts.
"""
- hook = SnowflakeSqlApiHook(snowflake_conn_id="test_conn")
+ hook = SnowflakeSqlApiHook(
+ snowflake_conn_id="test_conn", api_retry_args={"wait":
tenacity.wait_none()}
+ )
# Mock response that always fails with retryable error
failed_response = mock.MagicMock()
@@ -1498,7 +1500,9 @@ class TestSnowflakeSqlApiHook:
"""
Test that _make_api_call_with_retries_async respects max retry
attempts.
"""
- hook = SnowflakeSqlApiHook(snowflake_conn_id="test_conn")
+ hook = SnowflakeSqlApiHook(
+ snowflake_conn_id="test_conn", api_retry_args={"wait":
tenacity.wait_none()}
+ )
mock_request_429 =
create_async_request_client_response_error(status_code=429)
# Always returns 429
