This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 75d30a6af1 Resolve deprecations in the tests of `Databricks` operators
(#40370)
75d30a6af1 is described below
commit 75d30a6af1ef616a2a73a34598ebc7a8cb88d969
Author: Bora Berke Sahin <[email protected]>
AuthorDate: Sat Jun 22 00:59:31 2024 +0300
Resolve deprecations in the tests of `Databricks` operators (#40370)
---
tests/deprecations_ignore.yml | 13 -----
.../databricks/operators/test_databricks.py | 68 +++++-----------------
2 files changed, 16 insertions(+), 65 deletions(-)
diff --git a/tests/deprecations_ignore.yml b/tests/deprecations_ignore.yml
index c06e5e79ab..61df746fce 100644
--- a/tests/deprecations_ignore.yml
+++ b/tests/deprecations_ignore.yml
@@ -87,19 +87,6 @@
-
tests/providers/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_exception
-
tests/providers/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_resuming_status
-
tests/providers/amazon/aws/triggers/test_redshift_cluster.py::TestRedshiftClusterTrigger::test_redshift_cluster_sensor_trigger_success
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_databricks_run_now_deferrable_operator_failed_before_defer
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_databricks_run_now_deferrable_operator_success_before_defer
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_execute_complete_failure
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_execute_complete_incorrect_event_validation_failure
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_execute_complete_success
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_execute_task_deferred
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksRunNowDeferrableOperator::test_operator_failed_before_defer
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksSubmitRunDeferrableOperator::test_databricks_submit_run_deferrable_operator_failed_before_defer
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksSubmitRunDeferrableOperator::test_databricks_submit_run_deferrable_operator_success_before_defer
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksSubmitRunDeferrableOperator::test_execute_complete_failure
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksSubmitRunDeferrableOperator::test_execute_complete_incorrect_event_validation_failure
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksSubmitRunDeferrableOperator::test_execute_complete_success
--
tests/providers/databricks/operators/test_databricks.py::TestDatabricksSubmitRunDeferrableOperator::test_execute_task_deferred
-
tests/providers/google/common/auth_backend/test_google_openid.py::TestGoogleOpenID::test_success
-
tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_api_resource_configs
-
tests/providers/google/cloud/hooks/test_bigquery.py::TestBigQueryHookMethods::test_api_resource_configs_duplication_warning
diff --git a/tests/providers/databricks/operators/test_databricks.py
b/tests/providers/databricks/operators/test_databricks.py
index 6822bb5369..7ff2295eda 100644
--- a/tests/providers/databricks/operators/test_databricks.py
+++ b/tests/providers/databricks/operators/test_databricks.py
@@ -29,9 +29,7 @@ from airflow.providers.databricks.hooks.databricks import
RunState
from airflow.providers.databricks.operators.databricks import (
DatabricksCreateJobsOperator,
DatabricksNotebookOperator,
- DatabricksRunNowDeferrableOperator,
DatabricksRunNowOperator,
- DatabricksSubmitRunDeferrableOperator,
DatabricksSubmitRunOperator,
DatabricksTaskBaseOperator,
DatabricksTaskOperator,
@@ -978,8 +976,6 @@ class TestDatabricksSubmitRunOperator:
db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
db_mock.get_run.assert_not_called()
-
-class TestDatabricksSubmitRunDeferrableOperator:
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
def test_execute_task_deferred(self, db_mock_class):
"""
@@ -989,7 +985,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK,
}
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID, json=run)
+ op = DatabricksSubmitRunOperator(deferrable=True, task_id=TASK_ID,
json=run)
db_mock = db_mock_class.return_value
db_mock.submit_run.return_value = RUN_ID
db_mock.get_run = make_run_with_state_mock("RUNNING", "RUNNING")
@@ -1007,7 +1003,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
retry_limit=op.databricks_retry_limit,
retry_delay=op.databricks_retry_delay,
retry_args=None,
- caller="DatabricksSubmitRunDeferrableOperator",
+ caller="DatabricksSubmitRunOperator",
)
db_mock.submit_run.assert_called_once_with(expected)
@@ -1029,7 +1025,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
"errors": [],
}
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID, json=run)
+ op = DatabricksSubmitRunOperator(deferrable=True, task_id=TASK_ID,
json=run)
assert op.execute_complete(context=None, event=event) is None
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
@@ -1050,7 +1046,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
"errors": [],
}
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID, json=run)
+ op = DatabricksSubmitRunOperator(deferrable=True, task_id=TASK_ID,
json=run)
with pytest.raises(AirflowException):
op.execute_complete(context=None, event=event)
@@ -1063,7 +1059,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
def test_execute_complete_incorrect_event_validation_failure(self):
event = {"event_id": "no such column"}
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID)
+ op = DatabricksSubmitRunOperator(deferrable=True, task_id=TASK_ID)
with pytest.raises(AirflowException):
op.execute_complete(context=None, event=event)
@@ -1075,7 +1071,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK,
}
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID, json=run)
+ op = DatabricksSubmitRunOperator(deferrable=True, task_id=TASK_ID,
json=run)
db_mock = db_mock_class.return_value
db_mock.submit_run.return_value = RUN_ID
db_mock.get_run = make_run_with_state_mock("TERMINATED", "FAILED")
@@ -1089,7 +1085,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
retry_limit=op.databricks_retry_limit,
retry_delay=op.databricks_retry_delay,
retry_args=None,
- caller="DatabricksSubmitRunDeferrableOperator",
+ caller="DatabricksSubmitRunOperator",
)
db_mock.submit_run.assert_called_once_with(expected)
@@ -1105,7 +1101,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK,
}
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID, json=run)
+ op = DatabricksSubmitRunOperator(deferrable=True, task_id=TASK_ID,
json=run)
db_mock = db_mock_class.return_value
db_mock.submit_run.return_value = RUN_ID
db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
@@ -1119,7 +1115,7 @@ class TestDatabricksSubmitRunDeferrableOperator:
retry_limit=op.databricks_retry_limit,
retry_delay=op.databricks_retry_delay,
retry_args=None,
- caller="DatabricksSubmitRunDeferrableOperator",
+ caller="DatabricksSubmitRunOperator",
)
db_mock.submit_run.assert_called_once_with(expected)
@@ -1618,15 +1614,13 @@ class TestDatabricksRunNowOperator:
db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
db_mock.get_run.assert_not_called()
-
-class TestDatabricksRunNowDeferrableOperator:
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
def test_execute_task_deferred(self, db_mock_class):
"""
Test the execute function in case where the run is successful.
"""
run = {"notebook_params": NOTEBOOK_PARAMS, "notebook_task":
NOTEBOOK_TASK, "jar_params": JAR_PARAMS}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID,
job_id=JOB_ID, json=run)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID, json=run)
db_mock = db_mock_class.return_value
db_mock.run_now.return_value = RUN_ID
db_mock.get_run = make_run_with_state_mock("RUNNING", "RUNNING")
@@ -1670,7 +1664,7 @@ class TestDatabricksRunNowDeferrableOperator:
"errors": [],
}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID,
job_id=JOB_ID, json=run)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID, json=run)
assert op.execute_complete(context=None, event=event) is None
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
@@ -1688,7 +1682,7 @@ class TestDatabricksRunNowDeferrableOperator:
"errors": [],
}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID,
job_id=JOB_ID, json=run)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID, json=run)
with pytest.raises(AirflowException):
op.execute_complete(context=None, event=event)
@@ -1719,7 +1713,7 @@ class TestDatabricksRunNowDeferrableOperator:
"errors": [],
}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID,
job_id=JOB_ID, json=run)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID, json=run)
op.execute_complete(context=None, event=event)
db_mock = db_mock_class.return_value
@@ -1731,46 +1725,16 @@ class TestDatabricksRunNowDeferrableOperator:
def test_execute_complete_incorrect_event_validation_failure(self):
event = {"event_id": "no such column"}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID, job_id=JOB_ID)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID)
with pytest.raises(AirflowException):
op.execute_complete(context=None, event=event)
-
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
-
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksSubmitRunOperator.defer")
- def test_operator_failed_before_defer(self, mock_defer, db_mock_class):
- """Asserts that a task is not deferred when its failed"""
- run = {
- "new_cluster": NEW_CLUSTER,
- "notebook_task": NOTEBOOK_TASK,
- }
- op = DatabricksSubmitRunDeferrableOperator(task_id=TASK_ID, json=run)
- db_mock = db_mock_class.return_value
- db_mock.submit_run.return_value = RUN_ID
- db_mock.get_run = make_run_with_state_mock("TERMINATED", "FAILED")
- op.execute(None)
-
- expected = utils.normalise_json_content(
- {"new_cluster": NEW_CLUSTER, "notebook_task": NOTEBOOK_TASK,
"run_name": TASK_ID}
- )
- db_mock_class.assert_called_once_with(
- DEFAULT_CONN_ID,
- retry_limit=op.databricks_retry_limit,
- retry_delay=op.databricks_retry_delay,
- retry_args=None,
- caller="DatabricksSubmitRunDeferrableOperator",
- )
-
- db_mock.submit_run.assert_called_once_with(expected)
- db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
- assert op.run_id == RUN_ID
- assert not mock_defer.called
-
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksSubmitRunOperator.defer")
def test_databricks_run_now_deferrable_operator_failed_before_defer(self,
mock_defer, db_mock_class):
"""Asserts that a task is not deferred when its failed"""
run = {"notebook_params": NOTEBOOK_PARAMS, "notebook_task":
NOTEBOOK_TASK, "jar_params": JAR_PARAMS}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID,
job_id=JOB_ID, json=run)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID, json=run)
db_mock = db_mock_class.return_value
db_mock.run_now.return_value = RUN_ID
db_mock.get_run = make_run_with_state_mock("TERMINATED", "FAILED")
@@ -1803,7 +1767,7 @@ class TestDatabricksRunNowDeferrableOperator:
def test_databricks_run_now_deferrable_operator_success_before_defer(self,
mock_defer, db_mock_class):
"""Asserts that a task is not deferred when its succeeds"""
run = {"notebook_params": NOTEBOOK_PARAMS, "notebook_task":
NOTEBOOK_TASK, "jar_params": JAR_PARAMS}
- op = DatabricksRunNowDeferrableOperator(task_id=TASK_ID,
job_id=JOB_ID, json=run)
+ op = DatabricksRunNowOperator(deferrable=True, task_id=TASK_ID,
job_id=JOB_ID, json=run)
db_mock = db_mock_class.return_value
db_mock.run_now.return_value = RUN_ID
db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
