amoghrajesh commented on code in PR #68974:
URL: https://github.com/apache/airflow/pull/68974#discussion_r3473882458
##########
providers/databricks/tests/unit/databricks/operators/test_databricks.py:
##########
@@ -1651,6 +1668,161 @@ def
test_inject_parent_job_info_preserves_existing_config(self, db_mock_class, m
assert
submitted["new_cluster"]["spark_conf"]["spark.openlineage.parentJobNamespace"]
== "ns"
[email protected](
+ not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution)
requires Airflow 3.3+"
+)
+class TestDatabricksSubmitRunOperatorDurable:
+ @staticmethod
+ def _context(task_store=None):
+ ctx: dict = {"ti": MagicMock(autospec=True)}
+ if task_store is not None:
+ ctx["task_state_store"] = task_store
+ return ctx
+
+ @staticmethod
+ def _state(lifecycle: str, result: str = ""):
+ return {"state": {"life_cycle_state": lifecycle, "result_state":
result, "state_message": ""}}
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_persists_run_id_to_task_state_store_on_fresh_submit(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
+ task_store.get.return_value = None
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_run_id", RUN_ID)
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_reconnects_to_active_run_without_resubmitting(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ # status check sees the stored run still RUNNING, the poll then sees
it finish.
+ db_mock.get_run.side_effect = [self._state("RUNNING"),
self._state("TERMINATED", "SUCCESS")]
+ task_store = MagicMock()
+ task_store.get.return_value = RUN_ID
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_not_called()
+ task_store.set.assert_not_called()
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_skips_polling_when_stored_run_already_succeeded(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
+ task_store.get.return_value = RUN_ID
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_not_called()
+ db_mock.get_run_page_url.assert_not_called()
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_resubmits_when_stored_run_in_terminal_failure(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ # stored run failed, after fresh resubmit the poll sees success.
+ db_mock.get_run.side_effect = [
+ self._state("TERMINATED", "FAILED"),
+ self._state("TERMINATED", "SUCCESS"),
+ ]
+ task_store = MagicMock()
+ task_store.get.return_value = 999
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_run_id", RUN_ID)
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_durable_false_never_touches_task_state_store(self, db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}, durable=False
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
Review Comment:
Handled in 1cdec79f2a
##########
providers/databricks/tests/unit/databricks/operators/test_databricks.py:
##########
@@ -1651,6 +1668,161 @@ def
test_inject_parent_job_info_preserves_existing_config(self, db_mock_class, m
assert
submitted["new_cluster"]["spark_conf"]["spark.openlineage.parentJobNamespace"]
== "ns"
[email protected](
+ not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution)
requires Airflow 3.3+"
+)
+class TestDatabricksSubmitRunOperatorDurable:
+ @staticmethod
+ def _context(task_store=None):
+ ctx: dict = {"ti": MagicMock(autospec=True)}
+ if task_store is not None:
+ ctx["task_state_store"] = task_store
+ return ctx
+
+ @staticmethod
+ def _state(lifecycle: str, result: str = ""):
+ return {"state": {"life_cycle_state": lifecycle, "result_state":
result, "state_message": ""}}
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_persists_run_id_to_task_state_store_on_fresh_submit(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
+ task_store.get.return_value = None
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_run_id", RUN_ID)
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_reconnects_to_active_run_without_resubmitting(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ # status check sees the stored run still RUNNING, the poll then sees
it finish.
+ db_mock.get_run.side_effect = [self._state("RUNNING"),
self._state("TERMINATED", "SUCCESS")]
+ task_store = MagicMock()
+ task_store.get.return_value = RUN_ID
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_not_called()
+ task_store.set.assert_not_called()
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_skips_polling_when_stored_run_already_succeeded(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
+ task_store.get.return_value = RUN_ID
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_not_called()
+ db_mock.get_run_page_url.assert_not_called()
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_resubmits_when_stored_run_in_terminal_failure(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ # stored run failed, after fresh resubmit the poll sees success.
+ db_mock.get_run.side_effect = [
+ self._state("TERMINATED", "FAILED"),
+ self._state("TERMINATED", "SUCCESS"),
+ ]
+ task_store = MagicMock()
Review Comment:
Handled in 1cdec79f2a
##########
providers/databricks/tests/unit/databricks/operators/test_databricks.py:
##########
@@ -1651,6 +1668,161 @@ def
test_inject_parent_job_info_preserves_existing_config(self, db_mock_class, m
assert
submitted["new_cluster"]["spark_conf"]["spark.openlineage.parentJobNamespace"]
== "ns"
[email protected](
+ not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution)
requires Airflow 3.3+"
+)
+class TestDatabricksSubmitRunOperatorDurable:
+ @staticmethod
+ def _context(task_store=None):
+ ctx: dict = {"ti": MagicMock(autospec=True)}
+ if task_store is not None:
+ ctx["task_state_store"] = task_store
+ return ctx
+
+ @staticmethod
+ def _state(lifecycle: str, result: str = ""):
+ return {"state": {"life_cycle_state": lifecycle, "result_state":
result, "state_message": ""}}
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_persists_run_id_to_task_state_store_on_fresh_submit(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
+ task_store.get.return_value = None
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_run_id", RUN_ID)
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_reconnects_to_active_run_without_resubmitting(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ # status check sees the stored run still RUNNING, the poll then sees
it finish.
+ db_mock.get_run.side_effect = [self._state("RUNNING"),
self._state("TERMINATED", "SUCCESS")]
+ task_store = MagicMock()
+ task_store.get.return_value = RUN_ID
+
+ op.execute(self._context(task_store))
+
+ db_mock.submit_run.assert_not_called()
+ task_store.set.assert_not_called()
+ assert op.run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_skips_polling_when_stored_run_already_succeeded(self,
db_mock_class):
+ op = DatabricksSubmitRunOperator(
+ task_id=TASK_ID, json={"new_cluster": NEW_CLUSTER,
"notebook_task": NOTEBOOK_TASK}
+ )
+ db_mock = db_mock_class.return_value
+ db_mock.get_run = make_run_with_state_mock("TERMINATED", "SUCCESS")
+ task_store = MagicMock()
Review Comment:
Handled in 1cdec79f2a
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
