This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a1f9b7de28 Databricks: stop including user names in `list_jobs`
(#40178)
a1f9b7de28 is described below
commit a1f9b7de28a75d1a401bb0053e7b9b703ea88fbb
Author: Stephen Purcell <[email protected]>
AuthorDate: Fri Jun 14 18:54:54 2024 +0200
Databricks: stop including user names in `list_jobs` (#40178)
* Databricks: stop including user names in `list_jobs`
The user's name is not used on the Airflow side, and this argument saves
the lookup, which makes the request faster.
---
airflow/providers/databricks/hooks/databricks.py | 2 ++
.../providers/databricks/hooks/test_databricks.py | 34 ++++++++++++++++++----
2 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/airflow/providers/databricks/hooks/databricks.py
b/airflow/providers/databricks/hooks/databricks.py
index ee5349add0..4d38587e72 100644
--- a/airflow/providers/databricks/hooks/databricks.py
+++ b/airflow/providers/databricks/hooks/databricks.py
@@ -256,6 +256,7 @@ class DatabricksHook(BaseDatabricksHook):
expand_tasks: bool = False,
job_name: str | None = None,
page_token: str | None = None,
+ include_user_names: bool = False,
) -> list[dict[str, Any]]:
"""
List the jobs in the Databricks Job Service.
@@ -275,6 +276,7 @@ class DatabricksHook(BaseDatabricksHook):
payload: dict[str, Any] = {
"limit": limit,
"expand_tasks": expand_tasks,
+ "include_user_names": include_user_names,
}
payload["page_token"] = page_token
if job_name:
diff --git a/tests/providers/databricks/hooks/test_databricks.py
b/tests/providers/databricks/hooks/test_databricks.py
index 0f1d2c242e..7e61036973 100644
--- a/tests/providers/databricks/hooks/test_databricks.py
+++ b/tests/providers/databricks/hooks/test_databricks.py
@@ -912,7 +912,7 @@ class TestDatabricksHook:
mock_requests.get.assert_called_once_with(
list_jobs_endpoint(HOST),
json=None,
- params={"limit": 25, "page_token": "", "expand_tasks": False},
+ params={"limit": 25, "page_token": "", "expand_tasks": False,
"include_user_names": False},
auth=HTTPBasicAuth(LOGIN, PASSWORD),
headers=self.hook.user_agent_header,
timeout=self.hook.timeout_seconds,
@@ -936,7 +936,12 @@ class TestDatabricksHook:
first_call_args = mock_requests.method_calls[0]
assert first_call_args[1][0] == list_jobs_endpoint(HOST)
- assert first_call_args[2]["params"] == {"limit": 25, "page_token": "",
"expand_tasks": False}
+ assert first_call_args[2]["params"] == {
+ "limit": 25,
+ "page_token": "",
+ "expand_tasks": False,
+ "include_user_names": False,
+ }
second_call_args = mock_requests.method_calls[1]
assert second_call_args[1][0] == list_jobs_endpoint(HOST)
@@ -944,6 +949,7 @@ class TestDatabricksHook:
"limit": 25,
"page_token": "PAGETOKEN",
"expand_tasks": False,
+ "include_user_names": False,
}
assert len(jobs) == 2
@@ -959,7 +965,13 @@ class TestDatabricksHook:
mock_requests.get.assert_called_once_with(
list_jobs_endpoint(HOST),
json=None,
- params={"limit": 25, "page_token": "", "expand_tasks": False,
"name": JOB_NAME},
+ params={
+ "limit": 25,
+ "page_token": "",
+ "expand_tasks": False,
+ "include_user_names": False,
+ "name": JOB_NAME,
+ },
auth=HTTPBasicAuth(LOGIN, PASSWORD),
headers=self.hook.user_agent_header,
timeout=self.hook.timeout_seconds,
@@ -978,7 +990,13 @@ class TestDatabricksHook:
mock_requests.get.assert_called_once_with(
list_jobs_endpoint(HOST),
json=None,
- params={"limit": 25, "page_token": "", "expand_tasks": False,
"name": job_name},
+ params={
+ "limit": 25,
+ "page_token": "",
+ "expand_tasks": False,
+ "include_user_names": False,
+ "name": job_name,
+ },
auth=HTTPBasicAuth(LOGIN, PASSWORD),
headers=self.hook.user_agent_header,
timeout=self.hook.timeout_seconds,
@@ -1001,7 +1019,13 @@ class TestDatabricksHook:
mock_requests.get.assert_called_once_with(
list_jobs_endpoint(HOST),
json=None,
- params={"limit": 25, "page_token": "", "expand_tasks": False,
"name": JOB_NAME},
+ params={
+ "limit": 25,
+ "page_token": "",
+ "expand_tasks": False,
+ "include_user_names": False,
+ "name": JOB_NAME,
+ },
auth=HTTPBasicAuth(LOGIN, PASSWORD),
headers=self.hook.user_agent_header,
timeout=self.hook.timeout_seconds,
