This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 3f6d5eef42 `DataprocHook`: Remove deprecated function `submit` (#23389)
3f6d5eef42 is described below
commit 3f6d5eef427f3ea33d0cd342143983f54226bf05
Author: eladkal <[email protected]>
AuthorDate: Sun May 1 13:20:51 2022 +0300
`DataprocHook`: Remove deprecated function `submit` (#23389)
---
airflow/providers/google/CHANGELOG.rst | 2 ++
airflow/providers/google/cloud/hooks/dataproc.py | 24 +---------------------
.../providers/google/cloud/hooks/test_dataproc.py | 9 --------
3 files changed, 3 insertions(+), 32 deletions(-)
diff --git a/airflow/providers/google/CHANGELOG.rst
b/airflow/providers/google/CHANGELOG.rst
index 0630283c7c..8329bdbe7a 100644
--- a/airflow/providers/google/CHANGELOG.rst
+++ b/airflow/providers/google/CHANGELOG.rst
@@ -91,6 +91,8 @@ Breaking changes
* ``DataprocHook``: order of parameters in ``wait_for_job`` function has
changed.
+* ``DataprocHook``: Remove function ``submit``. Please use ``submit_job``
+
* ``DataprocSubmitJobOperator``: order of parameters has changed.
* ``CloudDatastoreImportEntitiesOperator`` : Remove ``xcom_push``. Please use
``BaseOperator.do_xcom_push``
diff --git a/airflow/providers/google/cloud/hooks/dataproc.py
b/airflow/providers/google/cloud/hooks/dataproc.py
index ae5c330d92..784b4dd169 100644
--- a/airflow/providers/google/cloud/hooks/dataproc.py
+++ b/airflow/providers/google/cloud/hooks/dataproc.py
@@ -20,8 +20,7 @@
import time
import uuid
-import warnings
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
from google.api_core.client_options import ClientOptions
from google.api_core.exceptions import ServerError
@@ -767,27 +766,6 @@ class DataprocHook(GoogleBaseHook):
metadata=metadata,
)
- def submit(
- self,
- project_id: str,
- job: dict,
- region: str = 'global',
- job_error_states: Optional[Iterable[str]] = None,
- ) -> None:
- """
- Submits Google Cloud Dataproc job.
-
- :param project_id: The id of Google Cloud Dataproc project.
- :param job: The job to be submitted
- :param region: The region of Google Dataproc cluster.
- :param job_error_states: Job states that should be considered error
states.
- """
- # TODO: Remover one day
- warnings.warn("This method is deprecated. Please use `submit_job`",
DeprecationWarning, stacklevel=2)
- job_object = self.submit_job(region=region, project_id=project_id,
job=job)
- job_id = job_object.reference.job_id
- self.wait_for_job(job_id=job_id, region=region, project_id=project_id)
-
@GoogleBaseHook.fallback_to_default_project_id
def cancel_job(
self,
diff --git a/tests/providers/google/cloud/hooks/test_dataproc.py
b/tests/providers/google/cloud/hooks/test_dataproc.py
index b52039dd2d..2356f3cc00 100644
--- a/tests/providers/google/cloud/hooks/test_dataproc.py
+++ b/tests/providers/google/cloud/hooks/test_dataproc.py
@@ -372,15 +372,6 @@ class TestDataprocHook(unittest.TestCase):
with pytest.raises(TypeError):
self.hook.submit_job(job=JOB, project_id=GCP_PROJECT)
- @mock.patch(DATAPROC_STRING.format("DataprocHook.wait_for_job"))
- @mock.patch(DATAPROC_STRING.format("DataprocHook.submit_job"))
- def test_submit(self, mock_submit_job, mock_wait_for_job):
- mock_submit_job.return_value.reference.job_id = JOB_ID
- with pytest.warns(DeprecationWarning):
- self.hook.submit(project_id=GCP_PROJECT, job=JOB,
region=GCP_LOCATION)
- mock_submit_job.assert_called_once_with(region=GCP_LOCATION,
project_id=GCP_PROJECT, job=JOB)
- mock_wait_for_job.assert_called_once_with(region=GCP_LOCATION,
project_id=GCP_PROJECT, job_id=JOB_ID)
-
@mock.patch(DATAPROC_STRING.format("DataprocHook.get_job_client"))
def test_cancel_job(self, mock_client):
self.hook.cancel_job(region=GCP_LOCATION, job_id=JOB_ID,
project_id=GCP_PROJECT)
