This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a92c47b6fa Removing deprecated code in hive provider (#38859)
a92c47b6fa is described below
commit a92c47b6fa06ccd746fdae21919cfc3cab28fa57
Author: Amogh Desai <[email protected]>
AuthorDate: Tue Apr 9 16:17:24 2024 +0530
Removing deprecated code in hive provider (#38859)
* Removing deprecated code in hive provider
* fixing tests and removing property
* removing some more tests
---
airflow/providers/apache/hive/CHANGELOG.rst | 8 +++++
airflow/providers/apache/hive/hooks/hive.py | 21 -----------
airflow/providers/apache/hive/operators/hive.py | 8 -----
.../providers/apache/hive/operators/hive_stats.py | 14 +-------
.../apache/hive/operators/test_hive_stats.py | 42 +---------------------
5 files changed, 10 insertions(+), 83 deletions(-)
diff --git a/airflow/providers/apache/hive/CHANGELOG.rst
b/airflow/providers/apache/hive/CHANGELOG.rst
index 2db27e19fd..8c1e2ac1e5 100644
--- a/airflow/providers/apache/hive/CHANGELOG.rst
+++ b/airflow/providers/apache/hive/CHANGELOG.rst
@@ -37,6 +37,14 @@ Breaking changes
Changed the default value of ``use_beeline`` in hive cli connection to True.
Beeline will be always enabled by default in this connection type.
+Removed deprecated parameter ``authMechanism`` from HiveHook and dependent
operators.
+Use the already existing ``auth_mechanism`` instead in your ``extra``.
+
+Removed the method ``get_hook`` from hive operator. Use the ``hook`` property
instead.
+
+Removed the deprecated ``col_blacklist`` property from
HiveStatsCollectionOperator.
+Please rename it to ``excluded_columns`` instead.
+
7.0.1
.....
diff --git a/airflow/providers/apache/hive/hooks/hive.py
b/airflow/providers/apache/hive/hooks/hive.py
index b3234fee4f..e6bc336938 100644
--- a/airflow/providers/apache/hive/hooks/hive.py
+++ b/airflow/providers/apache/hive/hooks/hive.py
@@ -23,12 +23,9 @@ import re
import socket
import subprocess
import time
-import warnings
from tempfile import NamedTemporaryFile, TemporaryDirectory
from typing import TYPE_CHECKING, Any, Iterable, Mapping
-from airflow.exceptions import AirflowProviderDeprecationWarning
-
if TYPE_CHECKING:
import pandas as pd
@@ -561,15 +558,6 @@ class HiveMetastoreHook(BaseHook):
if not host:
raise AirflowException("Failed to locate the valid server.")
- if "authMechanism" in conn.extra_dejson:
- warnings.warn(
- "The 'authMechanism' option is deprecated. Please use
'auth_mechanism'.",
- AirflowProviderDeprecationWarning,
- stacklevel=2,
- )
- conn.extra_dejson["auth_mechanism"] =
conn.extra_dejson["authMechanism"]
- del conn.extra_dejson["authMechanism"]
-
auth_mechanism = conn.extra_dejson.get("auth_mechanism", "NOSASL")
if conf.get("core", "security") == "kerberos":
@@ -872,15 +860,6 @@ class HiveServer2Hook(DbApiHook):
db = self.get_connection(self.hiveserver2_conn_id) # type: ignore
- if "authMechanism" in db.extra_dejson:
- warnings.warn(
- "The 'authMechanism' option is deprecated. Please use
'auth_mechanism'.",
- AirflowProviderDeprecationWarning,
- stacklevel=2,
- )
- db.extra_dejson["auth_mechanism"] =
db.extra_dejson["authMechanism"]
- del db.extra_dejson["authMechanism"]
-
auth_mechanism = db.extra_dejson.get("auth_mechanism", "NONE")
if auth_mechanism == "NONE" and db.login is None:
# we need to give a username
diff --git a/airflow/providers/apache/hive/operators/hive.py
b/airflow/providers/apache/hive/operators/hive.py
index 398cadce0d..0804b5f57e 100644
--- a/airflow/providers/apache/hive/operators/hive.py
+++ b/airflow/providers/apache/hive/operators/hive.py
@@ -22,10 +22,7 @@ import re
from functools import cached_property
from typing import TYPE_CHECKING, Any, Sequence
-from deprecated.classic import deprecated
-
from airflow.configuration import conf
-from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.models import BaseOperator
from airflow.providers.apache.hive.hooks.hive import HiveCliHook
from airflow.utils import operator_helpers
@@ -131,11 +128,6 @@ class HiveOperator(BaseOperator):
proxy_user=self.proxy_user,
)
- @deprecated(reason="use `hook` property instead.",
category=AirflowProviderDeprecationWarning)
- def get_hook(self) -> HiveCliHook:
- """Get Hive cli hook."""
- return self.hook
-
def prepare_template(self) -> None:
if self.hiveconf_jinja_translate:
self.hql = re.sub(r"(\$\{(hiveconf:)?([ a-zA-Z0-9_]*)\})", r"{{
\g<3> }}", self.hql)
diff --git a/airflow/providers/apache/hive/operators/hive_stats.py
b/airflow/providers/apache/hive/operators/hive_stats.py
index 643a5a4571..a21e81e40d 100644
--- a/airflow/providers/apache/hive/operators/hive_stats.py
+++ b/airflow/providers/apache/hive/operators/hive_stats.py
@@ -18,15 +18,13 @@
from __future__ import annotations
import json
-import warnings
from typing import TYPE_CHECKING, Any, Callable, Sequence
-from airflow.exceptions import AirflowException,
AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
from airflow.models import BaseOperator
from airflow.providers.apache.hive.hooks.hive import HiveMetastoreHook
from airflow.providers.mysql.hooks.mysql import MySqlHook
from airflow.providers.presto.hooks.presto import PrestoHook
-from airflow.utils.types import NOTSET, ArgNotSet
if TYPE_CHECKING:
from airflow.utils.context import Context
@@ -79,18 +77,8 @@ class HiveStatsCollectionOperator(BaseOperator):
mysql_conn_id: str = "airflow_db",
ds: str = "{{ ds }}",
dttm: str = "{{ logical_date.isoformat() }}",
- col_blacklist: list[str] | None | ArgNotSet = NOTSET,
**kwargs: Any,
) -> None:
- if col_blacklist is not NOTSET:
- warnings.warn(
- f"col_blacklist kwarg passed to {self.__class__.__name__} "
- f"(task_id: {kwargs.get('task_id')}) is deprecated, "
- f"please rename it to excluded_columns instead",
- category=AirflowProviderDeprecationWarning,
- stacklevel=2,
- )
- excluded_columns = col_blacklist # type: ignore[assignment]
super().__init__(**kwargs)
self.table = table
self.partition = partition
diff --git a/tests/providers/apache/hive/operators/test_hive_stats.py
b/tests/providers/apache/hive/operators/test_hive_stats.py
index 9d2157ffc3..e419d2da00 100644
--- a/tests/providers/apache/hive/operators/test_hive_stats.py
+++ b/tests/providers/apache/hive/operators/test_hive_stats.py
@@ -23,11 +23,9 @@ from unittest.mock import MagicMock, patch
import pytest
-from airflow.exceptions import AirflowException,
AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
from airflow.providers.apache.hive.operators.hive_stats import
HiveStatsCollectionOperator
from airflow.providers.presto.hooks.presto import PrestoHook
-from airflow.utils import timezone
-from airflow.utils.task_instance_session import
set_current_task_instance_session
from tests.providers.apache.hive import (
DEFAULT_DATE,
DEFAULT_DATE_DS,
@@ -372,41 +370,3 @@ class TestHiveStatsCollectionOperator(TestHiveEnvironment):
"value",
],
)
-
- def test_col_blacklist_deprecation(self):
- warn_message = "col_blacklist kwarg passed to.*task_id:
fake-task-id.*is deprecated"
- with pytest.warns(AirflowProviderDeprecationWarning,
match=warn_message):
- HiveStatsCollectionOperator(
- task_id="fake-task-id",
- table="airflow.static_babynames_partitioned",
- partition={"ds": DEFAULT_DATE_DS},
- col_blacklist=["foo", "bar"],
- )
-
- @pytest.mark.db_test
- @pytest.mark.parametrize(
- "col_blacklist",
- [pytest.param(None, id="none"), pytest.param(["foo", "bar"],
id="list")],
- )
- def test_partial_col_blacklist_deprecation(self, col_blacklist, dag_maker,
session):
- with dag_maker(
- dag_id="test_partial_col_blacklist_deprecation",
- start_date=timezone.datetime(2024, 1, 1),
- session=session,
- ):
- HiveStatsCollectionOperator.partial(
- task_id="fake-task-id",
- partition={"ds": DEFAULT_DATE_DS},
- col_blacklist=col_blacklist,
- excluded_columns=["spam", "egg"],
- ).expand(table=["airflow.table1", "airflow.table2"])
-
- dr = dag_maker.create_dagrun(execution_date=None)
- tis = dr.get_task_instances(session=session)
- with set_current_task_instance_session(session=session):
- warn_message = "col_blacklist kwarg passed to.*task_id:
fake-task-id.*is deprecated"
- for ti in tis:
- with pytest.warns(AirflowProviderDeprecationWarning,
match=warn_message):
- ti.render_templates()
- expected = col_blacklist or []
- assert ti.task.excluded_columns == expected
