This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2bac27fd66 Remove deprecations in Connection for Airflow 3 (#41762)
2bac27fd66 is described below
commit 2bac27fd669d3863d41b613b690bd74558ccc9d9
Author: Jens Scheffler <[email protected]>
AuthorDate: Tue Aug 27 15:49:18 2024 +0200
Remove deprecations in Connection for Airflow 3 (#41762)
* Remove deprecations in Connection for Airflow 3
* Fix pytests
---
airflow/models/connection.py | 75 ++------------------
newsfragments/41762.significant.rst | 7 ++
tests/always/test_connection.py | 133 +-----------------------------------
3 files changed, 16 insertions(+), 199 deletions(-)
diff --git a/airflow/models/connection.py b/airflow/models/connection.py
index 65db6d76df..01df162665 100644
--- a/airflow/models/connection.py
+++ b/airflow/models/connection.py
@@ -19,7 +19,6 @@ from __future__ import annotations
import json
import logging
-import warnings
from contextlib import suppress
from json import JSONDecodeError
from typing import Any
@@ -30,7 +29,7 @@ from sqlalchemy import Boolean, Column, Integer, String, Text
from sqlalchemy.orm import declared_attr, reconstructor, synonym
from airflow.configuration import ensure_secrets_loaded
-from airflow.exceptions import AirflowException, AirflowNotFoundException,
RemovedInAirflow3Warning
+from airflow.exceptions import AirflowException, AirflowNotFoundException
from airflow.models.base import ID_LEN, Base
from airflow.models.crypto import get_fernet
from airflow.secrets.cache import SecretCache
@@ -49,12 +48,6 @@ RE_SANITIZE_CONN_ID =
re2.compile(r"^[\w\#\!\(\)\-\.\:\/\\]{1,}$")
CONN_ID_MAX_LEN: int = 250
-def parse_netloc_to_hostname(*args, **kwargs):
- """Do not use, this method is deprecated."""
- warnings.warn("This method is deprecated.", RemovedInAirflow3Warning,
stacklevel=2)
- return _parse_netloc_to_hostname(*args, **kwargs)
-
-
def sanitize_conn_id(conn_id: str | None, max_length=CONN_ID_MAX_LEN) -> str |
None:
r"""
Sanitizes the connection id and allows only specific characters to be
within.
@@ -184,30 +177,19 @@ class Connection(Base, LoggingMixin):
@staticmethod
def _validate_extra(extra, conn_id) -> None:
- """
- Verify that ``extra`` is a JSON-encoded Python dict.
-
- From Airflow 3.0, we should no longer suppress these errors but raise
instead.
- """
+ """Verify that ``extra`` is a JSON-encoded Python dict."""
if extra is None:
return None
try:
extra_parsed = json.loads(extra)
if not isinstance(extra_parsed, dict):
- warnings.warn(
+ raise ValueError(
"Encountered JSON value in `extra` which does not parse as
a dictionary in "
- f"connection {conn_id!r}. From Airflow 3.0, the `extra`
field must contain a JSON "
- "representation of a Python dict.",
- RemovedInAirflow3Warning,
- stacklevel=3,
+ f"connection {conn_id!r}. The `extra` field must contain a
JSON "
+ "representation of a Python dict."
)
except json.JSONDecodeError:
- warnings.warn(
- f"Encountered non-JSON in `extra` field for connection
{conn_id!r}. Support for "
- "non-JSON `extra` will be removed in Airflow 3.0",
- RemovedInAirflow3Warning,
- stacklevel=2,
- )
+ raise ValueError(f"Encountered non-JSON in `extra` field for
connection {conn_id!r}.")
return None
@reconstructor
@@ -216,15 +198,6 @@ class Connection(Base, LoggingMixin):
mask_secret(self.password)
mask_secret(quote(self.password))
- def parse_from_uri(self, **uri):
- """Use uri parameter in constructor, this method is deprecated."""
- warnings.warn(
- "This method is deprecated. Please use uri parameter in
constructor.",
- RemovedInAirflow3Warning,
- stacklevel=2,
- )
- self._parse_from_uri(**uri)
-
@staticmethod
def _normalize_conn_type(conn_type):
if conn_type == "postgresql":
@@ -421,42 +394,6 @@ class Connection(Base, LoggingMixin):
def __repr__(self):
return self.conn_id or ""
- def log_info(self):
- """
- Read each field individually or use the default representation
(`__repr__`).
-
- This method is deprecated.
- """
- warnings.warn(
- "This method is deprecated. You can read each field individually
or "
- "use the default representation (__repr__).",
- RemovedInAirflow3Warning,
- stacklevel=2,
- )
- return (
- f"id: {self.conn_id}. Host: {self.host}, Port: {self.port},
Schema: {self.schema}, "
- f"Login: {self.login}, Password: {'XXXXXXXX' if self.password else
None}, "
- f"extra: {'XXXXXXXX' if self.extra_dejson else None}"
- )
-
- def debug_info(self):
- """
- Read each field individually or use the default representation
(`__repr__`).
-
- This method is deprecated.
- """
- warnings.warn(
- "This method is deprecated. You can read each field individually
or "
- "use the default representation (__repr__).",
- RemovedInAirflow3Warning,
- stacklevel=2,
- )
- return (
- f"id: {self.conn_id}. Host: {self.host}, Port: {self.port},
Schema: {self.schema}, "
- f"Login: {self.login}, Password: {'XXXXXXXX' if self.password else
None}, "
- f"extra: {self.extra_dejson}"
- )
-
def test_connection(self):
"""Calls out get_hook method and executes test_connection method on
that."""
status, message = False, ""
diff --git a/newsfragments/41762.significant.rst
b/newsfragments/41762.significant.rst
new file mode 100644
index 0000000000..7c8f3c8945
--- /dev/null
+++ b/newsfragments/41762.significant.rst
@@ -0,0 +1,7 @@
+Removed a set of deprecations in ``Connection`` from ``airflow.models``.
+
+- Validation of extra fields is now enforcing that JSON values are provided.
If a non-JSON value is provided
+ a ValueError will be raised.
+- Removed utility method ``parse_netloc_to_hostname()``
+- Removed utility method ``parse_from_uri()``.
+- Removed utility method ``log_info()`` and ``log_debug``.
diff --git a/tests/always/test_connection.py b/tests/always/test_connection.py
index 9deef826f6..5e41ddda79 100644
--- a/tests/always/test_connection.py
+++ b/tests/always/test_connection.py
@@ -28,7 +28,7 @@ import pytest
import sqlalchemy
from cryptography.fernet import Fernet
-from airflow.exceptions import AirflowException, RemovedInAirflow3Warning
+from airflow.exceptions import AirflowException
from airflow.hooks.base import BaseHook
from airflow.models import Connection, crypto
from airflow.providers.sqlite.hooks.sqlite import SqliteHook
@@ -341,49 +341,6 @@ class TestConnection:
description="login only",
),
]
- test_from_uri_params_deprecated = [
- UriTestCaseConfig(
-
test_conn_uri="scheme://user:password@host%2Flocation:1234/schema?__extra__=single+value",
- test_conn_attributes=dict(
- conn_type="scheme",
- host="host/location",
- schema="schema",
- login="user",
- password="password",
- port=1234,
- extra="single value",
- ),
- description="with extras single value",
- ),
- UriTestCaseConfig(
- test_conn_uri="scheme://user:password@host%2Flocation:1234/schema?"
- "__extra__=arbitrary+string+%2A%29%2A%24",
- test_conn_attributes=dict(
- conn_type="scheme",
- host="host/location",
- schema="schema",
- login="user",
- password="password",
- port=1234,
- extra="arbitrary string *)*$",
- ),
- description="with extra non-json",
- ),
- UriTestCaseConfig(
- test_conn_uri="scheme://user:password@host%2Flocation:1234/schema?"
- "__extra__=%5B%22list%22%2C+%22of%22%2C+%22values%22%5D",
- test_conn_attributes=dict(
- conn_type="scheme",
- host="host/location",
- schema="schema",
- login="user",
- password="password",
- port=1234,
- extra_dejson=["list", "of", "values"],
- ),
- description="with extras list",
- ),
- ]
@pytest.mark.parametrize("test_config", test_from_uri_params)
def test_connection_from_uri(self, test_config: UriTestCaseConfig):
@@ -407,34 +364,6 @@ class TestConnection:
self.mask_secret.assert_has_calls(expected_calls)
- @pytest.mark.parametrize("test_config", test_from_uri_params_deprecated)
- def test_connection_from_uri_deprecated_extra_type(self, test_config:
UriTestCaseConfig):
- with pytest.warns(RemovedInAirflow3Warning):
- connection = Connection(uri=test_config.test_uri)
- for conn_attr, expected_val in
test_config.test_conn_attributes.items():
- if conn_attr in ("extra", "extra_dejson"):
- with pytest.warns(RemovedInAirflow3Warning):
- actual_val = getattr(connection, conn_attr)
- else:
- actual_val = getattr(connection, conn_attr)
-
- if expected_val is None:
- assert expected_val is None
- if isinstance(expected_val, dict):
- assert expected_val == actual_val
- else:
- assert expected_val == actual_val
-
- expected_calls = []
- if test_config.test_conn_attributes.get("password"):
-
expected_calls.append(mock.call(test_config.test_conn_attributes["password"]))
-
expected_calls.append(mock.call(quote(test_config.test_conn_attributes["password"])))
-
- if test_config.test_conn_attributes.get("extra_dejson"):
-
expected_calls.append(mock.call(test_config.test_conn_attributes["extra_dejson"]))
-
- self.mask_secret.assert_has_calls(expected_calls)
-
@pytest.mark.parametrize("test_config", test_from_uri_params)
def test_connection_get_uri_from_uri(self, test_config: UriTestCaseConfig):
"""
@@ -456,23 +385,6 @@ class TestConnection:
assert connection.schema == new_conn.schema
assert connection.extra_dejson == new_conn.extra_dejson
- @pytest.mark.parametrize("test_config", test_from_uri_params_deprecated)
- def test_connection_get_uri_from_uri_deprecated_extra_type(self,
test_config: UriTestCaseConfig):
- with pytest.warns(RemovedInAirflow3Warning):
- connection = Connection(uri=test_config.test_uri)
- with pytest.warns(RemovedInAirflow3Warning):
- generated_uri = connection.get_uri()
- with pytest.warns(RemovedInAirflow3Warning):
- new_conn = Connection(uri=generated_uri)
- assert connection.conn_type == new_conn.conn_type
- assert connection.login == new_conn.login
- assert connection.password == new_conn.password
- assert connection.host == new_conn.host
- assert connection.port == new_conn.port
- assert connection.schema == new_conn.schema
- with pytest.warns(RemovedInAirflow3Warning):
- assert connection.extra_dejson == new_conn.extra_dejson
-
@pytest.mark.parametrize("test_config", test_from_uri_params)
def test_connection_get_uri_from_conn(self, test_config:
UriTestCaseConfig):
"""
@@ -501,41 +413,6 @@ class TestConnection:
else:
assert actual_val == expected_val
- @pytest.mark.parametrize("test_config", test_from_uri_params_deprecated)
- def test_connection_get_uri_from_conn_deprecated_extra_type(self,
test_config: UriTestCaseConfig):
- """
- This test verifies that if we create conn_1 from attributes (rather
than from URI), and we generate a
- URI, that when we create conn_2 from this URI, we get an equivalent
conn.
- 1. Build conn init params using `test_conn_attributes` and store in
`conn_kwargs`
- 2. Instantiate conn `connection` from `conn_kwargs`.
- 3. Generate uri `get_uri` from this conn.
- 4. Create conn `new_conn` from this uri.
- 5. Verify `new_conn` has same attributes as `connection`.
- """
- conn_kwargs = {}
- for k, v in test_config.test_conn_attributes.items():
- if k == "extra_dejson":
- conn_kwargs.update({"extra": json.dumps(v)})
- else:
- conn_kwargs.update({k: v})
-
- with pytest.warns(RemovedInAirflow3Warning):
- connection = Connection(conn_id="test_conn", **conn_kwargs) #
type: ignore
- with pytest.warns(RemovedInAirflow3Warning):
- gen_uri = connection.get_uri()
- with pytest.warns(RemovedInAirflow3Warning):
- new_conn = Connection(conn_id="test_conn", uri=gen_uri)
- for conn_attr, expected_val in
test_config.test_conn_attributes.items():
- if conn_attr in ("extra", "extra_dejson"):
- with pytest.warns(RemovedInAirflow3Warning):
- actual_val = getattr(new_conn, conn_attr)
- else:
- actual_val = getattr(new_conn, conn_attr)
- if expected_val is None:
- assert actual_val is None
- else:
- assert actual_val == expected_val
-
@pytest.mark.parametrize(
"uri,uri_parts",
[
@@ -670,10 +547,6 @@ class TestConnection:
"""json serialization should support extra stored as object _or_ as
object string representation"""
assert Connection.from_json(extra).extra == expected
- def test_from_json_extra_string(self):
- with pytest.warns(RemovedInAirflow3Warning, match="Support for
non-JSON `extra` will be removed"):
- assert Connection.from_json('{"extra": "hi"}').extra == "hi"
-
@pytest.mark.parametrize(
"val,expected",
[
@@ -899,11 +772,11 @@ class TestConnection:
assert res[1] == "Hook GrpcHook doesn't implement or inherit
test_connection method"
def test_extra_warnings_non_json(self):
- with pytest.warns(DeprecationWarning, match="non-JSON"):
+ with pytest.raises(ValueError, match="non-JSON"):
Connection(conn_id="test_extra", conn_type="none", extra="hi")
def test_extra_warnings_non_dict_json(self):
- with pytest.warns(DeprecationWarning, match="not parse as a
dictionary"):
+ with pytest.raises(ValueError, match="not parse as a dictionary"):
Connection(conn_id="test_extra", conn_type="none", extra='"hi"')
def test_get_uri_no_conn_type(self):
