This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2bf3af0a656 Remove deprecated hook code from plugins (#43291)
2bf3af0a656 is described below
commit 2bf3af0a65631bb6b9271fca8a0c6724c996393c
Author: Kaxil Naik <[email protected]>
AuthorDate: Wed Oct 23 17:24:51 2024 +0100
Remove deprecated hook code from plugins (#43291)
We had removed registering hooks via plugin in
https://github.com/apache/airflow/pull/12108 for Airflow 2.0. We kept it for
registering Connection form.
As I understand it, we don't use that code but `ProvidersManager`
(`airflow/providers_manager.py`) to register Connections from providers.
---
.../api_fastapi/core_api/openapi/v1-generated.yaml | 6 -----
.../api_fastapi/core_api/serializers/plugins.py | 1 -
airflow/plugins_manager.py | 14 ++---------
airflow/ui/openapi-gen/requests/schemas.gen.ts | 8 ------
airflow/ui/openapi-gen/requests/types.gen.ts | 1 -
.../authoring-and-scheduling/plugins.rst | 8 ------
newsfragments/43291.significant.rst | 16 ++++++++++++
tests/always/test_connection.py | 29 ----------------------
.../endpoints/test_plugin_endpoint.py | 6 -----
tests/api_connexion/schemas/test_plugin_schema.py | 8 ------
tests/cli/commands/test_plugins_command.py | 20 +++++++++------
tests/plugins/test_plugin.py | 9 -------
tests/plugins/test_plugins_manager.py | 24 ------------------
tests_common/test_utils/mock_plugins.py | 2 --
14 files changed, 30 insertions(+), 122 deletions(-)
diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
index 3d890d4da31..5eb7aed7705 100644
--- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
+++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
@@ -2207,11 +2207,6 @@ components:
name:
type: string
title: Name
- hooks:
- items:
- type: string
- type: array
- title: Hooks
macros:
items:
type: string
@@ -2268,7 +2263,6 @@ components:
type: object
required:
- name
- - hooks
- macros
- flask_blueprints
- fastapi_apps
diff --git a/airflow/api_fastapi/core_api/serializers/plugins.py
b/airflow/api_fastapi/core_api/serializers/plugins.py
index 68bc8ea443c..e16b56a6aca 100644
--- a/airflow/api_fastapi/core_api/serializers/plugins.py
+++ b/airflow/api_fastapi/core_api/serializers/plugins.py
@@ -64,7 +64,6 @@ class PluginResponse(BaseModel):
"""Plugin serializer."""
name: str
- hooks: list[str]
macros: list[str]
flask_blueprints: list[str]
fastapi_apps: list[FastAPIAppResponse]
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index fc7adc5993f..881e07a81ed 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -50,7 +50,6 @@ if TYPE_CHECKING:
from types import ModuleType
from typing import Generator
- from airflow.hooks.base import BaseHook
from airflow.listeners.listener import ListenerManager
from airflow.timetables.base import Timetable
@@ -62,7 +61,6 @@ plugins: list[AirflowPlugin] | None = None
loaded_plugins: set[str] = set()
# Plugin components to integrate as modules
-registered_hooks: list[BaseHook] | None = None
macros_modules: list[Any] | None = None
# Plugin components to integrate directly
@@ -86,7 +84,6 @@ Used by the DAG serialization code to only allow specific
classes to be created
during deserialization
"""
PLUGINS_ATTRIBUTES_TO_DUMP = {
- "hooks",
"macros",
"admin_views",
"flask_blueprints",
@@ -151,7 +148,6 @@ class AirflowPlugin:
name: str | None = None
source: AirflowPluginSource | None = None
- hooks: list[Any] = []
macros: list[Any] = []
admin_views: list[Any] = []
flask_blueprints: list[Any] = []
@@ -345,7 +341,7 @@ def ensure_plugins_loaded():
"""
from airflow.stats import Stats
- global plugins, registered_hooks
+ global plugins
if plugins is not None:
log.debug("Plugins are already loaded. Skipping.")
@@ -358,7 +354,6 @@ def ensure_plugins_loaded():
with Stats.timer() as timer:
plugins = []
- registered_hooks = []
load_plugins_from_plugin_directory()
load_entrypoint_plugins()
@@ -366,11 +361,6 @@ def ensure_plugins_loaded():
if not settings.LAZY_LOAD_PROVIDERS:
load_providers_plugins()
- # We don't do anything with these for now, but we want to keep track of
- # them so we can integrate them in to the UI's Connection screens
- for plugin in plugins:
- registered_hooks.extend(plugin.hooks)
-
if plugins:
log.debug("Loading %d plugin(s) took %.2f seconds", len(plugins),
timer.duration)
@@ -598,7 +588,7 @@ def get_plugin_info(attrs_to_dump: Iterable[str] | None =
None) -> list[dict[str
for attr in attrs_to_dump:
if attr in ("global_operator_extra_links",
"operator_extra_links"):
info[attr] = [f"<{qualname(d.__class__)} object>" for d in
getattr(plugin, attr)]
- elif attr in ("macros", "timetables", "hooks",
"priority_weight_strategies"):
+ elif attr in ("macros", "timetables",
"priority_weight_strategies"):
info[attr] = [qualname(d) for d in getattr(plugin, attr)]
elif attr == "listeners":
# listeners may be modules or class instances
diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts
b/airflow/ui/openapi-gen/requests/schemas.gen.ts
index 356814d15d5..afd16d80409 100644
--- a/airflow/ui/openapi-gen/requests/schemas.gen.ts
+++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts
@@ -1326,13 +1326,6 @@ export const $PluginResponse = {
type: "string",
title: "Name",
},
- hooks: {
- items: {
- type: "string",
- },
- type: "array",
- title: "Hooks",
- },
macros: {
items: {
type: "string",
@@ -1411,7 +1404,6 @@ export const $PluginResponse = {
type: "object",
required: [
"name",
- "hooks",
"macros",
"flask_blueprints",
"fastapi_apps",
diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts
b/airflow/ui/openapi-gen/requests/types.gen.ts
index 40054121470..aef0f76f28e 100644
--- a/airflow/ui/openapi-gen/requests/types.gen.ts
+++ b/airflow/ui/openapi-gen/requests/types.gen.ts
@@ -310,7 +310,6 @@ export type PluginCollectionResponse = {
*/
export type PluginResponse = {
name: string;
- hooks: Array<string>;
macros: Array<string>;
flask_blueprints: Array<string>;
fastapi_apps: Array<FastAPIAppResponse>;
diff --git a/docs/apache-airflow/authoring-and-scheduling/plugins.rst
b/docs/apache-airflow/authoring-and-scheduling/plugins.rst
index 626f20eab1b..a46fe329da1 100644
--- a/docs/apache-airflow/authoring-and-scheduling/plugins.rst
+++ b/docs/apache-airflow/authoring-and-scheduling/plugins.rst
@@ -108,8 +108,6 @@ looks like:
class AirflowPlugin:
# The name of your plugin (str)
name = None
- # A list of class(es) derived from BaseHook
- hooks = []
# A list of references to inject into the macros namespace
macros = []
# A list of Blueprint object created from flask.Blueprint. For use
with the flask_appbuilder based GUI
@@ -182,11 +180,6 @@ definitions in Airflow.
from airflow.providers.amazon.aws.transfers.gcs_to_s3 import
GCSToS3Operator
- # Will show up in Connections screen in a future version
- class PluginHook(BaseHook):
- pass
-
-
# Will show up under airflow.macros.test_plugin.plugin_macro
# and in templates through {{ macros.test_plugin.plugin_macro }}
def plugin_macro():
@@ -267,7 +260,6 @@ definitions in Airflow.
# Defining the plugin class
class AirflowTestPlugin(AirflowPlugin):
name = "test_plugin"
- hooks = [PluginHook]
macros = [plugin_macro]
flask_blueprints = [bp]
fastapi_apps = [app_with_metadata]
diff --git a/newsfragments/43291.significant.rst
b/newsfragments/43291.significant.rst
new file mode 100644
index 00000000000..ec7cacf6f15
--- /dev/null
+++ b/newsfragments/43291.significant.rst
@@ -0,0 +1,16 @@
+Support for adding Hooks via Airflow Plugins is removed
+
+Hooks should no longer be registered or imported via Airflow's plugin
mechanism -- these types of classes
+are just treated as plain Python classes by Airflow, so there is no need to
register them with Airflow.
+
+Before:
+
+.. code-block:: python
+
+ from airflow.hooks.my_plugin import MyHook
+
+You should instead import it as:
+
+.. code-block:: python
+
+ from my_plugin import MyHook
diff --git a/tests/always/test_connection.py b/tests/always/test_connection.py
index 03a8e136f2a..3cc24b6d9ce 100644
--- a/tests/always/test_connection.py
+++ b/tests/always/test_connection.py
@@ -32,7 +32,6 @@ from airflow.exceptions import AirflowException
from airflow.hooks.base import BaseHook
from airflow.models import Connection, crypto
from airflow.providers.sqlite.hooks.sqlite import SqliteHook
-from airflow.providers_manager import HookInfo
from tests_common.test_utils.config import conf_vars
@@ -835,31 +834,3 @@ class TestConnection:
assert restored_conn.schema == conn.schema
assert restored_conn.port == conn.port
assert restored_conn.extra_dejson == conn.extra_dejson
-
- def test_get_hook_not_found(self):
- with mock.patch("airflow.providers_manager.ProvidersManager") as m:
- m.return_value.hooks = {}
- with pytest.raises(AirflowException, match='Unknown hook type
"awesome-test-conn-type"'):
- Connection(conn_type="awesome-test-conn-type").get_hook()
-
- def test_get_hook_import_error(self, caplog):
- with mock.patch("airflow.providers_manager.ProvidersManager") as m:
- m.return_value.hooks = {
- "awesome-test-conn-type": HookInfo(
- hook_class_name="foo.bar.AwesomeTest",
- connection_id_attribute_name="conn-id",
- package_name="test.package",
- hook_name="Awesome Connection",
- connection_type="awesome-test-conn-type",
- connection_testable=False,
- )
- }
- caplog.clear()
- caplog.set_level("ERROR", "airflow.models.connection")
- with pytest.raises(ImportError):
- Connection(conn_type="awesome-test-conn-type").get_hook()
-
- assert caplog.messages
- assert caplog.messages[0] == (
- "Could not import foo.bar.AwesomeTest when discovering Awesome
Connection test.package"
- )
diff --git a/tests/api_connexion/endpoints/test_plugin_endpoint.py
b/tests/api_connexion/endpoints/test_plugin_endpoint.py
index 487ba53a300..2f24347bef6 100644
--- a/tests/api_connexion/endpoints/test_plugin_endpoint.py
+++ b/tests/api_connexion/endpoints/test_plugin_endpoint.py
@@ -23,7 +23,6 @@ from fastapi import FastAPI
from flask import Blueprint
from flask_appbuilder import BaseView
-from airflow.hooks.base import BaseHook
from airflow.plugins_manager import AirflowPlugin
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
from airflow.timetables.base import Timetable
@@ -37,9 +36,6 @@ from tests_common.test_utils.mock_plugins import
mock_plugin_manager
pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode]
-class PluginHook(BaseHook): ...
-
-
def plugin_macro(): ...
@@ -100,7 +96,6 @@ class MockPlugin(AirflowPlugin):
appbuilder_menu_items = [appbuilder_menu_items]
global_operator_extra_links = [MockOperatorLink()]
operator_extra_links = [MockOperatorLink()]
- hooks = [PluginHook]
macros = [plugin_macro]
ti_deps = [ti_dep]
timetables = [CustomTimetable]
@@ -156,7 +151,6 @@ class TestGetPlugins(TestPluginsEndpoint):
}
],
"global_operator_extra_links":
[f"<{qualname(MockOperatorLink().__class__)} object>"],
- "hooks": [qualname(PluginHook)],
"macros": [qualname(plugin_macro)],
"operator_extra_links":
[f"<{qualname(MockOperatorLink().__class__)} object>"],
"source": None,
diff --git a/tests/api_connexion/schemas/test_plugin_schema.py
b/tests/api_connexion/schemas/test_plugin_schema.py
index 951933e9ffc..d722b52977a 100644
--- a/tests/api_connexion/schemas/test_plugin_schema.py
+++ b/tests/api_connexion/schemas/test_plugin_schema.py
@@ -26,15 +26,11 @@ from airflow.api_connexion.schemas.plugin_schema import (
plugin_collection_schema,
plugin_schema,
)
-from airflow.hooks.base import BaseHook
from airflow.plugins_manager import AirflowPlugin
from tests_common.test_utils.compat import BaseOperatorLink
-class PluginHook(BaseHook): ...
-
-
def plugin_macro(): ...
@@ -67,7 +63,6 @@ class MockPlugin(AirflowPlugin):
appbuilder_menu_items = [appbuilder_menu_items]
global_operator_extra_links = [MockOperatorLink()]
operator_extra_links = [MockOperatorLink()]
- hooks = [PluginHook]
macros = [plugin_macro]
@@ -91,7 +86,6 @@ class TestPluginSchema(TestPluginBase):
{"app": app, "name": "App name", "url_prefix": "/some_prefix"},
],
"global_operator_extra_links": [str(MockOperatorLink())],
- "hooks": [str(PluginHook)],
"macros": [str(plugin_macro)],
"operator_extra_links": [str(MockOperatorLink())],
"source": None,
@@ -117,7 +111,6 @@ class TestPluginCollectionSchema(TestPluginBase):
{"app": app, "name": "App name", "url_prefix":
"/some_prefix"},
],
"global_operator_extra_links": [str(MockOperatorLink())],
- "hooks": [str(PluginHook)],
"macros": [str(plugin_macro)],
"operator_extra_links": [str(MockOperatorLink())],
"source": None,
@@ -134,7 +127,6 @@ class TestPluginCollectionSchema(TestPluginBase):
{"app": app, "name": "App name", "url_prefix":
"/some_prefix"},
],
"global_operator_extra_links": [str(MockOperatorLink())],
- "hooks": [str(PluginHook)],
"macros": [str(plugin_macro)],
"operator_extra_links": [str(MockOperatorLink())],
"source": None,
diff --git a/tests/cli/commands/test_plugins_command.py
b/tests/cli/commands/test_plugins_command.py
index c9807520e4e..993d4085fe1 100644
--- a/tests/cli/commands/test_plugins_command.py
+++ b/tests/cli/commands/test_plugins_command.py
@@ -25,8 +25,8 @@ import pytest
from airflow.cli import cli_parser
from airflow.cli.commands import plugins_command
-from airflow.hooks.base import BaseHook
from airflow.listeners.listener import get_listener_manager
+from airflow.models.baseoperatorlink import BaseOperatorLink
from airflow.plugins_manager import AirflowPlugin
from tests.plugins.test_plugin import AirflowTestPlugin as ComplexAirflowPlugin
@@ -35,13 +35,18 @@ from tests_common.test_utils.mock_plugins import
mock_plugin_manager
pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode]
-class PluginHook(BaseHook):
- pass
+class AirflowNewLink(BaseOperatorLink):
+ """Operator Link for Apache Airflow Website."""
+
+ name = "airflowtestlink"
+
+ def get_link(self, operator, *, ti_key):
+ return "https://airflow.apache.org";
class TestPlugin(AirflowPlugin):
name = "test-plugin-cli"
- hooks = [PluginHook]
+ global_operator_extra_links = [AirflowNewLink()]
class TestPluginsCommand:
@@ -98,7 +103,6 @@ class TestPluginsCommand:
"<tests_common.test_utils.mock_operators.CustomOpLink
object>",
"<tests_common.test_utils.mock_operators.CustomBaseIndexOpLink object>",
],
- "hooks": ["tests.plugins.test_plugin.PluginHook"],
"listeners": [
"tests.listeners.empty_listener",
"tests.listeners.class_listener.ClassBasedListener",
@@ -129,9 +133,9 @@ class TestPluginsCommand:
# Assert that only columns with values are displayed
expected_output = textwrap.dedent(
"""\
- name | hooks
-
================+===================================================
- test-plugin-cli |
tests.cli.commands.test_plugins_command.PluginHook
+ name | global_operator_extra_links
+
================+================================================================
+ test-plugin-cli |
<tests.cli.commands.test_plugins_command.AirflowNewLink object>
"""
)
assert stdout == expected_output
diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py
index 98f64e75456..531085ffba5 100644
--- a/tests/plugins/test_plugin.py
+++ b/tests/plugins/test_plugin.py
@@ -21,9 +21,6 @@ from fastapi import FastAPI
from flask import Blueprint
from flask_appbuilder import BaseView as AppBuilderBaseView, expose
-# Importing base classes that we need to derive
-from airflow.hooks.base import BaseHook
-
# This is the class you derive to create a plugin
from airflow.plugins_manager import AirflowPlugin
from airflow.task.priority_strategy import PriorityWeightStrategy
@@ -42,11 +39,6 @@ from tests_common.test_utils.mock_operators import (
)
-# Will show up under airflow.hooks.test_plugin.PluginHook
-class PluginHook(BaseHook):
- pass
-
-
# Will show up under airflow.macros.test_plugin.plugin_macro
def plugin_macro():
pass
@@ -115,7 +107,6 @@ class CustomPriorityWeightStrategy(PriorityWeightStrategy):
# Defining the plugin class
class AirflowTestPlugin(AirflowPlugin):
name = "test_plugin"
- hooks = [PluginHook]
macros = [plugin_macro]
flask_blueprints = [bp]
fastapi_apps = [app_with_metadata]
diff --git a/tests/plugins/test_plugins_manager.py
b/tests/plugins/test_plugins_manager.py
index 158e2040abf..e6777901b80 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -28,7 +28,6 @@ from unittest import mock
import pytest
-from airflow.hooks.base import BaseHook
from airflow.listeners.listener import get_listener_manager
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.module_loading import qualname
@@ -181,29 +180,6 @@ class TestPluginsManager:
assert caplog.record_tuples == []
- def test_should_load_plugins_from_property(self, caplog):
- class AirflowTestPropertyPlugin(AirflowPlugin):
- name = "test_property_plugin"
-
- @property
- def hooks(self):
- class TestPropertyHook(BaseHook):
- pass
-
- return [TestPropertyHook]
-
- with mock_plugin_manager(plugins=[AirflowTestPropertyPlugin()]):
- from airflow import plugins_manager
-
- caplog.set_level(logging.DEBUG, "airflow.plugins_manager")
- plugins_manager.ensure_plugins_loaded()
-
- assert "AirflowTestPropertyPlugin" in str(plugins_manager.plugins)
- assert "TestPropertyHook" in str(plugins_manager.registered_hooks)
-
- assert caplog.records[-1].levelname == "DEBUG"
- assert caplog.records[-1].msg == "Loading %d plugin(s) took %.2f
seconds"
-
def test_loads_filesystem_plugins(self, caplog):
from airflow import plugins_manager
diff --git a/tests_common/test_utils/mock_plugins.py
b/tests_common/test_utils/mock_plugins.py
index 875a9abbd3a..0a5e199fcc0 100644
--- a/tests_common/test_utils/mock_plugins.py
+++ b/tests_common/test_utils/mock_plugins.py
@@ -23,7 +23,6 @@ from airflow import __version__ as airflow_version
PLUGINS_MANAGER_NULLABLE_ATTRIBUTES = [
"plugins",
- "registered_hooks",
"macros_modules",
"admin_views",
"flask_blueprints",
@@ -41,7 +40,6 @@ PLUGINS_MANAGER_NULLABLE_ATTRIBUTES = [
PLUGINS_MANAGER_NULLABLE_ATTRIBUTES_V2_10 = [
"plugins",
- "registered_hooks",
"macros_modules",
"admin_views",
"flask_blueprints",
