This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b341b5927e docs: Add doc page with providers deprecations (#37075)
b341b5927e is described below
commit b341b5927e6a7271a85558d3da766b07165ee22a
Author: Kacper Muda <[email protected]>
AuthorDate: Wed Feb 14 15:19:54 2024 +0100
docs: Add doc page with providers deprecations (#37075)
---
airflow/providers/amazon/aws/hooks/base_aws.py | 4 +-
.../providers/amazon/aws/hooks/redshift_cluster.py | 2 +-
docs/README.rst | 13 +++
.../core-extensions/deprecations.rst | 31 ++++++
docs/exts/docs_build/errors.py | 5 +-
docs/exts/docs_build/spelling_checks.py | 13 ++-
docs/exts/operators_and_hooks_ref.py | 115 +++++++++++++++++++++
docs/exts/templates/deprecations.rst.jinja2 | 37 +++++++
8 files changed, 212 insertions(+), 8 deletions(-)
diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py
b/airflow/providers/amazon/aws/hooks/base_aws.py
index 5f1e075cc3..ff2f33dbe8 100644
--- a/airflow/providers/amazon/aws/hooks/base_aws.py
+++ b/airflow/providers/amazon/aws/hooks/base_aws.py
@@ -1022,7 +1022,7 @@ except ImportError:
@deprecated(
reason=(
- "airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory "
+ "`airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory` "
"has been deprecated and will be removed in future"
),
category=AirflowProviderDeprecationWarning,
@@ -1116,7 +1116,7 @@ class BaseAsyncSessionFactory(BaseSessionFactory):
@deprecated(
reason=(
- "airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook "
+ "`airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook` "
"has been deprecated and will be removed in future"
),
category=AirflowProviderDeprecationWarning,
diff --git a/airflow/providers/amazon/aws/hooks/redshift_cluster.py
b/airflow/providers/amazon/aws/hooks/redshift_cluster.py
index 286299661c..6168278a59 100644
--- a/airflow/providers/amazon/aws/hooks/redshift_cluster.py
+++ b/airflow/providers/amazon/aws/hooks/redshift_cluster.py
@@ -197,7 +197,7 @@ class RedshiftHook(AwsBaseHook):
@deprecated(
reason=(
- "airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook "
+ "`airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook` "
"has been deprecated and will be removed in future"
),
category=AirflowProviderDeprecationWarning,
diff --git a/docs/README.rst b/docs/README.rst
index d5cddec486..9cb436c0f2 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -258,6 +258,19 @@ warning to report the wrong line in the file for your
missing white space. If yo
and the line number it reports is wrong, this is a known error. You can find
the missing blank space by searching for the syntax you used to make your
list, code block, or other whitespace-sensitive markup element.
+
+spelling error with class or method name
+****************************************
+When a spelling error occurs that has a class/function/method name as
incorrectly spelled,
+instead of whitelisting it in docs/spelling_wordlist.txt you should make sure
that
+this name is quoted with backticks "`" - this should exclude it from
spellchecking process.
+
+In this example error, You should change the line with error so that the whole
path is inside backticks "`".
+.. code-block:: text
+
+ Incorrect Spelling: 'BaseAsyncSessionFactory'
+ Line with Error: '
airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory has been
deprecated'
+
Support
=======
diff --git a/docs/apache-airflow-providers/core-extensions/deprecations.rst
b/docs/apache-airflow-providers/core-extensions/deprecations.rst
new file mode 100644
index 0000000000..177ea77ab9
--- /dev/null
+++ b/docs/apache-airflow-providers/core-extensions/deprecations.rst
@@ -0,0 +1,31 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Deprecations
+==========================================
+
+This is a summary of deprecated objects in latest versions of Apache Airflow
Providers Packages.
+
+.. note::
+ This is an experimental page that may not contain all deprecated entities.
+ At the moment we only show deprecations for classes, functions, methods,
properties etc.
+ Support for an argument deprecation or an argument value deprecation will
be added in the future.
+
+.. airflow-deprecations::
+ :tags: None
+ :header-separator: "
diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py
index 0abfc8fe57..79a403924b 100644
--- a/docs/exts/docs_build/errors.py
+++ b/docs/exts/docs_build/errors.py
@@ -77,8 +77,9 @@ def display_errors_summary(build_errors: dict[str,
list[DocBuildError]]) -> None
console.print(
f"File path: {os.path.relpath(error.file_path,
start=DOCS_DIR)} ({error.line_no})"
)
- console.print()
- console.print(prepare_code_snippet(error.file_path,
error.line_no))
+ if os.path.isfile(error.file_path):
+ console.print()
+ console.print(prepare_code_snippet(error.file_path,
error.line_no))
elif error.file_path:
console.print(f"File path: {error.file_path}")
console.print()
diff --git a/docs/exts/docs_build/spelling_checks.py
b/docs/exts/docs_build/spelling_checks.py
index 2e3f7d409f..a1d56b7746 100644
--- a/docs/exts/docs_build/spelling_checks.py
+++ b/docs/exts/docs_build/spelling_checks.py
@@ -154,7 +154,9 @@ def display_spelling_error_summary(spelling_errors:
dict[str, list[SpellingError
console.print("=" * 100)
console.print()
- msg = """
+ msg = """[green]
+If there are spelling errors related to class or function name, make sure
+those names are quoted with backticks '`' - this should exclude it from
spellcheck process.
If there are spelling errors in the summary above, and the spelling is
correct, add the spelling to docs/spelling_wordlist.txt or use the
spelling directive.
@@ -177,11 +179,16 @@ def _display_error(error: SpellingError):
if error.file_path:
console.print(f"File path: {os.path.relpath(error.file_path,
start=DOCS_DIR)}")
if error.spelling:
- console.print(f"Incorrect Spelling: '{error.spelling}'")
+ console.print(f"[red]Incorrect Spelling: '{error.spelling}'")
if error.suggestion:
console.print(f"Suggested Spelling: '{error.suggestion}'")
if error.context_line:
console.print(f"Line with Error: '{error.context_line}'")
- if error.file_path and not error.file_path.endswith("<unknown>") and
error.line_no:
+ if (
+ error.file_path
+ and not error.file_path.endswith("<unknown>")
+ and error.line_no
+ and os.path.isfile(error.file_path)
+ ):
console.print(f"Line Number: {error.line_no}")
console.print(prepare_code_snippet(error.file_path, error.line_no))
diff --git a/docs/exts/operators_and_hooks_ref.py
b/docs/exts/operators_and_hooks_ref.py
index 272c8224a1..413cda59f4 100644
--- a/docs/exts/operators_and_hooks_ref.py
+++ b/docs/exts/operators_and_hooks_ref.py
@@ -221,6 +221,111 @@ def _render_deferrable_operator_content(*,
header_separator: str):
)
+def _get_decorator_details(decorator):
+ def get_full_name(node):
+ if isinstance(node, ast.Attribute):
+ return f"{get_full_name(node.value)}.{node.attr}"
+ elif isinstance(node, ast.Name):
+ return node.id
+ else:
+ return ast.dump(node)
+
+ def eval_node(node):
+ try:
+ return ast.literal_eval(node)
+ except ValueError:
+ return ast.dump(node)
+
+ if isinstance(decorator, ast.Call):
+ name = get_full_name(decorator.func)
+ args = [eval_node(arg) for arg in decorator.args]
+ kwargs = {kw.arg: eval_node(kw.value) for kw in decorator.keywords if
kw.arg != "category"}
+ return name, args, kwargs
+ elif isinstance(decorator, ast.Name):
+ return decorator.id, [], {}
+ elif isinstance(decorator, ast.Attribute):
+ return decorator.attr, [], {}
+ else:
+ return decorator, [], {}
+
+
+def _iter_module_for_deprecations(ast_node, file_path, class_name=None) ->
list[dict[str, Any]]:
+ deprecations = []
+ decorators_of_deprecation = {"deprecated"}
+
+ def analyze_decorators(node, _file_path, object_type, _class_name=None):
+ for decorator in node.decorator_list:
+ if str(_class_name).startswith("_") or
str(node.name).startswith("_"):
+ continue
+ decorator_name, decorator_args, decorator_kwargs =
_get_decorator_details(decorator)
+
+ instructions = decorator_kwargs.get("reason", "No instructions
were provided.")
+ if len(decorator_args) == 1 and isinstance(decorator_args[0], str)
and not instructions:
+ instructions = decorator_args[0]
+
+ if decorator_name in (
+ "staticmethod",
+ "classmethod",
+ "property",
+ "abstractmethod",
+ "cached_property",
+ ):
+ object_type = decorator_name
+
+ if decorator_name in decorators_of_deprecation:
+ object_name = f"{_class_name}.{node.name}" if _class_name else
node.name
+ object_path = os.path.join(_file_path,
object_name).replace("/", ".").lstrip(".")
+ deprecations.append(
+ {
+ "object_path": object_path,
+ "object_name": object_name,
+ "object_type": object_type,
+ "instructions": instructions,
+ }
+ )
+
+ for child in ast.iter_child_nodes(ast_node):
+ if isinstance(child, ast.ClassDef):
+ analyze_decorators(child, file_path, object_type="class")
+ deprecations.extend(_iter_module_for_deprecations(child,
file_path, class_name=child.name))
+ elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ analyze_decorators(
+ child, file_path, _class_name=class_name, object_type="method"
if class_name else "function"
+ )
+ else:
+ deprecations.extend(_iter_module_for_deprecations(child,
file_path, class_name=class_name))
+
+ return deprecations
+
+
+def _render_deprecations_content(*, header_separator: str):
+ providers = []
+ for provider_yaml_path in get_provider_yaml_paths():
+ provider_parent_path = Path(provider_yaml_path).parent
+ provider_info: dict[str, Any] = {"name": "", "deprecations": []}
+ for root, _, file_names in os.walk(provider_parent_path):
+ file_names = [f for f in file_names if f.endswith(".py") and f !=
"__init__.py"]
+ for file_name in file_names:
+ file_path = f"{os.path.relpath(root)}/{file_name}"
+ with open(file_path) as file:
+ ast_obj = ast.parse(file.read())
+
provider_info["deprecations"].extend(_iter_module_for_deprecations(ast_obj,
file_path[:-3]))
+
+ if provider_info["deprecations"]:
+ provider_info["deprecations"] = sorted(
+ provider_info["deprecations"], key=lambda p: p["object_path"]
+ )
+ provider_yaml_content =
yaml.safe_load(Path(provider_yaml_path).read_text())
+ provider_info["name"] = provider_yaml_content["package-name"]
+ providers.append(provider_info)
+
+ return _render_template(
+ "deprecations.rst.jinja2",
+ providers=sorted(providers, key=lambda p: p["name"]),
+ header_separator=header_separator,
+ )
+
+
class BaseJinjaReferenceDirective(Directive):
"""The base directive for OperatorsHooksReferenceDirective and
TransfersReferenceDirective"""
@@ -399,6 +504,15 @@ class
DeferrableOperatorDirective(BaseJinjaReferenceDirective):
)
+class DeprecationsDirective(BaseJinjaReferenceDirective):
+ """Generate list of deprecated entities"""
+
+ def render_content(self, *, tags: set[str] | None, header_separator: str =
DEFAULT_HEADER_SEPARATOR):
+ return _render_deprecations_content(
+ header_separator=header_separator,
+ )
+
+
def setup(app):
"""Setup plugin"""
app.add_directive("operators-hooks-ref", OperatorsHooksReferenceDirective)
@@ -412,6 +526,7 @@ def setup(app):
app.add_directive("airflow-notifications", NotificationsDirective)
app.add_directive("airflow-executors", ExecutorsDirective)
app.add_directive("airflow-deferrable-operators",
DeferrableOperatorDirective)
+ app.add_directive("airflow-deprecations", DeprecationsDirective)
return {"parallel_read_safe": True, "parallel_write_safe": True}
diff --git a/docs/exts/templates/deprecations.rst.jinja2
b/docs/exts/templates/deprecations.rst.jinja2
new file mode 100644
index 0000000000..f134a58ce9
--- /dev/null
+++ b/docs/exts/templates/deprecations.rst.jinja2
@@ -0,0 +1,37 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+#}
+{%- for provider in providers %}
+{{ provider["name"] }}
+{{ header_separator * (provider['name']|length) }}
+
+{% for deprecation in provider['deprecations'] %}
+
+{% set prefix = ':attr:' %}
+{% if deprecation["object_type"] == 'class' %}
+ {% set prefix = ':class:' %}
+{% elif deprecation["object_type"] == 'function' %}
+ {% set prefix = ':func:' %}
+{% elif 'method' in deprecation["object_type"] %}
+ {% set prefix = ':meth:' %}
+{% endif %}
+
+- {{ prefix }}`{{ deprecation["object_name"] }} <{{ deprecation["object_path"]
}}>` - {{ deprecation["instructions"] }}
+{% endfor %}
+
+{% endfor %}
