This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e39362130b Remove "unknown warning" check in provider verification
(#32444)
e39362130b is described below
commit e39362130b8659942672a728a233887f0b02dc8b
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Jul 8 20:30:40 2023 +0200
Remove "unknown warning" check in provider verification (#32444)
So far we've been checking if there are no "unknown" warnings
generated during provider import check, but this has proven to be
rather useless other than occasionally annoying everone with failing
main when new warnings have been introduced by a dependency.
There was likely not a single time when seeing those new warnings
have been actionable or useful other than adding them to the
exclusion list.
Removing the warning check seems to be reasonable thing to do.
---
scripts/in_container/verify_providers.py | 166 ++-----------------------------
1 file changed, 9 insertions(+), 157 deletions(-)
diff --git a/scripts/in_container/verify_providers.py
b/scripts/in_container/verify_providers.py
index 78cd4c8118..3f547b0087 100755
--- a/scripts/in_container/verify_providers.py
+++ b/scripts/in_container/verify_providers.py
@@ -30,7 +30,6 @@ from enum import Enum
from inspect import isclass
from pathlib import Path
from typing import NamedTuple
-from warnings import WarningMessage
from rich.console import Console
@@ -123,94 +122,6 @@ EXPECTED_SUFFIXES: dict[EntityType, str] = {
}
-# The set of known deprecation messages that we know about.
-# It contains tuples of "message" and the module that generates the warning -
so when the
-# Same warning is generated by different module, it is not treated as "known"
warning.
-KNOWN_DEPRECATED_MESSAGES: set[tuple[str, str]] = {
- ("This version of Apache Beam has not been sufficiently tested on Python",
"apache_beam"),
- ("Using or importing the ABCs from 'collections' instead of from
'collections.abc'", "apache_beam"),
- ("pyarrow.HadoopFileSystem is deprecated as of", "papermill"),
- ("You have an incompatible version of 'pyarrow' installed", "snowflake"),
- ("dns.hash module will be removed in future versions.", "dns"),
- ("PKCS#7 support in pyOpenSSL is deprecated.", "eventlet"),
- ("PKCS#12 support in pyOpenSSL is deprecated.", "eventlet"),
- ("the imp module is deprecated in favour of importlib", "hdfs"),
- ("SelectableGroups dict interface is deprecated.", "kombu"),
- ("The module cloudant is now deprecated.", "cloudant"),
- ("'nteract-scrapbook' package has been renamed to `scrapbook`",
"scrapbook"),
- ("SelectableGroups dict interface is deprecated. Use select.", "markdown"),
- ("'_app_ctx_stack' is deprecated and will be", "flask_sqlalchemy"),
- ("'_request_ctx_stack' is deprecated and will be", "flask_appbuilder"),
- ("'_request_ctx_stack' is deprecated and will be", "flask_jwt_extended"),
- ("'urllib3.contrib.pyopenssl' module is deprecated and will be",
"azure/datalake/store"),
- ("'urllib3.contrib.pyopenssl' module is deprecated and will be",
"botocore"),
- ("'urllib3.contrib.pyopenssl' module is deprecated and will be",
"requests_toolbelt"),
- ("zmq.eventloop.ioloop is deprecated in pyzmq 17.", "jupyter_client"),
- ("Support for grpcio-gcp is deprecated.", "google"),
-}
-
-KNOWN_COMMON_DEPRECATED_MESSAGES: set[str] = {
- "distutils Version classes are deprecated. Use packaging.version instead.",
- "the imp module is deprecated in favour of importlib",
- "Param `schedule_interval` is deprecated and will be removed in a future
release.",
- "'urllib3.contrib.pyopenssl' module is deprecated",
- "Deprecated API features detected! These feature(s) are not compatible
with SQLAlchemy 2.0",
- (
- "Implementing implicit namespace packages (as specified in PEP 420) is
"
- "preferred to `pkg_resources.declare_namespace`"
- ),
- "This module is deprecated. Please use
`airflow.providers.cncf.kubernetes.operators.pod` instead.",
- "urllib3 (1.26.6) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
- "urllib3 (1.26.9) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
- "urllib3 (1.26.12) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
- "Jupyter is migrating its paths to use standard platformdirs given by the
platformdirs library.",
- (
- "Importing ErrorTree directly from the jsonschema package is
deprecated and will become"
- " an ImportError. Import it from jsonschema.exceptions instead."
- ),
-}
-
-# The set of warning messages generated by direct importing of some deprecated
modules. We should only
-# ignore those messages when the warnings are generated directly by importlib
- which means that
-# we imported it directly during module walk by the importlib library
-KNOWN_DEPRECATED_DIRECT_IMPORTS: set[str] = {
- "This module is deprecated. Please use
`kubernetes.client.models.V1Volume`.",
- "This module is deprecated. Please use
`kubernetes.client.models.V1VolumeMount`.",
- "This module is deprecated. Please use
`kubernetes.client.models.V1ResourceRequirements`",
- "This module is deprecated. Please use
`kubernetes.client.models.V1EnvVar`.",
- "numpy.ufunc size changed, may indicate binary incompatibility.",
- "This module is deprecated. Please use
`airflow.providers.amazon.aws.operators.lambda_function`.",
- "S3ToSnowflakeOperator is deprecated.",
- "This module is deprecated. Please use
`airflow.providers.cncf.kubernetes.operators.pod` instead.",
- "This module is deprecated. Please use
`airflow.providers.cncf.kubernetes.triggers.pod` instead.",
- "This module is deprecated. Please use
`airflow.providers.slack.notifications.slack`",
-}
-
-
-def filter_known_warnings(warn: warnings.WarningMessage) -> bool:
- msg_string = str(warn.message).replace("\n", " ")
- for message, origin in KNOWN_DEPRECATED_MESSAGES:
- if message in msg_string and warn.filename.find(f"/{origin}/") != -1:
- return False
- return True
-
-
-def filter_direct_importlib_warning(warn: warnings.WarningMessage) -> bool:
- msg_string = str(warn.message).replace("\n", " ")
- for m in KNOWN_DEPRECATED_DIRECT_IMPORTS:
- if m in msg_string and warn.filename.find("/importlib/") != -1:
- return False
- return True
-
-
-def filter_known_common_deprecated_messages(warn: warnings.WarningMessage) ->
bool:
- msg_string = str(warn.message).replace("\n", " ")
- for m in KNOWN_COMMON_DEPRECATED_MESSAGES:
- if m in msg_string:
- return False
- return True
-
-
def get_all_providers() -> list[str]:
"""Returns all providers for regular packages.
@@ -227,7 +138,7 @@ def import_all_classes(
provider_ids: list[str] | None = None,
print_imports: bool = False,
print_skips: bool = False,
-) -> tuple[list[str], list[WarningMessage], list[str]]:
+) -> tuple[list[str], list[str]]:
"""Imports all classes in providers packages.
This method loads and imports all the classes found in providers, so that
we
@@ -239,8 +150,7 @@ def import_all_classes(
:param provider_ids - provider ids that should be loaded.
:param print_imports - if imported class should also be printed in output
:param print_skips - if skipped classes should also be printed in output
- :return: tuple of list of all imported classes and all warnings and all
classes
- with potential recursion side effects
+ :return: tuple of list of all imported classes and
"""
console.print()
console.print(f"Walking all package with prefixes in
{walkable_paths_and_prefixes}")
@@ -269,7 +179,6 @@ def import_all_classes(
tracebacks.append((package, exception_string))
break
- all_warnings: list[WarningMessage] = []
for path, prefix in walkable_paths_and_prefixes.items():
for modinfo in pkgutil.walk_packages(path=[path], prefix=prefix,
onerror=onerror):
if not any(modinfo.name.startswith(provider_prefix) for
provider_prefix in provider_prefixes):
@@ -282,7 +191,7 @@ def import_all_classes(
printed_packages.add(package_to_print)
console.print(f"Importing package: {package_to_print}")
try:
- with warnings.catch_warnings(record=True) as w:
+ with warnings.catch_warnings(record=True):
warnings.filterwarnings("always",
category=DeprecationWarning)
_module = importlib.import_module(modinfo.name)
for attribute_name in dir(_module):
@@ -295,8 +204,6 @@ def import_all_classes(
or issubclass(attribute, BaseSecretsBackend)
):
classes_with_potential_circular_import.append(class_name)
- if w:
- all_warnings.extend(w)
except AirflowOptionalProviderFeatureException:
# We ignore optional features
...
@@ -324,7 +231,7 @@ def import_all_classes(
console.print("[red]----------------------------------------[/]")
sys.exit(1)
else:
- return imported_classes, all_warnings,
classes_with_potential_circular_import
+ return imported_classes, classes_with_potential_circular_import
def is_imported_from_same_module(the_class: str, imported_name: str) -> bool:
@@ -688,9 +595,8 @@ def
verify_provider_classes_for_single_provider(imported_classes: list[str], pro
return total, bad
-def summarise_total_vs_bad_and_warnings(total: int, bad: int, warns:
list[warnings.WarningMessage]) -> bool:
- """Summarises Bad/Good class names for providers and warnings"""
- raise_error = False
+def summarise_total_vs_bad(total: int, bad: int) -> bool:
+ """Summarises Bad/Good class names for providers"""
if bad == 0:
console.print()
console.print(f"[green]OK: All {total} entities are properly named[/]")
@@ -708,49 +614,6 @@ def summarise_total_vs_bad_and_warnings(total: int, bad:
int, warns: list[warnin
f"[red]ERROR! There are in total: {bad} entities badly named out
of {total} entities[/]"
)
console.print()
- raise_error = True
- if warns:
- if os.environ.get("CI") != "" and bad == 0:
- console.print("::endgroup::")
- console.print()
- console.print("[red]Unknown warnings generated:[/]")
- console.print()
- for w in warns:
- one_line_message = str(w.message).replace("\n", " ")
-
console.print(f"{w.filename}:{w.lineno}:[yellow]{one_line_message}[/]")
- console.print()
- console.print(f"[red]ERROR! There were {len(warns)} warnings generated
during the import[/]")
- console.print()
- console.print("[yellow]Ideally, fix it, so that no warnings are
generated during import.[/]")
- console.print("[yellow]There are three cases that are legitimate
deprecation warnings though:[/]")
- console.print("[yellow] 1) when you deprecate whole module or class
and replace it in provider[/]")
- console.print("[yellow] 2) when 3rd-party module generates Deprecation
and you cannot upgrade it[/]")
- console.print(
- "[yellow] 3) when many 3rd-party module generates same Deprecation
warning that "
- "comes from another common library[/]"
- )
- console.print()
- console.print(
- "[yellow]In case 1), add distinctive subset of the deprecation
message to "
- "the KNOWN_DEPRECATED_DIRECT_IMPORTS in
./scripts/ci/in_container/verify_providers.py[/]"
- )
- console.print(
- "[yellow]In case 2), add distinctive subset of the deprecation
message together with "
- "module it is generates from to "
- "the KNOWN_DEPRECATED_MESSAGES in
./scripts/ci/in_container/verify_providers.py[/]"
- )
- console.print(
- "[yellow]In case 3), add distinctive subset of the deprecation
message to "
- "the KNOWN_COMMON_DEPRECATED_MESSAGES in
./scripts/ci/in_container/verify_providers.py[/]"
- )
- console.print()
- raise_error = True
- else:
- console.print()
- console.print("[green]OK: No warnings generated[/]")
- console.print()
-
- if raise_error:
console.print("[red]Please fix the problems listed above [/]")
return False
return True
@@ -798,7 +661,7 @@ def verify_provider_classes() -> tuple[list[str],
list[str]]:
for provider_path in get_providers_paths():
walkable_paths_and_prefixes[provider_path] = provider_prefix
add_all_namespaced_packages(walkable_paths_and_prefixes,
provider_path, provider_prefix)
- imported_classes, warns, classes_with_potential_circular_import =
import_all_classes(
+ imported_classes, classes_with_potential_circular_import =
import_all_classes(
walkable_paths_and_prefixes=walkable_paths_and_prefixes,
provider_ids=provider_ids,
print_imports=True,
@@ -812,25 +675,14 @@ def verify_provider_classes() -> tuple[list[str],
list[str]]:
)
total += inc_total
bad += inc_bad
- warns = list(filter(filter_known_warnings, warns))
- warns = list(filter(filter_direct_importlib_warning, warns))
- warns = list(filter(filter_known_common_deprecated_messages, warns))
- if not summarise_total_vs_bad_and_warnings(total, bad, warns):
+ if not summarise_total_vs_bad(total, bad):
sys.exit(1)
if len(imported_classes) == 0:
console.print("[red]Something is seriously wrong - no classes
imported[/]")
sys.exit(1)
- if warns:
- console.print("[yellow]There were warnings generated during the
import[/]")
- for w in warns:
- one_line_message = str(w.message).replace("\n", " ")
- print(f"[yellow]{w.filename}:{w.lineno}: {one_line_message}[/]")
-
console.print()
- console.print(
- "[green]SUCCESS: All provider packages are importable and no unknown
warnings generated![/]\n"
- )
+ console.print("[green]SUCCESS: All provider packages are importable![/]\n")
console.print(f"Imported {len(imported_classes)} classes.")
console.print()
return imported_classes, classes_with_potential_circular_import
