This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c501d3a1f6e3743566a34c20a4e6514710e48df6 Author: Andrey Anshin <[email protected]> AuthorDate: Wed Apr 3 12:42:16 2024 +0400 Use `methodtools.lru_cache` instead of `functools.lru_cache` in class methods (#37757) * Use `methodtools.lru_cache` instead of `functools.lru_cache` in class methods * Rename pre-commit script (cherry picked from commit c65b08399d11410795b470640eb4fe6f42748c36) --- .pre-commit-config.yaml | 7 +++ airflow/compat/functools.pyi | 27 ---------- airflow/models/abstractoperator.py | 4 +- airflow/models/mappedoperator.py | 8 +-- airflow/utils/task_group.py | 4 +- airflow/utils/weight_rule.py | 4 +- contributing-docs/08_static_code_checks.rst | 2 + dev/breeze/doc/images/output_static-checks.svg | 26 ++++----- dev/breeze/doc/images/output_static-checks.txt | 2 +- dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 + hatch_build.py | 1 + pyproject.toml | 3 +- scripts/ci/pre_commit/compat_cache_on_methods.py | 69 ++++++++++++++++++++++++ 13 files changed, 106 insertions(+), 52 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db7325ef13..b2f998afa9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -980,6 +980,13 @@ repos: files: \.py$ exclude: ^.*/.*_vendor/ additional_dependencies: ['rich>=12.4.4'] + - id: check-compat-cache-on-methods + name: Check that compat cache do not use on class methods + entry: ./scripts/ci/pre_commit/compat_cache_on_methods.py + language: python + pass_filenames: true + files: ^airflow/.*\.py$ + exclude: ^.*/.*_vendor/ - id: lint-chart-schema name: Lint chart/values.schema.json file entry: ./scripts/ci/pre_commit/chart_schema.py diff --git a/airflow/compat/functools.pyi b/airflow/compat/functools.pyi deleted file mode 100644 index 7d1bef5939..0000000000 --- a/airflow/compat/functools.pyi +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This stub exists to work around false linter errors due to python/mypy#10408. -# TODO: Remove this file after the upstream fix is available in our toolchain. -from __future__ import annotations - -from typing import TypeVar - -T = TypeVar("T") - -def cache(f: T) -> T: ... diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 380c6fcf00..5d9469749c 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -23,9 +23,9 @@ from abc import abstractproperty from functools import cached_property from typing import TYPE_CHECKING, Any, Callable, ClassVar, Collection, Iterable, Iterator, Sequence +import methodtools from sqlalchemy import select -from airflow.compat.functools import cache from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models.expandinput import NotFullyPopulated @@ -492,7 +492,7 @@ class AbstractOperator(Templater, DAGNode): return link.get_link(self.unmap(None), ti.dag_run.logical_date) # type: ignore[misc] return link.get_link(self.unmap(None), ti_key=ti.key) - @cache + @methodtools.lru_cache(maxsize=None) def get_parse_time_mapped_ti_count(self) -> int: """ Return the number of mapped task instances that can be created on DAG run creation. diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py index 994e041d9f..ba5e8c174d 100644 --- a/airflow/models/mappedoperator.py +++ b/airflow/models/mappedoperator.py @@ -24,8 +24,8 @@ import warnings from typing import TYPE_CHECKING, Any, ClassVar, Collection, Iterable, Iterator, Mapping, Sequence, Union import attr +import methodtools -from airflow.compat.functools import cache from airflow.exceptions import AirflowException, UnmappableOperator from airflow.models.abstractoperator import ( DEFAULT_IGNORE_FIRST_DEPENDS_ON_PAST, @@ -334,8 +334,8 @@ class MappedOperator(AbstractOperator): f"{self.task_id!r}." ) + @methodtools.lru_cache(maxsize=None) @classmethod - @cache def get_serialized_fields(cls): # Not using 'cls' here since we only want to serialize base fields. return frozenset(attr.fields_dict(MappedOperator)) - { @@ -351,8 +351,8 @@ class MappedOperator(AbstractOperator): "_on_failure_fail_dagrun", } + @methodtools.lru_cache(maxsize=None) @staticmethod - @cache def deps_for(operator_class: type[BaseOperator]) -> frozenset[BaseTIDep]: operator_deps = operator_class.deps if not isinstance(operator_deps, collections.abc.Set): @@ -784,7 +784,7 @@ class MappedOperator(AbstractOperator): for operator, _ in XComArg.iter_xcom_references(self._get_specified_expand_input()): yield operator - @cache + @methodtools.lru_cache(maxsize=None) def get_parse_time_mapped_ti_count(self) -> int: current_count = self._get_specified_expand_input().get_parse_time_mapped_ti_count() try: diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index 5f79600c47..50ce7b4089 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -25,9 +25,9 @@ import operator import weakref from typing import TYPE_CHECKING, Any, Generator, Iterator, Sequence +import methodtools import re2 -from airflow.compat.functools import cache from airflow.exceptions import ( AirflowDagCycleException, AirflowException, @@ -586,7 +586,7 @@ class MappedTaskGroup(TaskGroup): for op, _ in XComArg.iter_xcom_references(self._expand_input): yield op - @cache + @methodtools.lru_cache(maxsize=None) def get_parse_time_mapped_ti_count(self) -> int: """ Return the Number of instances a task in this group should be mapped to, when a DAG run is created. diff --git a/airflow/utils/weight_rule.py b/airflow/utils/weight_rule.py index f65f2fa77e..a63358b032 100644 --- a/airflow/utils/weight_rule.py +++ b/airflow/utils/weight_rule.py @@ -19,7 +19,7 @@ from __future__ import annotations from enum import Enum -from airflow.compat.functools import cache +import methodtools class WeightRule(str, Enum): @@ -34,8 +34,8 @@ class WeightRule(str, Enum): """Check if weight rule is valid.""" return weight_rule in cls.all_weight_rules() + @methodtools.lru_cache(maxsize=None) @classmethod - @cache def all_weight_rules(cls) -> set[str]: """Return all weight rules.""" return set(cls.__members__.values()) diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 0b331bf3e9..b02c8749a7 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -146,6 +146,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-cncf-k8s-only-for-executors | Check cncf.kubernetes imports used for executors only | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-compat-cache-on-methods | Check that compat cache do not use on class methods | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-core-deprecation-classes | Verify usage of Airflow deprecation classes in core | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-daysago-import-from-utils | Make sure days_ago is imported from airflow.utils.dates | | diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index 679db3dfee..a4c1734c30 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -319,19 +319,19 @@ </text><text class="breeze-static-checks-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-static-checks-line-10)">│</text><text class="breeze-static-checks-r7" x="451.4" y="264" textLength="988.2" clip-path="url(#breeze-static-checks-line-10)">check-base-operator-partial-arguments | check-base-operator-usage |              </text><text class="breeze-static-checks-r5" x="1451.8" y="264" textLeng [...] </text><text class="breeze-static-checks-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-11)">│</text><text class="breeze-static-checks-r7" x="451.4" y="288.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-11)">check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |      </text><text class="breeze-static-checks-r5" x="1451.8" y="288.4" textLength="12.2" clip-path="url(#breeze-s [...] </text><text class="breeze-static-checks-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-12)">│</text><text class="breeze-static-checks-r7" x="451.4" y="312.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-12)">check-builtin-literals | check-changelog-has-no-duplicates |                     </text><text class="breeze-static [...] -</text><text class="breeze-static-checks-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-13)">│</text><text class="breeze-static-checks-r7" x="451.4" y="337.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-13)">check-cncf-k8s-only-for-executors | check-core-deprecation-classes |             </text><text class="breeze-static-checks-r5" x="1451.8" y="337.2" textLen [...] -</text><text class="breeze-static-checks-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-14)">│</text><text class="breeze-static-checks-r7" x="451.4" y="361.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-14)">check-daysago-import-from-utils | check-decorated-operator-implements-custom-name</text><text class="breeze-static-checks-r5" x="1451.8" y="361.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-14)">│</text><tex [...] -</text><text class="breeze-static-checks-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-static-checks-line-15)">│</text><text class="breeze-static-checks-r7" x="451.4" y="386" textLength="988.2" clip-path="url(#breeze-static-checks-line-15)">| check-deferrable-default-value | check-docstring-param-types |                 </text><text class="breeze-static-checks-r5" x="1451 [...] -</text><text class="breeze-static-checks-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-16)">│</text><text class="breeze-static-checks-r7" x="451.4" y="410.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-16)">check-example-dags-urls | check-executables-have-shebangs |                      </text><text class="breeze-s [...] -</text><text class="breeze-static-checks-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-17)">│</text><text class="breeze-static-checks-r7" x="451.4" y="434.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-17)">check-extra-packages-references | check-extras-order | check-fab-migrations |    </text><text class="breeze-static-checks-r5" x="1451.8" y="434.8" textLength="12.2" clip-path="url(#breeze-s [...] -</text><text class="breeze-static-checks-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-18)">│</text><text class="breeze-static-checks-r7" x="451.4" y="459.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-18)">check-for-inclusive-language | check-google-re2-as-dependency |                  </text><text class="breeze-static-checks-r5" x=" [...] -</text><text class="breeze-static-checks-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-19)">│</text><text class="breeze-static-checks-r7" x="451.4" y="483.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-19)">check-hatch-build-order | check-hooks-apply | check-incorrect-use-of-LoggingMixin</text><text class="breeze-static-checks-r5" x="1451.8" y="483.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-19)">│< [...] -</text><text class="breeze-static-checks-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-static-checks-line-20)">│</text><text class="breeze-static-checks-r7" x="451.4" y="508" textLength="988.2" clip-path="url(#breeze-static-checks-line-20)">| check-init-decorator-arguments | check-integrations-list-consistent |          </text><text class="breeze-static-checks-r5" x="1451.8" y="508" textLength="12.2" clip- [...] -</text><text class="breeze-static-checks-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-21)">│</text><text class="breeze-static-checks-r7" x="451.4" y="532.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-21)">check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | </text><text class="breeze-static-checks-r5" x="1451.8" y="532.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-21)">│< [...] -</text><text class="breeze-static-checks-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-22)">│</text><text class="breeze-static-checks-r7" x="451.4" y="556.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-22)">check-merge-conflict | check-newsfragments-are-valid |                           </t [...] -</text><text class="breeze-static-checks-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-23)">│</text><text class="breeze-static-checks-r7" x="451.4" y="581.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-23)">check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |</text><text class="breeze-static-checks-r5" x="1451.8" y="581.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-23)">│</text [...] -</text><text class="breeze-static-checks-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-24)">│</text><text class="breeze-static-checks-r7" x="451.4" y="605.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-24)">check-only-new-session-with-provide-session |                               [...] -</text><text class="breeze-static-checks-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-static-checks-line-25)">│</text><text class="breeze-static-checks-r7" x="451.4" y="630" textLength="988.2" clip-path="url(#breeze-static-checks-line-25)">check-persist-credentials-disabled-in-github-workflows |                         </text><text class="breeze- [...] +</text><text class="breeze-static-checks-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-13)">│</text><text class="breeze-static-checks-r7" x="451.4" y="337.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-13)">check-cncf-k8s-only-for-executors | check-compat-cache-on-methods |              </text><text class="breeze-static-checks-r5" x="1451.8" y="337.2" te [...] +</text><text class="breeze-static-checks-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-14)">│</text><text class="breeze-static-checks-r7" x="451.4" y="361.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-14)">check-core-deprecation-classes | check-daysago-import-from-utils |               </text><text class="breeze-static-checks-r5" x="1451.8" y="361. [...] +</text><text class="breeze-static-checks-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-static-checks-line-15)">│</text><text class="breeze-static-checks-r7" x="451.4" y="386" textLength="988.2" clip-path="url(#breeze-static-checks-line-15)">check-decorated-operator-implements-custom-name | check-deferrable-default-value </text><text class="breeze-static-checks-r5" x="1451.8" y="386" textLength="12.2" clip-path="url(#breeze-static-checks-line-15)">│</text><text [...] +</text><text class="breeze-static-checks-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-16)">│</text><text class="breeze-static-checks-r7" x="451.4" y="410.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-16)">| check-docstring-param-types | check-example-dags-urls |                        </text><text [...] +</text><text class="breeze-static-checks-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-17)">│</text><text class="breeze-static-checks-r7" x="451.4" y="434.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-17)">check-executables-have-shebangs | check-extra-packages-references |              </text><text class="breeze-static-checks-r5" x="1451.8" y="434.8" te [...] +</text><text class="breeze-static-checks-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-18)">│</text><text class="breeze-static-checks-r7" x="451.4" y="459.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-18)">check-extras-order | check-fab-migrations | check-for-inclusive-language |       </text><text class="breeze-static-checks-r5" x="1451.8" y="459.2" textLength="12.2" clip-path [...] +</text><text class="breeze-static-checks-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-19)">│</text><text class="breeze-static-checks-r7" x="451.4" y="483.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-19)">check-google-re2-as-dependency | check-hatch-build-order | check-hooks-apply |   </text><text class="breeze-static-checks-r5" x="1451.8" y="483.6" textLength="12.2" clip-path="url(#breeze-static [...] +</text><text class="breeze-static-checks-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-static-checks-line-20)">│</text><text class="breeze-static-checks-r7" x="451.4" y="508" textLength="988.2" clip-path="url(#breeze-static-checks-line-20)">check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           </text><text class="breeze-static-checks-r5" x="1451.8" y="508" textLength="12.2" clip- [...] +</text><text class="breeze-static-checks-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-21)">│</text><text class="breeze-static-checks-r7" x="451.4" y="532.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-21)">check-integrations-list-consistent | check-lazy-logging |                        </text><text class [...] +</text><text class="breeze-static-checks-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-22)">│</text><text class="breeze-static-checks-r7" x="451.4" y="556.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-22)">check-links-to-example-dags-do-not-use-hardcoded-versions | check-merge-conflict </text><text class="breeze-static-checks-r5" x="1451.8" y="556.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-22)">│</text [...] +</text><text class="breeze-static-checks-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-23)">│</text><text class="breeze-static-checks-r7" x="451.4" y="581.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-23)">| check-newsfragments-are-valid | check-no-airflow-deprecation-in-providers |    </text><text class="breeze-static-checks-r5" x="1451.8" y="581.2" textLength="12.2" clip-path="url(#breeze-static [...] +</text><text class="breeze-static-checks-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-24)">│</text><text class="breeze-static-checks-r7" x="451.4" y="605.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-24)">check-no-providers-in-core-examples | check-only-new-session-with-provide-session</text><text class="breeze-static-checks-r5" x="1451.8" y="605.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-24)">│</text><tex [...] +</text><text class="breeze-static-checks-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-static-checks-line-25)">│</text><text class="breeze-static-checks-r7" x="451.4" y="630" textLength="988.2" clip-path="url(#breeze-static-checks-line-25)">| check-persist-credentials-disabled-in-github-workflows |                       </text><text class="breeze-stati [...] </text><text class="breeze-static-checks-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-26)">│</text><text class="breeze-static-checks-r7" x="451.4" y="654.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-26)">check-pre-commit-information-consistent | check-provide-create-sessions-imports |</text><text class="breeze-static-checks-r5" x="1451.8" y="654.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-26)">│</text [...] </text><text class="breeze-static-checks-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-27)">│</text><text class="breeze-static-checks-r7" x="451.4" y="678.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-27)">check-provider-docs-valid | check-provider-yaml-valid |                          </text>< [...] </text><text class="breeze-static-checks-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-28)">│</text><text class="breeze-static-checks-r7" x="451.4" y="703.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-28)">check-providers-init-file-missing | check-providers-subpackages-init-file-exist |</text><text class="breeze-static-checks-r5" x="1451.8" y="703.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-28)">│</text [...] diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index 66ca569ffb..ebec23568e 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -e8fa3d7a6215d2565dc536cbc50e0465 +e83bbc2b97062b42c940992d0c2d80f6 diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 516fc85c51..eb5c46e1ab 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -38,6 +38,7 @@ PRE_COMMIT_LIST = [ "check-builtin-literals", "check-changelog-has-no-duplicates", "check-cncf-k8s-only-for-executors", + "check-compat-cache-on-methods", "check-core-deprecation-classes", "check-daysago-import-from-utils", "check-decorated-operator-implements-custom-name", diff --git a/hatch_build.py b/hatch_build.py index ca85510780..439e486ec5 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -463,6 +463,7 @@ DEPENDENCIES = [ "markupsafe>=1.1.1", "marshmallow-oneofschema>=2.0.1", "mdit-py-plugins>=0.3.0", + "methodtools>=0.4.7", "opentelemetry-api>=1.15.0", "opentelemetry-exporter-otlp", "packaging>=14.0", diff --git a/pyproject.toml b/pyproject.toml index 2fc9981328..7ab695097c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -284,9 +284,10 @@ extend-select = [ "D419", "PGH004", # Use specific rule codes when using noqa "PGH005", # Invalid unittest.mock.Mock methods/attributes/properties - "B006", # Checks for uses of mutable objects as function argument defaults. "S101", # Checks use `assert` outside the test cases, test cases should be added into the exclusions "B004", # Checks for use of hasattr(x, "__call__") and replaces it with callable(x) + "B006", # Checks for uses of mutable objects as function argument defaults. + "B019", # Use of functools.lru_cache or functools.cache on methods can lead to memory leaks ] ignore = [ "D203", diff --git a/scripts/ci/pre_commit/compat_cache_on_methods.py b/scripts/ci/pre_commit/compat_cache_on_methods.py new file mode 100755 index 0000000000..5fee74ff2a --- /dev/null +++ b/scripts/ci/pre_commit/compat_cache_on_methods.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import ast +import pathlib +import sys + +COMPAT_MODULE = "airflow.compat.functools" + + +def check_test_file(file: str) -> int: + node = ast.parse(pathlib.Path(file).read_text("utf-8"), file) + if not (classes := [c for c in node.body if isinstance(c, ast.ClassDef)]): + # Exit early if module doesn't contain any classes + return 0 + + compat_cache_aliases = [] + for stmt in node.body: + if not isinstance(stmt, ast.ImportFrom) or stmt.module != COMPAT_MODULE: + continue + for alias in stmt.names: + if "cache" in alias.name: + compat_cache_aliases.append(alias.asname or alias.name) + if not compat_cache_aliases: + # Exit early in case if there are no imports from `airflow.compat.functools.cache` + return 0 + + found = 0 + for klass in classes: + for cls_stmt in klass.body: + if not isinstance(cls_stmt, ast.FunctionDef) or not cls_stmt.decorator_list: + continue + for decorator in cls_stmt.decorator_list: + if (isinstance(decorator, ast.Name) and decorator.id in compat_cache_aliases) or ( + isinstance(decorator, ast.Attribute) and decorator.attr in compat_cache_aliases + ): + found += 1 + prefix = f"{file}:{decorator.lineno}:" + print(f"{prefix} Use of `{COMPAT_MODULE}.cache` on methods can lead to memory leaks") + + return found + + +def main(*args: str) -> int: + errors = sum(check_test_file(file) for file in args[1:]) + if not errors: + return 0 + print(f"Found {errors} error{'s' if errors > 1 else ''}.") + return 1 + + +if __name__ == "__main__": + sys.exit(main(*sys.argv))
