Lee-W commented on code in PR #43383:
URL: https://github.com/apache/airflow/pull/43383#discussion_r1942149782
##########
airflow/utils/decorators.py:
##########
@@ -18,54 +18,56 @@
from __future__ import annotations
import sys
-from collections import deque
from typing import Callable, TypeVar
+import libcst as cst
+
T = TypeVar("T", bound=Callable)
+class _TaskDecoratorRemover(cst.CSTTransformer):
+ def __init__(self, task_decorator_name: str) -> None:
+ self.decorators_to_remove: set[str] = {
+ "setup",
+ "teardown",
+ "task.skip_if",
+ "task.run_if",
+ task_decorator_name.strip("@"),
+ }
+
+ def _is_task_decorator(self, decorator_node: cst.Decorator) -> bool:
+ if isinstance(decorator_node.decorator, cst.Name):
Review Comment:
```suggestion
decorator_expr = decorator_node.decorator
if isinstance(decorator_expr, cst.Name):
```
##########
providers/standard/tests/provider_tests/standard/utils/test_python_virtualenv.py:
##########
@@ -191,26 +192,29 @@ def
test_should_create_virtualenv_with_extra_packages_uv(self, mock_execute_in_s
["uv", "pip", "install", "--python", "/VENV/bin/python",
"apache-beam[gcp]"]
)
- def test_remove_task_decorator(self):
- py_source = '@task.virtualenv(serializer="dill")\ndef f():\nimport
funcsigs'
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "def f():\nimport funcsigs"
-
- def test_remove_decorator_no_parens(self):
- py_source = "@task.virtualenv\ndef f():\nimport funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "def f():\nimport funcsigs"
-
- def test_remove_decorator_including_comment(self):
- py_source = "@task.virtualenv\ndef f():\n# @task.virtualenv\nimport
funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "def f():\n# @task.virtualenv\nimport funcsigs"
-
- def test_remove_decorator_nested(self):
- py_source = "@foo\[email protected]\n@bar\ndef f():\nimport funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "@foo\n@bar\ndef f():\nimport funcsigs"
+ @pytest.mark.parametrize(
+ "decorators, expected_decorators",
+ [
+ (["@task.virtualenv"], []),
+ (["@task.virtualenv()"], []),
+ (['@task.virtualenv(serializer="dill")'], []),
+ (["@foo", "@task.virtualenv", "@bar"], ["@foo", "@bar"]),
+ (["@foo", "@task.virtualenv()", "@bar"], ["@foo", "@bar"]),
+ ],
+ ids=["without_parens", "parens", "with_args", "nested_without_parens",
"nested_with_parens"],
+ )
+ def test_remove_task_decorator(self, decorators: list[str],
expected_decorators: list[str]):
+ decorator = "\n".join(decorators)
Review Comment:
```suggestion
concated_decorators = "\n".join(decorators)
```
This might make the variable name more accurate. was a bit confused when I
first saw it. but nice refacoring!
##########
providers/standard/tests/provider_tests/standard/utils/test_python_virtualenv.py:
##########
@@ -191,26 +192,29 @@ def
test_should_create_virtualenv_with_extra_packages_uv(self, mock_execute_in_s
["uv", "pip", "install", "--python", "/VENV/bin/python",
"apache-beam[gcp]"]
)
- def test_remove_task_decorator(self):
- py_source = '@task.virtualenv(serializer="dill")\ndef f():\nimport
funcsigs'
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "def f():\nimport funcsigs"
-
- def test_remove_decorator_no_parens(self):
- py_source = "@task.virtualenv\ndef f():\nimport funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "def f():\nimport funcsigs"
-
- def test_remove_decorator_including_comment(self):
- py_source = "@task.virtualenv\ndef f():\n# @task.virtualenv\nimport
funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "def f():\n# @task.virtualenv\nimport funcsigs"
-
- def test_remove_decorator_nested(self):
- py_source = "@foo\[email protected]\n@bar\ndef f():\nimport funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.virtualenv")
- assert res == "@foo\n@bar\ndef f():\nimport funcsigs"
+ @pytest.mark.parametrize(
+ "decorators, expected_decorators",
+ [
+ (["@task.virtualenv"], []),
+ (["@task.virtualenv()"], []),
+ (['@task.virtualenv(serializer="dill")'], []),
+ (["@foo", "@task.virtualenv", "@bar"], ["@foo", "@bar"]),
+ (["@foo", "@task.virtualenv()", "@bar"], ["@foo", "@bar"]),
+ ],
+ ids=["without_parens", "parens", "with_args", "nested_without_parens",
"nested_with_parens"],
+ )
+ def test_remove_task_decorator(self, decorators: list[str],
expected_decorators: list[str]):
+ decorator = "\n".join(decorators)
+ expected_decorator = "\n".join(expected_decorators)
+ SCRIPT = dedent(
+ """
+ def f():
+ # @task.virtualenv
+ import funcsigs
+ """
+ )
+ py_source = decorator + SCRIPT
+ expected_source = expected_decorator + SCRIPT if expected_decorator
else SCRIPT.lstrip()
Review Comment:
```suggestion
py_source = f"{decorator}{SCRIPT}"
expected_source = f"{expected_decorator}{SCRIPT}" if
expected_decorator else SCRIPT.lstrip()
```
##########
tests/utils/test_preexisting_python_virtualenv_decorator.py:
##########
@@ -17,25 +17,36 @@
# under the License.
from __future__ import annotations
-from airflow.utils.decorators import remove_task_decorator
+from textwrap import dedent
+import pytest
-class TestExternalPythonDecorator:
- def test_remove_task_decorator(self):
- py_source = '@task.external_python(serializer="dill")\ndef
f():\nimport funcsigs'
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.external_python")
- assert res == "def f():\nimport funcsigs"
+from airflow.utils.decorators import remove_task_decorator
- def test_remove_decorator_no_parens(self):
- py_source = "@task.external_python\ndef f():\nimport funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.external_python")
- assert res == "def f():\nimport funcsigs"
- def test_remove_decorator_nested(self):
- py_source = "@foo\[email protected]_python\n@bar\ndef f():\nimport
funcsigs"
- res = remove_task_decorator(python_source=py_source,
task_decorator_name="@task.external_python")
- assert res == "@foo\n@bar\ndef f():\nimport funcsigs"
+class TestExternalPythonDecorator:
+ @pytest.mark.parametrize(
+ "decorators, expected_decorators",
+ [
+ (["@task.external_python"], []),
+ (["@task.external_python()"], []),
+ (['@task.external_python(serializer="dill")'], []),
+ (["@foo", "@task.external_python", "@bar"], ["@foo", "@bar"]),
+ (["@foo", "@task.external_python()", "@bar"], ["@foo", "@bar"]),
+ ],
+ ids=["without_parens", "parens", "with_args", "nested_without_parens",
"nested_with_parens"],
+ )
+ def test_remove_task_decorator(self, decorators: list[str],
expected_decorators: list[str]):
+ decorator = "\n".join(decorators)
Review Comment:
same
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
