jscheffl commented on code in PR #40775:
URL: https://github.com/apache/airflow/pull/40775#discussion_r1683320345
##########
airflow/operators/python.py:
##########
@@ -770,87 +784,27 @@ def _calculate_cache_hash(self, exclude_cloudpickle: bool
= False) -> tuple[str,
Returns a hash and the data dict which is the base for the hash as
text.
"""
- hash_dict = {
- "requirements_list":
self._requirements_list(exclude_cloudpickle=exclude_cloudpickle),
- "pip_install_options": self.pip_install_options,
- "index_urls": self.index_urls,
- "cache_key":
str(Variable.get("PythonVirtualenvOperator.cache_key", "")),
- "python_version": self.python_version,
- "system_site_packages": self.system_site_packages,
- }
- hash_text = json.dumps(hash_dict, sort_keys=True)
- hash_object = hashlib_wrapper.md5(hash_text.encode())
- requirements_hash = hash_object.hexdigest()
- return requirements_hash[:8], hash_text
+ warnings.warn(
+ "PythonVirtualenvOperator._calculate_cache_hash is deprecated",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return
self.hook._calculate_cache_hash(exclude_cloudpickle=exclude_cloudpickle)
Review Comment:
Same here
##########
airflow/operators/python.py:
##########
@@ -728,33 +736,39 @@ def __init__(
**kwargs,
)
+ @property
+ def hook(self) -> PythonVirtualenvHook:
+ """Hook to manage the virtual environment."""
+ if self._hook is None:
+ self._hook = PythonVirtualenvHook(
+ venv_conn_id=self.venv_conn_id,
+ requirements=self.requirements,
+ python_version=self.python_version,
+ system_site_packages=self.system_site_packages,
+ pip_install_options=self.pip_install_options,
+ index_urls=self.index_urls,
+ venv_cache_path=self.venv_cache_path,
+ serializer=self.serializer,
+ )
+ return self._hook
+
def _requirements_list(self, exclude_cloudpickle: bool = False) ->
list[str]:
"""Prepare a list of requirements that need to be installed for the
virtual environment."""
- requirements = [str(dependency) for dependency in self.requirements]
- if not self.system_site_packages:
- if (
- self.serializer == "cloudpickle"
- and not exclude_cloudpickle
- and "cloudpickle" not in requirements
- ):
- requirements.append("cloudpickle")
- elif self.serializer == "dill" and "dill" not in requirements:
- requirements.append("dill")
- requirements.sort() # Ensure a hash is stable
- return requirements
+ warnings.warn(
+ "PythonVirtualenvOperator._requirements_list is deprecated",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return
self.hook._requirements_list(exclude_cloudpickle=exclude_cloudpickle)
def _prepare_venv(self, venv_path: Path) -> None:
"""Prepare the requirements and installs the virtual environment."""
- requirements_file = venv_path / "requirements.txt"
- requirements_file.write_text("\n".join(self._requirements_list()))
- prepare_virtualenv(
- venv_directory=str(venv_path),
- python_bin=f"python{self.python_version}" if self.python_version
else "python",
- system_site_packages=self.system_site_packages,
- requirements_file_path=str(requirements_file),
- pip_install_options=self.pip_install_options,
- index_urls=self.index_urls,
+ warnings.warn(
+ "PythonVirtualenvOperator._prepare_venv is deprecated",
+ DeprecationWarning,
+ stacklevel=2,
Review Comment:
Same here
##########
airflow/hooks/python_virtualenv.py:
##########
@@ -0,0 +1,353 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import sys
+from contextlib import contextmanager
+from functools import cached_property
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, Generator
+
+from airflow.exceptions import (
+ AirflowException,
+)
+from airflow.hooks.base import BaseHook
+from airflow.models.variable import Variable
+from airflow.utils import hashlib_wrapper
+from airflow.utils.python_virtualenv import prepare_virtualenv
+from airflow.utils.types import NOTSET, ArgNotSet
+
+if TYPE_CHECKING:
+ from airflow.typing_compat import Literal
+
+log = logging.getLogger(__name__)
+
+
+class PythonVirtualenvHook(BaseHook):
+ """
+ To run a function in a virtualenv, create virtualenv first and destroy
automatically.
+
+ :param venv_conn_id: The ID of Python Virtualenv Connection.
+ If set, the other arguments in the Operator will override the values
from the connection.
+ :param requirements: Either a list of requirement strings, or a (templated)
+ "requirements file" as specified by pip.
+ :param python_version: The Python version to run the virtual environment
with. Note that
+ both 2 and 2.7 are acceptable forms.
+ :param serializer: Which serializer use to serialize the args and result.
It can be one of the following:
+
+ - ``"pickle"``: (default) Use pickle for serialization. Included in
the Python Standard Library.
+ - ``"cloudpickle"``: Use cloudpickle for serialize more complex types,
+ this requires to include cloudpickle in your requirements.
+ - ``"dill"``: Use dill for serialize more complex types,
+ this requires to include dill in your requirements.
+ :param system_site_packages: Whether to include
+ system_site_packages in your virtual environment.
+ If not specified, system_site_packages will be set to ``True`` by
default.
+ See virtualenv documentation for more information.
+ :param pip_install_options: a list of pip install options when installing
requirements
+ See 'pip install -h' for available options
+ :param index_urls: an optional list of index urls to load Python packages
from.
+ If not provided the system pip conf will be used to source packages
from.
+ If set to ``[]``, `--no-index`` will be used to ignore package index.
+ :param venv_cache_path: Optional path to the virtual environment parent
folder in which the
+ virtual environment will be cached, creates a sub-folder venv-{hash}
whereas hash will be replaced
+ with a checksum of requirements. If not provided the virtual
environment will be created and deleted
+ in a temp folder for every execution.
+ """
+
+ conn_name_attr = "venv_conn_id"
+ default_conn_name = "venv_default"
+ conn_type = "python_venv"
+ hook_name = "Python Virtualenv"
+
+ @classmethod
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
+ v = sys.version_info
+ return {
+ "hidden_fields": ["host", "schema", "port", "login", "password",
"extra"],
+ "relabeling": {},
+ "placeholders": {"python_version": f"{v.major}.{v.minor}"},
+ }
+
+ @classmethod
+ def get_connection_form_widgets(cls) -> dict[str, Any]:
+ from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget,
BS3TextFieldWidget
+ from flask_babel import lazy_gettext
+ from wtforms import BooleanField, StringField
+ from wtforms.validators import InputRequired, Optional
+
+ return {
+ "requirements": StringField(
+ lazy_gettext("Requirements"),
+ description=lazy_gettext("Python requirement strings separated
by newline."),
+ widget=BS3TextAreaFieldWidget(),
+ validators=[InputRequired()],
+ ),
+ "python_version": StringField(
+ lazy_gettext("Python Version"),
+ description=lazy_gettext("The Python version to run the
virtual environment with."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "system_site_packages": BooleanField(
+ lazy_gettext("System Site Packages"),
+ description=lazy_gettext(
+ "Whether to include system_site_packages in the virtual
environment."
+ ),
+ default=False,
+ ),
+ "pip_install_options": StringField(
+ lazy_gettext("Pip Install Options"),
+ description=lazy_gettext("A list of pip install options when
installing requirements."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "index_urls": StringField(
+ lazy_gettext("Index URLs"),
+ description=lazy_gettext("Comma separated list of URLs to
search for packages."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "venv_cache_path": StringField(
+ lazy_gettext("Virtual Environment Cache Path"),
+ description=lazy_gettext("Path to cache the virtual
environment."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ }
+
+ def __init__(
+ self,
+ venv_conn_id: str | None = None,
+ requirements: None | list[str] = None,
+ python_version: str | None = None,
+ system_site_packages: bool | ArgNotSet = NOTSET,
+ pip_install_options: list[str] | None = None,
+ index_urls: None | list[str] = None,
+ venv_cache_path: None | os.PathLike[str] = None,
+ serializer: None | Literal["pickle", "cloudpickle", "dill"] = None,
+ ) -> None:
+ super().__init__()
+
+ self._venv_conn_id = venv_conn_id
+ self._requirements = requirements
+ self._python_version = python_version
+ self._system_site_packages = system_site_packages
+ self._pip_install_options = pip_install_options
+ self._index_urls = index_urls
+ self._venv_cache_path = venv_cache_path
+ self._serializer = serializer
+ self._conn = self.get_connection(venv_conn_id) if venv_conn_id else
None
+
+ def get_conn(self) -> None:
+ pass
+
+ @cached_property
+ def extra(self) -> dict[str, Any]:
+ if self._conn:
+ return self._conn.extra_dejson
+ return {}
+
+ @cached_property
+ def requirements(self) -> list[str]:
+ """Extends the requirements from PythonVirtualenvOperator with the
requirements from connection."""
+ requirements = []
+ if self._requirements:
+ requirements.extend([str(requirement) for requirement in
self._requirements])
+ if self.extra.get("requirements"):
+ requirements.extend(
+ [requirement.strip() for requirement in
self.extra["requirements"].split("\n")]
+ )
+ return requirements
+
+ @cached_property
+ def python_version(self) -> str | None:
+ return self._python_version or self.extra.get("python_version")
+
+ @cached_property
+ def system_site_packages(self) -> bool:
+ if self._system_site_packages is not NOTSET:
+ return bool(self._system_site_packages)
+ return self.extra.get("system_site_packages", True)
+
+ @cached_property
+ def pip_install_options(self) -> list[str]:
+ if self._pip_install_options:
+ return self._pip_install_options
+ if self.extra.get("pip_install_options"):
+ return self.extra["pip_install_options"].split()
+ return []
+
+ @cached_property
+ def index_urls(self) -> list[str] | None:
+ if self._index_urls:
+ return [index_url.strip() for index_url in self._index_urls]
+ if self.extra.get("index_urls"):
+ return [index_url.strip() for index_url in
self.extra["index_urls"].split(",")]
+ return None
+
+ @cached_property
+ def venv_cache_path(self) -> Path | None:
+ if self._venv_cache_path:
+ return Path(self._venv_cache_path)
+ if self.extra.get("venv_cache_path"):
+ return Path(self.extra["venv_cache_path"])
+ return None
+
+ def _requirements_list(self, exclude_cloudpickle: bool = False) ->
list[str]:
+ """Prepare a list of requirements that need to be installed for the
virtual environment."""
+ requirements = self.requirements
+ requirements.sort() # Ensure a hash is stable
+ if not self.system_site_packages:
+ serializer = self._serializer or "pickle"
+ if serializer == "cloudpickle" and not exclude_cloudpickle:
+ is_cloudpickle_installed = False
+ for requirement in requirements:
+ if requirement.lower().startswith("cloudpickle"):
+ is_cloudpickle_installed = True
+ break
+ if not is_cloudpickle_installed:
+ requirements.append("cloudpickle")
+ elif serializer == "dill":
+ is_dill_installed = False
+ for requirement in requirements:
+ if requirement.lower().startswith("dill"):
+ is_dill_installed = True
+ break
+ if not is_dill_installed:
+ requirements.append("dill")
+ return requirements
+
+ def _prepare_venv(self, venv_path: Path) -> None:
+ """Prepare the requirements and installs the virtual environment."""
+ requirements_file = venv_path / "requirements.txt"
+ requirements_file.write_text("\n".join(self._requirements_list()))
+ prepare_virtualenv(
+ venv_directory=str(venv_path),
+ python_bin=f"python{self.python_version}" if self.python_version
else "python",
+ system_site_packages=self.system_site_packages,
+ requirements_file_path=str(requirements_file),
+ pip_install_options=self.pip_install_options,
+ index_urls=self.index_urls,
+ )
+
+ def _calculate_cache_hash(self, exclude_cloudpickle: bool = False) ->
tuple[str, str]:
+ """
+ Generate the hash of the cache folder to use.
+
+ The following factors are used as input for the hash:
+ - (sorted) list of requirements
+ - pip install options
+ - flag of system site packages
+ - python version
+ - Variable to override the hash with a cache key
+ - Index URLs
+
+ Returns a hash and the data dict which is the base for the hash as
text.
+ """
+ hash_dict = {
+ "requirements_list":
self._requirements_list(exclude_cloudpickle=exclude_cloudpickle),
+ "pip_install_options": self.pip_install_options,
+ "index_urls": self.index_urls,
+ "cache_key":
str(Variable.get("PythonVirtualenvOperator.cache_key", "")),
+ "python_version": self.python_version,
+ "system_site_packages": self.system_site_packages,
+ }
+ hash_text = json.dumps(hash_dict, sort_keys=True)
+ hash_object = hashlib_wrapper.md5(hash_text.encode())
+ requirements_hash = hash_object.hexdigest()
+ return requirements_hash[:8], hash_text
+
+ def _ensure_venv_cache_exists(self, venv_cache_path: Path) -> Path:
+ """Ensure a valid virtual environment is set up and will create
inplace."""
+ cache_hash, hash_data = self._calculate_cache_hash()
+ venv_path = venv_cache_path / f"venv-{cache_hash}"
+ self.log.info("Python virtual environment will be cached in %s",
venv_path)
+ venv_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(f"{venv_path}.lock", "w") as f:
+ # Ensure that cache is not build by parallel workers
+ import fcntl
+
+ fcntl.flock(f, fcntl.LOCK_EX)
+
+ hash_marker = venv_path / "install_complete_marker.json"
+ try:
+ if venv_path.exists():
+ if hash_marker.exists():
+ previous_hash_data =
hash_marker.read_text(encoding="utf8")
+ if previous_hash_data == hash_data:
+ self.log.info("Re-using cached Python virtual
environment in %s", venv_path)
+ return venv_path
+
+ _, hash_data_before_upgrade =
self._calculate_cache_hash(exclude_cloudpickle=True)
+ if previous_hash_data == hash_data_before_upgrade:
+ self.log.warning(
+ "Found a previous virtual environment in with
outdated dependencies %s, "
+ "deleting and re-creating.",
+ venv_path,
+ )
+ else:
+ self.log.error(
+ "Unicorn alert: Found a previous virtual
environment in %s "
+ "with the same hash but different parameters.
Previous setup: '%s' / "
+ "Requested venv setup: '%s'. Please report a
bug to airflow!",
+ venv_path,
+ previous_hash_data,
+ hash_data,
+ )
+ else:
+ self.log.warning(
+ "Found a previous (probably partial installed)
virtual environment in %s, "
+ "deleting and re-creating.",
+ venv_path,
+ )
+
+ shutil.rmtree(venv_path)
+
Review Comment:
Update after reading below: The function in my eyes should NOT be moved
outside of the operator. Venv install should be core competency of the
operator, the hook might be helping of details
##########
airflow/operators/python.py:
##########
@@ -770,87 +784,27 @@ def _calculate_cache_hash(self, exclude_cloudpickle: bool
= False) -> tuple[str,
Returns a hash and the data dict which is the base for the hash as
text.
"""
- hash_dict = {
- "requirements_list":
self._requirements_list(exclude_cloudpickle=exclude_cloudpickle),
- "pip_install_options": self.pip_install_options,
- "index_urls": self.index_urls,
- "cache_key":
str(Variable.get("PythonVirtualenvOperator.cache_key", "")),
- "python_version": self.python_version,
- "system_site_packages": self.system_site_packages,
- }
- hash_text = json.dumps(hash_dict, sort_keys=True)
- hash_object = hashlib_wrapper.md5(hash_text.encode())
- requirements_hash = hash_object.hexdigest()
- return requirements_hash[:8], hash_text
+ warnings.warn(
+ "PythonVirtualenvOperator._calculate_cache_hash is deprecated",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return
self.hook._calculate_cache_hash(exclude_cloudpickle=exclude_cloudpickle)
def _ensure_venv_cache_exists(self, venv_cache_path: Path) -> Path:
"""Ensure a valid virtual environment is set up and will create
inplace."""
- cache_hash, hash_data = self._calculate_cache_hash()
- venv_path = venv_cache_path / f"venv-{cache_hash}"
- self.log.info("Python virtual environment will be cached in %s",
venv_path)
- venv_path.parent.mkdir(parents=True, exist_ok=True)
- with open(f"{venv_path}.lock", "w") as f:
- # Ensure that cache is not build by parallel workers
- import fcntl
-
- fcntl.flock(f, fcntl.LOCK_EX)
-
- hash_marker = venv_path / "install_complete_marker.json"
- try:
- if venv_path.exists():
- if hash_marker.exists():
- previous_hash_data =
hash_marker.read_text(encoding="utf8")
- if previous_hash_data == hash_data:
- self.log.info("Re-using cached Python virtual
environment in %s", venv_path)
- return venv_path
-
- _, hash_data_before_upgrade =
self._calculate_cache_hash(exclude_cloudpickle=True)
- if previous_hash_data == hash_data_before_upgrade:
- self.log.warning(
- "Found a previous virtual environment in with
outdated dependencies %s, "
- "deleting and re-creating.",
- venv_path,
- )
- else:
- self.log.error(
- "Unicorn alert: Found a previous virtual
environment in %s "
- "with the same hash but different parameters.
Previous setup: '%s' / "
- "Requested venv setup: '%s'. Please report a
bug to airflow!",
- venv_path,
- previous_hash_data,
- hash_data,
- )
- else:
- self.log.warning(
- "Found a previous (probably partial installed)
virtual environment in %s, "
- "deleting and re-creating.",
- venv_path,
- )
-
- shutil.rmtree(venv_path)
-
- venv_path.mkdir(parents=True)
- self._prepare_venv(venv_path)
- hash_marker.write_text(hash_data, encoding="utf8")
- except Exception as e:
- shutil.rmtree(venv_path)
- raise AirflowException(f"Unable to create new virtual
environment in {venv_path}") from e
- self.log.info("New Python virtual environment created in %s",
venv_path)
- return venv_path
+ warnings.warn(
+ "PythonVirtualenvOperator._ensure_venv_cache_exists is deprecated",
+ DeprecationWarning,
+ stacklevel=2,
+ )
Review Comment:
Same here
##########
airflow/providers_manager.py:
##########
@@ -467,7 +468,7 @@ def _init_airflow_core_hooks(self):
connection_type=None,
connection_testable=False,
)
- for cls in [FSHook, PackageIndexHook]:
+ for cls in [FSHook, PackageIndexHook, PythonVirtualenvHook]:
Review Comment:
Actually it would be the right place to add. Have not seen this error before.
##########
airflow/operators/python.py:
##########
@@ -728,33 +736,39 @@ def __init__(
**kwargs,
)
+ @property
+ def hook(self) -> PythonVirtualenvHook:
+ """Hook to manage the virtual environment."""
+ if self._hook is None:
+ self._hook = PythonVirtualenvHook(
+ venv_conn_id=self.venv_conn_id,
+ requirements=self.requirements,
+ python_version=self.python_version,
+ system_site_packages=self.system_site_packages,
+ pip_install_options=self.pip_install_options,
+ index_urls=self.index_urls,
+ venv_cache_path=self.venv_cache_path,
+ serializer=self.serializer,
+ )
+ return self._hook
+
def _requirements_list(self, exclude_cloudpickle: bool = False) ->
list[str]:
"""Prepare a list of requirements that need to be installed for the
virtual environment."""
- requirements = [str(dependency) for dependency in self.requirements]
- if not self.system_site_packages:
- if (
- self.serializer == "cloudpickle"
- and not exclude_cloudpickle
- and "cloudpickle" not in requirements
- ):
- requirements.append("cloudpickle")
- elif self.serializer == "dill" and "dill" not in requirements:
- requirements.append("dill")
- requirements.sort() # Ensure a hash is stable
- return requirements
+ warnings.warn(
+ "PythonVirtualenvOperator._requirements_list is deprecated",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return
self.hook._requirements_list(exclude_cloudpickle=exclude_cloudpickle)
Review Comment:
This is a private method, in my view no need to deprecate. Whomever is using
this outside of this code has bac luck.
##########
airflow/hooks/python_virtualenv.py:
##########
@@ -0,0 +1,353 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import sys
+from contextlib import contextmanager
+from functools import cached_property
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, Generator
+
+from airflow.exceptions import (
+ AirflowException,
+)
+from airflow.hooks.base import BaseHook
+from airflow.models.variable import Variable
+from airflow.utils import hashlib_wrapper
+from airflow.utils.python_virtualenv import prepare_virtualenv
+from airflow.utils.types import NOTSET, ArgNotSet
+
+if TYPE_CHECKING:
+ from airflow.typing_compat import Literal
+
+log = logging.getLogger(__name__)
+
+
+class PythonVirtualenvHook(BaseHook):
+ """
+ To run a function in a virtualenv, create virtualenv first and destroy
automatically.
+
+ :param venv_conn_id: The ID of Python Virtualenv Connection.
+ If set, the other arguments in the Operator will override the values
from the connection.
+ :param requirements: Either a list of requirement strings, or a (templated)
+ "requirements file" as specified by pip.
+ :param python_version: The Python version to run the virtual environment
with. Note that
+ both 2 and 2.7 are acceptable forms.
+ :param serializer: Which serializer use to serialize the args and result.
It can be one of the following:
+
+ - ``"pickle"``: (default) Use pickle for serialization. Included in
the Python Standard Library.
+ - ``"cloudpickle"``: Use cloudpickle for serialize more complex types,
+ this requires to include cloudpickle in your requirements.
+ - ``"dill"``: Use dill for serialize more complex types,
+ this requires to include dill in your requirements.
+ :param system_site_packages: Whether to include
+ system_site_packages in your virtual environment.
+ If not specified, system_site_packages will be set to ``True`` by
default.
+ See virtualenv documentation for more information.
+ :param pip_install_options: a list of pip install options when installing
requirements
+ See 'pip install -h' for available options
+ :param index_urls: an optional list of index urls to load Python packages
from.
+ If not provided the system pip conf will be used to source packages
from.
+ If set to ``[]``, `--no-index`` will be used to ignore package index.
+ :param venv_cache_path: Optional path to the virtual environment parent
folder in which the
+ virtual environment will be cached, creates a sub-folder venv-{hash}
whereas hash will be replaced
+ with a checksum of requirements. If not provided the virtual
environment will be created and deleted
+ in a temp folder for every execution.
+ """
+
+ conn_name_attr = "venv_conn_id"
+ default_conn_name = "venv_default"
+ conn_type = "python_venv"
+ hook_name = "Python Virtualenv"
+
+ @classmethod
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
+ v = sys.version_info
+ return {
+ "hidden_fields": ["host", "schema", "port", "login", "password",
"extra"],
+ "relabeling": {},
+ "placeholders": {"python_version": f"{v.major}.{v.minor}"},
+ }
+
+ @classmethod
+ def get_connection_form_widgets(cls) -> dict[str, Any]:
+ from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget,
BS3TextFieldWidget
+ from flask_babel import lazy_gettext
+ from wtforms import BooleanField, StringField
+ from wtforms.validators import InputRequired, Optional
+
+ return {
+ "requirements": StringField(
+ lazy_gettext("Requirements"),
+ description=lazy_gettext("Python requirement strings separated
by newline."),
+ widget=BS3TextAreaFieldWidget(),
+ validators=[InputRequired()],
+ ),
+ "python_version": StringField(
+ lazy_gettext("Python Version"),
+ description=lazy_gettext("The Python version to run the
virtual environment with."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "system_site_packages": BooleanField(
+ lazy_gettext("System Site Packages"),
+ description=lazy_gettext(
+ "Whether to include system_site_packages in the virtual
environment."
+ ),
+ default=False,
+ ),
+ "pip_install_options": StringField(
+ lazy_gettext("Pip Install Options"),
+ description=lazy_gettext("A list of pip install options when
installing requirements."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "index_urls": StringField(
+ lazy_gettext("Index URLs"),
+ description=lazy_gettext("Comma separated list of URLs to
search for packages."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "venv_cache_path": StringField(
+ lazy_gettext("Virtual Environment Cache Path"),
+ description=lazy_gettext("Path to cache the virtual
environment."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ }
+
+ def __init__(
+ self,
+ venv_conn_id: str | None = None,
+ requirements: None | list[str] = None,
+ python_version: str | None = None,
+ system_site_packages: bool | ArgNotSet = NOTSET,
+ pip_install_options: list[str] | None = None,
+ index_urls: None | list[str] = None,
+ venv_cache_path: None | os.PathLike[str] = None,
+ serializer: None | Literal["pickle", "cloudpickle", "dill"] = None,
+ ) -> None:
+ super().__init__()
+
+ self._venv_conn_id = venv_conn_id
+ self._requirements = requirements
+ self._python_version = python_version
+ self._system_site_packages = system_site_packages
+ self._pip_install_options = pip_install_options
+ self._index_urls = index_urls
+ self._venv_cache_path = venv_cache_path
+ self._serializer = serializer
+ self._conn = self.get_connection(venv_conn_id) if venv_conn_id else
None
+
+ def get_conn(self) -> None:
+ pass
+
+ @cached_property
+ def extra(self) -> dict[str, Any]:
+ if self._conn:
+ return self._conn.extra_dejson
+ return {}
+
+ @cached_property
+ def requirements(self) -> list[str]:
+ """Extends the requirements from PythonVirtualenvOperator with the
requirements from connection."""
+ requirements = []
+ if self._requirements:
+ requirements.extend([str(requirement) for requirement in
self._requirements])
+ if self.extra.get("requirements"):
+ requirements.extend(
+ [requirement.strip() for requirement in
self.extra["requirements"].split("\n")]
+ )
+ return requirements
+
+ @cached_property
+ def python_version(self) -> str | None:
+ return self._python_version or self.extra.get("python_version")
+
+ @cached_property
+ def system_site_packages(self) -> bool:
+ if self._system_site_packages is not NOTSET:
+ return bool(self._system_site_packages)
+ return self.extra.get("system_site_packages", True)
+
+ @cached_property
+ def pip_install_options(self) -> list[str]:
+ if self._pip_install_options:
+ return self._pip_install_options
+ if self.extra.get("pip_install_options"):
+ return self.extra["pip_install_options"].split()
+ return []
+
+ @cached_property
+ def index_urls(self) -> list[str] | None:
+ if self._index_urls:
+ return [index_url.strip() for index_url in self._index_urls]
+ if self.extra.get("index_urls"):
+ return [index_url.strip() for index_url in
self.extra["index_urls"].split(",")]
+ return None
+
+ @cached_property
+ def venv_cache_path(self) -> Path | None:
+ if self._venv_cache_path:
+ return Path(self._venv_cache_path)
+ if self.extra.get("venv_cache_path"):
+ return Path(self.extra["venv_cache_path"])
+ return None
+
+ def _requirements_list(self, exclude_cloudpickle: bool = False) ->
list[str]:
+ """Prepare a list of requirements that need to be installed for the
virtual environment."""
+ requirements = self.requirements
+ requirements.sort() # Ensure a hash is stable
+ if not self.system_site_packages:
+ serializer = self._serializer or "pickle"
+ if serializer == "cloudpickle" and not exclude_cloudpickle:
+ is_cloudpickle_installed = False
+ for requirement in requirements:
+ if requirement.lower().startswith("cloudpickle"):
+ is_cloudpickle_installed = True
+ break
+ if not is_cloudpickle_installed:
+ requirements.append("cloudpickle")
+ elif serializer == "dill":
+ is_dill_installed = False
+ for requirement in requirements:
+ if requirement.lower().startswith("dill"):
+ is_dill_installed = True
+ break
+ if not is_dill_installed:
+ requirements.append("dill")
+ return requirements
+
+ def _prepare_venv(self, venv_path: Path) -> None:
+ """Prepare the requirements and installs the virtual environment."""
+ requirements_file = venv_path / "requirements.txt"
+ requirements_file.write_text("\n".join(self._requirements_list()))
+ prepare_virtualenv(
+ venv_directory=str(venv_path),
+ python_bin=f"python{self.python_version}" if self.python_version
else "python",
+ system_site_packages=self.system_site_packages,
+ requirements_file_path=str(requirements_file),
+ pip_install_options=self.pip_install_options,
+ index_urls=self.index_urls,
+ )
+
+ def _calculate_cache_hash(self, exclude_cloudpickle: bool = False) ->
tuple[str, str]:
+ """
+ Generate the hash of the cache folder to use.
+
+ The following factors are used as input for the hash:
+ - (sorted) list of requirements
+ - pip install options
+ - flag of system site packages
+ - python version
+ - Variable to override the hash with a cache key
+ - Index URLs
+
+ Returns a hash and the data dict which is the base for the hash as
text.
+ """
+ hash_dict = {
+ "requirements_list":
self._requirements_list(exclude_cloudpickle=exclude_cloudpickle),
+ "pip_install_options": self.pip_install_options,
+ "index_urls": self.index_urls,
+ "cache_key":
str(Variable.get("PythonVirtualenvOperator.cache_key", "")),
+ "python_version": self.python_version,
+ "system_site_packages": self.system_site_packages,
+ }
+ hash_text = json.dumps(hash_dict, sort_keys=True)
+ hash_object = hashlib_wrapper.md5(hash_text.encode())
+ requirements_hash = hash_object.hexdigest()
+ return requirements_hash[:8], hash_text
+
+ def _ensure_venv_cache_exists(self, venv_cache_path: Path) -> Path:
+ """Ensure a valid virtual environment is set up and will create
inplace."""
+ cache_hash, hash_data = self._calculate_cache_hash()
+ venv_path = venv_cache_path / f"venv-{cache_hash}"
+ self.log.info("Python virtual environment will be cached in %s",
venv_path)
+ venv_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(f"{venv_path}.lock", "w") as f:
+ # Ensure that cache is not build by parallel workers
+ import fcntl
+
+ fcntl.flock(f, fcntl.LOCK_EX)
+
+ hash_marker = venv_path / "install_complete_marker.json"
+ try:
+ if venv_path.exists():
+ if hash_marker.exists():
+ previous_hash_data =
hash_marker.read_text(encoding="utf8")
+ if previous_hash_data == hash_data:
+ self.log.info("Re-using cached Python virtual
environment in %s", venv_path)
+ return venv_path
+
+ _, hash_data_before_upgrade =
self._calculate_cache_hash(exclude_cloudpickle=True)
+ if previous_hash_data == hash_data_before_upgrade:
+ self.log.warning(
+ "Found a previous virtual environment in with
outdated dependencies %s, "
+ "deleting and re-creating.",
+ venv_path,
+ )
+ else:
+ self.log.error(
+ "Unicorn alert: Found a previous virtual
environment in %s "
+ "with the same hash but different parameters.
Previous setup: '%s' / "
+ "Requested venv setup: '%s'. Please report a
bug to airflow!",
+ venv_path,
+ previous_hash_data,
+ hash_data,
+ )
+ else:
+ self.log.warning(
+ "Found a previous (probably partial installed)
virtual environment in %s, "
+ "deleting and re-creating.",
+ venv_path,
+ )
+
+ shutil.rmtree(venv_path)
+
Review Comment:
The functions here are a 1:1 copy of the code of the
PythonVirtualenvOperator. That code should not be copied. I rather propose if a
test is needed (which I think is very probably not a good idea because it
pollutes the disk of the webserver) then the code should be imported from the
Operator, not copied.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]