This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 463e28b9ce add `env_vars` into `_BasePythonVirtualenvOperator` (#40942)
463e28b9ce is described below
commit 463e28b9ce6b58a64795c96ae7b94aba37770c93
Author: Youngha, Park <[email protected]>
AuthorDate: Fri Aug 9 00:09:13 2024 +0900
add `env_vars` into `_BasePythonVirtualenvOperator` (#40942)
* add env_vars into _BasePythonVirtualenvOperator
* fix BranchPythonVirtualenvOperator
* update feedback
---
airflow/decorators/__init__.pyi | 16 +++++++
airflow/operators/python.py | 31 ++++++++++++-
airflow/utils/process_utils.py | 7 ++-
tests/operators/test_python.py | 93 +++++++++++++++++++++++++++++++++++++++
tests/utils/test_process_utils.py | 4 ++
5 files changed, 148 insertions(+), 3 deletions(-)
diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi
index 9b4fbccc2e..faf77e8240 100644
--- a/airflow/decorators/__init__.pyi
+++ b/airflow/decorators/__init__.pyi
@@ -122,6 +122,8 @@ class TaskDecoratorCollection:
index_urls: None | Collection[str] | str = None,
venv_cache_path: None | str = None,
show_return_value_in_logs: bool = True,
+ env_vars: dict[str, str] | None = None,
+ inherit_env: bool = True,
use_dill: bool = False,
**kwargs,
) -> TaskDecorator:
@@ -165,6 +167,12 @@ class TaskDecoratorCollection:
logs. Defaults to True, which allows return value log output.
It can be set to False to prevent log output of return value when
you return huge data
such as transmission a large amount of XCom to TaskAPI.
+ :param env_vars: A dictionary containing additional environment
variables to set for the virtual
+ environment when it is executed.
+ :param inherit_env: Whether to inherit the current environment
variables when executing the virtual
+ environment. If set to ``True``, the virtual environment will
inherit the environment variables
+ of the parent process (``os.environ``). If set to ``False``, the
virtual environment will be
+ executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to
use dill to serialize
the args and result (pickle is default). This allows more complex
types
but requires you to include dill in your requirements.
@@ -181,6 +189,8 @@ class TaskDecoratorCollection:
serializer: Literal["pickle", "cloudpickle", "dill"] | None = None,
templates_dict: Mapping[str, Any] | None = None,
show_return_value_in_logs: bool = True,
+ env_vars: dict[str, str] | None = None,
+ inherit_env: bool = True,
use_dill: bool = False,
**kwargs,
) -> TaskDecorator:
@@ -206,6 +216,12 @@ class TaskDecoratorCollection:
logs. Defaults to True, which allows return value log output.
It can be set to False to prevent log output of return value when
you return huge data
such as transmission a large amount of XCom to TaskAPI.
+ :param env_vars: A dictionary containing additional environment
variables to set for the virtual
+ environment when it is executed.
+ :param inherit_env: Whether to inherit the current environment
variables when executing the virtual
+ environment. If set to ``True``, the virtual environment will
inherit the environment variables
+ of the parent process (``os.environ``). If set to ``False``, the
virtual environment will be
+ executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to
use dill to serialize
the args and result (pickle is default). This allows more complex
types
but requires you to include dill in your requirements.
diff --git a/airflow/operators/python.py b/airflow/operators/python.py
index 46dbff9f01..fdfe575fb9 100644
--- a/airflow/operators/python.py
+++ b/airflow/operators/python.py
@@ -439,6 +439,8 @@ class _BasePythonVirtualenvOperator(PythonOperator,
metaclass=ABCMeta):
templates_exts: list[str] | None = None,
expect_airflow: bool = True,
skip_on_exit_code: int | Container[int] | None = None,
+ env_vars: dict[str, str] | None = None,
+ inherit_env: bool = True,
use_dill: bool = False,
**kwargs,
):
@@ -490,6 +492,8 @@ class _BasePythonVirtualenvOperator(PythonOperator,
metaclass=ABCMeta):
if skip_on_exit_code is not None
else []
)
+ self.env_vars = env_vars
+ self.inherit_env = inherit_env
@abstractmethod
def _iter_serializable_context_keys(self):
@@ -558,6 +562,10 @@ class _BasePythonVirtualenvOperator(PythonOperator,
metaclass=ABCMeta):
render_template_as_native_obj=self.dag.render_template_as_native_obj,
)
+ env_vars = dict(os.environ) if self.inherit_env else {}
+ if self.env_vars:
+ env_vars.update(self.env_vars)
+
try:
execute_in_subprocess(
cmd=[
@@ -567,7 +575,8 @@ class _BasePythonVirtualenvOperator(PythonOperator,
metaclass=ABCMeta):
os.fspath(output_path),
os.fspath(string_args_path),
os.fspath(termination_log_path),
- ]
+ ],
+ env=env_vars,
)
except subprocess.CalledProcessError as e:
if e.returncode in self.skip_on_exit_code:
@@ -648,6 +657,12 @@ class
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
virtual environment will be cached, creates a sub-folder venv-{hash}
whereas hash will be replaced
with a checksum of requirements. If not provided the virtual
environment will be created and deleted
in a temp folder for every execution.
+ :param env_vars: A dictionary containing additional environment variables
to set for the virtual
+ environment when it is executed.
+ :param inherit_env: Whether to inherit the current environment variables
when executing the virtual
+ environment. If set to ``True``, the virtual environment will inherit
the environment variables
+ of the parent process (``os.environ``). If set to ``False``, the
virtual environment will be
+ executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use
dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
@@ -676,6 +691,8 @@ class
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
skip_on_exit_code: int | Container[int] | None = None,
index_urls: None | Collection[str] | str = None,
venv_cache_path: None | os.PathLike[str] = None,
+ env_vars: dict[str, str] | None = None,
+ inherit_env: bool = True,
use_dill: bool = False,
**kwargs,
):
@@ -724,6 +741,8 @@ class
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
templates_exts=templates_exts,
expect_airflow=expect_airflow,
skip_on_exit_code=skip_on_exit_code,
+ env_vars=env_vars,
+ inherit_env=inherit_env,
use_dill=use_dill,
**kwargs,
)
@@ -934,6 +953,12 @@ class
ExternalPythonOperator(_BasePythonVirtualenvOperator):
:param skip_on_exit_code: If python_callable exits with this exit code,
leave the task
in ``skipped`` state (default: None). If set to ``None``, any non-zero
exit code will be treated as a failure.
+ :param env_vars: A dictionary containing additional environment variables
to set for the virtual
+ environment when it is executed.
+ :param inherit_env: Whether to inherit the current environment variables
when executing the virtual
+ environment. If set to ``True``, the virtual environment will inherit
the environment variables
+ of the parent process (``os.environ``). If set to ``False``, the
virtual environment will be
+ executed with a clean environment.
:param use_dill: Deprecated, use ``serializer`` instead. Whether to use
dill to serialize
the args and result (pickle is default). This allows more complex types
but requires you to include dill in your requirements.
@@ -955,6 +980,8 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
expect_airflow: bool = True,
expect_pendulum: bool = False,
skip_on_exit_code: int | Container[int] | None = None,
+ env_vars: dict[str, str] | None = None,
+ inherit_env: bool = True,
use_dill: bool = False,
**kwargs,
):
@@ -972,6 +999,8 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
templates_exts=templates_exts,
expect_airflow=expect_airflow,
skip_on_exit_code=skip_on_exit_code,
+ env_vars=env_vars,
+ inherit_env=inherit_env,
use_dill=use_dill,
**kwargs,
)
diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py
index e8b3e8840d..7120bd5df8 100644
--- a/airflow/utils/process_utils.py
+++ b/airflow/utils/process_utils.py
@@ -162,14 +162,17 @@ def reap_process_group(
return returncodes
-def execute_in_subprocess(cmd: list[str], cwd: str | None = None) -> None:
+def execute_in_subprocess(cmd: list[str], cwd: str | None = None, env: dict |
None = None) -> None:
"""
Execute a process and stream output to logger.
:param cmd: command and arguments to run
:param cwd: Current working directory passed to the Popen constructor
+ :param env: Additional environment variables to set for the subprocess. If
None,
+ the subprocess will inherit the current environment variables of the
parent process
+ (``os.environ``)
"""
- execute_in_subprocess_with_kwargs(cmd, cwd=cwd)
+ execute_in_subprocess_with_kwargs(cmd, cwd=cwd, env=env)
def execute_in_subprocess_with_kwargs(cmd: list[str], **kwargs) -> None:
diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py
index 893a215d9d..993d70cad3 100644
--- a/tests/operators/test_python.py
+++ b/tests/operators/test_python.py
@@ -963,6 +963,48 @@ class BaseTestPythonVirtualenvOperator(BasePythonTest):
self.run_as_task(f, op_args=[42], serializer=serializer)
assert f"Unable to import `{serializer}` module." in caplog.text
+ def test_environment_variables(self):
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ task = self.run_as_task(f, env_vars={"MY_ENV_VAR": "ABCDE"})
+ assert task.execute_callable() == "ABCDE"
+
+ def test_environment_variables_with_inherit_env_true(self, monkeypatch):
+ monkeypatch.setenv("MY_ENV_VAR", "QWERT")
+
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ task = self.run_as_task(f, inherit_env=True)
+ assert task.execute_callable() == "QWERT"
+
+ def test_environment_variables_with_inherit_env_false(self, monkeypatch):
+ monkeypatch.setenv("MY_ENV_VAR", "TYUIO")
+
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ with pytest.raises(AirflowException):
+ self.run_as_task(f, inherit_env=False)
+
+ def test_environment_variables_overriding(self, monkeypatch):
+ monkeypatch.setenv("MY_ENV_VAR", "ABCDE")
+
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ task = self.run_as_task(f, env_vars={"MY_ENV_VAR": "EFGHI"},
inherit_env=True)
+ assert task.execute_callable() == "EFGHI"
+
venv_cache_path = tempfile.mkdtemp(prefix="venv_cache_path")
@@ -1489,6 +1531,57 @@ class
BaseTestBranchPythonVirtualenvOperator(BaseTestPythonVirtualenvOperator):
with pytest.raises(AirflowException, match="Invalid tasks found:"):
self.run_as_task(f, templates_dict={"ds": "{{ ds }}"})
+ def test_environment_variables(self):
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ with pytest.raises(
+ AirflowException,
+ match=r"'branch_task_ids' must contain only valid task_ids.
Invalid tasks found: {'ABCDE'}",
+ ):
+ self.run_as_task(f, env_vars={"MY_ENV_VAR": "ABCDE"})
+
+ def test_environment_variables_with_inherit_env_true(self, monkeypatch):
+ monkeypatch.setenv("MY_ENV_VAR", "QWERT")
+
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ with pytest.raises(
+ AirflowException,
+ match=r"'branch_task_ids' must contain only valid task_ids.
Invalid tasks found: {'QWERT'}",
+ ):
+ self.run_as_task(f, inherit_env=True)
+
+ def test_environment_variables_with_inherit_env_false(self, monkeypatch):
+ monkeypatch.setenv("MY_ENV_VAR", "TYUIO")
+
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ with pytest.raises(AirflowException):
+ self.run_as_task(f, inherit_env=False)
+
+ def test_environment_variables_overriding(self, monkeypatch):
+ monkeypatch.setenv("MY_ENV_VAR", "ABCDE")
+
+ def f():
+ import os
+
+ return os.environ["MY_ENV_VAR"]
+
+ with pytest.raises(
+ AirflowException,
+ match=r"'branch_task_ids' must contain only valid task_ids.
Invalid tasks found: {'EFGHI'}",
+ ):
+ self.run_as_task(f, env_vars={"MY_ENV_VAR": "EFGHI"},
inherit_env=True)
+
def test_with_no_caching(self):
"""
Most of venv tests use caching to speed up the tests. This test
ensures that
diff --git a/tests/utils/test_process_utils.py
b/tests/utils/test_process_utils.py
index 54bdc5bf9f..bf9ecf4352 100644
--- a/tests/utils/test_process_utils.py
+++ b/tests/utils/test_process_utils.py
@@ -120,6 +120,10 @@ class TestExecuteInSubProcess:
"KITTY",
] == msgs
+ def test_using_env_works(self, caplog):
+ execute_in_subprocess(["bash", "-c", 'echo "My value is ${VALUE}"'],
env=dict(VALUE="1"))
+ assert "My value is 1" in caplog.text
+
def test_should_raise_exception(self):
with pytest.raises(CalledProcessError):
process_utils.execute_in_subprocess(["bash", "-c", "exit 1"])