This is an automated email from the ASF dual-hosted git repository.
taragolis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new eb450b95fd Refactor tempfile to tmp_path in some tests (#34400)
eb450b95fd is described below
commit eb450b95fdb29b4c1252c2a1ccb703ac397a2de8
Author: Miroslav Šedivý <[email protected]>
AuthorDate: Mon Sep 18 19:15:38 2023 +0000
Refactor tempfile to tmp_path in some tests (#34400)
---
docker_tests/test_prod_image.py | 49 +++++++--------
kubernetes_tests/test_kubernetes_pod_operator.py | 15 ++---
tests/always/test_secrets_local_filesystem.py | 28 ++++-----
tests/jobs/test_scheduler_job.py | 8 +--
tests/operators/test_bash.py | 76 +++++++++++-------------
5 files changed, 80 insertions(+), 96 deletions(-)
diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py
index 01e271a407..8fd467be58 100644
--- a/docker_tests/test_prod_image.py
+++ b/docker_tests/test_prod_image.py
@@ -19,9 +19,7 @@ from __future__ import annotations
import json
import os
import subprocess
-import tempfile
from importlib.util import find_spec
-from pathlib import Path
import pytest
@@ -196,27 +194,26 @@ class TestExecuteAsRoot:
]
)
- def test_run_custom_python_packages_as_root(self):
- with tempfile.TemporaryDirectory() as tmp_dir:
- (Path(tmp_dir) / "__init__.py").write_text("")
- (Path(tmp_dir) / "awesome.py").write_text('print("Awesome")')
-
- run_command(
- [
- "docker",
- "run",
- "--rm",
- "-e",
- f"PYTHONPATH={tmp_dir}",
- "-e",
- "PYTHONDONTWRITEBYTECODE=true",
- "-v",
- f"{tmp_dir}:{tmp_dir}",
- "--user",
- "0",
- docker_image,
- "python",
- "-c",
- "import awesome",
- ]
- )
+ def test_run_custom_python_packages_as_root(self, tmp_path):
+ (tmp_path / "__init__.py").write_text("")
+ (tmp_path / "awesome.py").write_text('print("Awesome")')
+
+ run_command(
+ [
+ "docker",
+ "run",
+ "--rm",
+ "-e",
+ f"PYTHONPATH={tmp_path}",
+ "-e",
+ "PYTHONDONTWRITEBYTECODE=true",
+ "-v",
+ f"{tmp_path}:{tmp_path}",
+ "--user",
+ "0",
+ docker_image,
+ "python",
+ "-c",
+ "import awesome",
+ ]
+ )
diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py
b/kubernetes_tests/test_kubernetes_pod_operator.py
index 899bcf1f6f..dd2f3068cc 100644
--- a/kubernetes_tests/test_kubernetes_pod_operator.py
+++ b/kubernetes_tests/test_kubernetes_pod_operator.py
@@ -22,7 +22,6 @@ import os
import shutil
import sys
from copy import copy
-from tempfile import NamedTemporaryFile
from unittest import mock
from unittest.mock import ANY, MagicMock
from uuid import uuid4
@@ -149,9 +148,8 @@ class TestKubernetesPodOperatorSystem:
return None
return ",".join([f"{key}={value}" for key, value in
enumerate(self.labels)])
- def test_do_xcom_push_defaults_false(self, kubeconfig_path,
mock_get_connection):
- with NamedTemporaryFile(prefix="kube_config", suffix=".cfg") as f:
- new_config_path = f.name
+ def test_do_xcom_push_defaults_false(self, kubeconfig_path,
mock_get_connection, tmp_path):
+ new_config_path = tmp_path / "kube_config.cfg"
shutil.copy(kubeconfig_path, new_config_path)
k = KubernetesPodOperator(
namespace="default",
@@ -162,13 +160,12 @@ class TestKubernetesPodOperatorSystem:
task_id=str(uuid4()),
in_cluster=False,
do_xcom_push=False,
- config_file=new_config_path,
+ config_file=os.fspath(new_config_path),
)
assert not k.do_xcom_push
- def test_config_path_move(self, kubeconfig_path, mock_get_connection):
- with NamedTemporaryFile(prefix="kube_config", suffix=".cfg") as f:
- new_config_path = f.name
+ def test_config_path_move(self, kubeconfig_path, mock_get_connection,
tmp_path):
+ new_config_path = tmp_path / "kube_config.cfg"
shutil.copy(kubeconfig_path, new_config_path)
k = KubernetesPodOperator(
@@ -181,7 +178,7 @@ class TestKubernetesPodOperatorSystem:
in_cluster=False,
do_xcom_push=False,
is_delete_operator_pod=False,
- config_file=new_config_path,
+ config_file=os.fspath(new_config_path),
)
context = create_context(k)
k.execute(context)
diff --git a/tests/always/test_secrets_local_filesystem.py
b/tests/always/test_secrets_local_filesystem.py
index f54e592d71..47899266c4 100644
--- a/tests/always/test_secrets_local_filesystem.py
+++ b/tests/always/test_secrets_local_filesystem.py
@@ -17,9 +17,9 @@
from __future__ import annotations
import json
+import os
import re
from contextlib import contextmanager
-from tempfile import NamedTemporaryFile
from unittest import mock
import pytest
@@ -421,13 +421,12 @@ class TestLoadConnection:
class TestLocalFileBackend:
- def test_should_read_variable(self):
- with NamedTemporaryFile(suffix="var.env") as tmp_file:
- tmp_file.write(b"KEY_A=VAL_A")
- tmp_file.flush()
- backend = LocalFilesystemBackend(variables_file_path=tmp_file.name)
- assert "VAL_A" == backend.get_variable("KEY_A")
- assert backend.get_variable("KEY_B") is None
+ def test_should_read_variable(self, tmp_path):
+ path = tmp_path / "testfile.var.env"
+ path.write_text("KEY_A=VAL_A")
+ backend = LocalFilesystemBackend(variables_file_path=os.fspath(path))
+ assert "VAL_A" == backend.get_variable("KEY_A")
+ assert backend.get_variable("KEY_B") is None
@conf_vars(
{
@@ -446,13 +445,12 @@ class TestLocalFileBackend:
assert "LocalFilesystemBackend" in backend_classes
assert Variable.get("KEY_A") == "VAL_A"
- def test_should_read_connection(self):
- with NamedTemporaryFile(suffix=".env") as tmp_file:
- tmp_file.write(b"CONN_A=mysql://host_a")
- tmp_file.flush()
- backend =
LocalFilesystemBackend(connections_file_path=tmp_file.name)
- assert "mysql://host_a" ==
backend.get_connection("CONN_A").get_uri()
- assert backend.get_variable("CONN_B") is None
+ def test_should_read_connection(self, tmp_path):
+ path = tmp_path / "testfile.env"
+ path.write_text("CONN_A=mysql://host_a")
+ backend = LocalFilesystemBackend(connections_file_path=os.fspath(path))
+ assert "mysql://host_a" == backend.get_connection("CONN_A").get_uri()
+ assert backend.get_variable("CONN_B") is None
def test_files_are_optional(self):
backend = LocalFilesystemBackend()
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index b2bbe34457..ae142cc128 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -22,9 +22,7 @@ import contextlib
import datetime
import logging
import os
-import shutil
from datetime import timedelta
-from tempfile import mkdtemp
from typing import Generator
from unittest import mock
from unittest.mock import MagicMock, patch
@@ -210,16 +208,14 @@ class TestSchedulerJob:
scheduler_job.heartrate = 0
run_job(scheduler_job, execute_callable=self.job_runner._execute)
- def test_no_orphan_process_will_be_left(self):
- empty_dir = mkdtemp()
+ def test_no_orphan_process_will_be_left(self, tmp_path):
current_process = psutil.Process()
old_children = current_process.children(recursive=True)
scheduler_job = Job(
executor=MockExecutor(do_update=False),
)
- self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=empty_dir, num_runs=1)
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.fspath(tmp_path), num_runs=1)
run_job(scheduler_job, execute_callable=self.job_runner._execute)
- shutil.rmtree(empty_dir)
# Remove potential noise created by previous tests.
current_children = set(current_process.children(recursive=True)) -
set(old_children)
diff --git a/tests/operators/test_bash.py b/tests/operators/test_bash.py
index 3c7c7d4748..61b08f25d3 100644
--- a/tests/operators/test_bash.py
+++ b/tests/operators/test_bash.py
@@ -20,7 +20,6 @@ from __future__ import annotations
import os
import signal
from datetime import datetime, timedelta
-from tempfile import NamedTemporaryFile, TemporaryDirectory
from time import sleep
from unittest import mock
@@ -46,7 +45,7 @@ class TestBashOperator:
(True, {"AIRFLOW_HOME": "OVERRIDDEN_AIRFLOW_HOME"},
"OVERRIDDEN_AIRFLOW_HOME"),
],
)
- def test_echo_env_variables(self, append_env, user_defined_env,
expected_airflow_home):
+ def test_echo_env_variables(self, append_env, user_defined_env,
expected_airflow_home, tmp_path):
"""
Test that env variables are exported correctly to the task bash
environment.
"""
@@ -75,28 +74,26 @@ class TestBashOperator:
external_trigger=False,
)
- with NamedTemporaryFile() as tmp_file:
- task = BashOperator(
- task_id="echo_env_vars",
- dag=dag,
- bash_command="echo $AIRFLOW_HOME>> {0};"
- "echo $PYTHONPATH>> {0};"
- "echo $AIRFLOW_CTX_DAG_ID >> {0};"
- "echo $AIRFLOW_CTX_TASK_ID>> {0};"
- "echo $AIRFLOW_CTX_EXECUTION_DATE>> {0};"
- "echo $AIRFLOW_CTX_DAG_RUN_ID>> {0};".format(tmp_file.name),
- append_env=append_env,
- env=user_defined_env,
- )
+ tmp_file = tmp_path / "testfile"
+ task = BashOperator(
+ task_id="echo_env_vars",
+ dag=dag,
+ bash_command="echo $AIRFLOW_HOME>> {0};"
+ "echo $PYTHONPATH>> {0};"
+ "echo $AIRFLOW_CTX_DAG_ID >> {0};"
+ "echo $AIRFLOW_CTX_TASK_ID>> {0};"
+ "echo $AIRFLOW_CTX_EXECUTION_DATE>> {0};"
+ "echo $AIRFLOW_CTX_DAG_RUN_ID>> {0};".format(tmp_file),
+ append_env=append_env,
+ env=user_defined_env,
+ )
- with mock.patch.dict(
- "os.environ", {"AIRFLOW_HOME": "MY_PATH_TO_AIRFLOW_HOME",
"PYTHONPATH": "AWESOME_PYTHONPATH"}
- ):
- task.run(utc_now, utc_now, ignore_first_depends_on_past=True,
ignore_ti_state=True)
+ with mock.patch.dict(
+ "os.environ", {"AIRFLOW_HOME": "MY_PATH_TO_AIRFLOW_HOME",
"PYTHONPATH": "AWESOME_PYTHONPATH"}
+ ):
+ task.run(utc_now, utc_now, ignore_first_depends_on_past=True,
ignore_ti_state=True)
- with open(tmp_file.name) as file:
- output = "".join(file.readlines())
- assert expected == output
+ assert expected == tmp_file.read_text()
@pytest.mark.parametrize(
"val,expected",
@@ -143,31 +140,30 @@ class TestBashOperator:
line = op.execute({})
assert line == val
- def test_cwd_does_not_exist(self):
+ def test_cwd_does_not_exist(self, tmp_path):
test_cmd = 'set -e; echo "xxxx" |tee outputs.txt'
- with TemporaryDirectory(prefix="test_command_with_cwd") as tmp_dir:
- # Get a nonexistent temporary directory to do the test
- pass
+ test_cwd_folder = os.fspath(tmp_path / "test_command_with_cwd")
# There should be no exceptions when creating the operator even the
`cwd` doesn't exist
- bash_operator = BashOperator(task_id="abc", bash_command=test_cmd,
cwd=tmp_dir)
- with pytest.raises(AirflowException, match=f"Can not find the cwd:
{tmp_dir}"):
+ bash_operator = BashOperator(task_id="abc", bash_command=test_cmd,
cwd=os.fspath(test_cwd_folder))
+ with pytest.raises(AirflowException, match=f"Can not find the cwd:
{test_cwd_folder}"):
bash_operator.execute({})
- def test_cwd_is_file(self):
+ def test_cwd_is_file(self, tmp_path):
test_cmd = 'set -e; echo "xxxx" |tee outputs.txt'
- with NamedTemporaryFile(suffix="var.env") as tmp_file:
- # Test if the cwd is a file_path
- with pytest.raises(AirflowException, match=f"The cwd
{tmp_file.name} must be a directory"):
- BashOperator(task_id="abc", bash_command=test_cmd,
cwd=tmp_file.name).execute({})
+ tmp_file = tmp_path / "testfile.var.env"
+ tmp_file.touch()
+ # Test if the cwd is a file_path
+ with pytest.raises(AirflowException, match=f"The cwd {tmp_file} must
be a directory"):
+ BashOperator(task_id="abc", bash_command=test_cmd,
cwd=os.fspath(tmp_file)).execute({})
- def test_valid_cwd(self):
+ def test_valid_cwd(self, tmp_path):
test_cmd = 'set -e; echo "xxxx" |tee outputs.txt'
- with TemporaryDirectory(prefix="test_command_with_cwd") as
test_cwd_folder:
- # Test everything went alright
- result = BashOperator(task_id="abc", bash_command=test_cmd,
cwd=test_cwd_folder).execute({})
- assert result == "xxxx"
- with open(f"{test_cwd_folder}/outputs.txt") as tmp_file:
- assert tmp_file.read().splitlines()[0] == "xxxx"
+ test_cwd_path = tmp_path / "test_command_with_cwd"
+ test_cwd_path.mkdir()
+ # Test everything went alright
+ result = BashOperator(task_id="abc", bash_command=test_cmd,
cwd=os.fspath(test_cwd_path)).execute({})
+ assert result == "xxxx"
+ assert (test_cwd_path / "outputs.txt").read_text().splitlines()[0] ==
"xxxx"
@pytest.mark.parametrize(
"extra_kwargs,actual_exit_code,expected_exc",
