This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d2c0bbebac Refactor: Simplify code in tests (#33293)
d2c0bbebac is described below
commit d2c0bbebacd1c58d357d647707a1bdb0c5b3b425
Author: Miroslav Šedivý <[email protected]>
AuthorDate: Sat Aug 12 18:44:35 2023 +0000
Refactor: Simplify code in tests (#33293)
---
tests/always/test_example_dags.py | 7 +++----
tests/always/test_project_structure.py | 4 ++--
tests/cli/commands/test_config_command.py | 4 +---
tests/decorators/test_python.py | 2 +-
tests/jobs/test_scheduler_job.py | 4 ++--
tests/models/test_dagrun.py | 2 +-
tests/operators/test_python.py | 8 ++++----
tests/serialization/test_dag_serialization.py | 2 +-
tests/system/providers/amazon/aws/example_ec2.py | 3 +--
tests/system/providers/amazon/aws/utils/ec2.py | 2 +-
tests/test_utils/providers.py | 2 +-
tests/www/test_utils.py | 6 +++---
tests/www/views/test_views_acl.py | 2 +-
tests/www/views/test_views_home.py | 6 +++---
14 files changed, 25 insertions(+), 29 deletions(-)
diff --git a/tests/always/test_example_dags.py
b/tests/always/test_example_dags.py
index 2666a5b93a..9e3f7b4099 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -61,16 +61,15 @@ def example_not_suspended_dags():
for example_dir in example_dirs:
candidates = glob(f"{AIRFLOW_SOURCES_ROOT.as_posix()}/{example_dir}",
recursive=True)
for candidate in candidates:
- if any(candidate.startswith(s) for s in
suspended_providers_folders):
- continue
- yield candidate
+ if not candidate.startswith(tuple(suspended_providers_folders)):
+ yield candidate
def example_dags_except_db_exception():
return [
dag_file
for dag_file in example_not_suspended_dags()
- if not any(dag_file.endswith(e) for e in NO_DB_QUERY_EXCEPTION)
+ if not dag_file.endswith(tuple(NO_DB_QUERY_EXCEPTION))
]
diff --git a/tests/always/test_project_structure.py
b/tests/always/test_project_structure.py
index b17e285bdb..116f8f99d2 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -150,7 +150,7 @@ class ProjectStructureTest:
if not isinstance(current_node, ast.ClassDef):
continue
name = current_node.name
- if not any(name.endswith(suffix) for suffix in
self.CLASS_SUFFIXES):
+ if not name.endswith(tuple(self.CLASS_SUFFIXES)):
continue
results[f"{module}.{name}"] = current_node
return results
@@ -463,6 +463,6 @@ class TestOperatorsHooks:
)
)
- invalid_files = [f for f in files if any(f.endswith(suffix) for suffix
in illegal_suffixes)]
+ invalid_files = [f for f in files if
f.endswith(tuple(illegal_suffixes))]
assert [] == invalid_files
diff --git a/tests/cli/commands/test_config_command.py
b/tests/cli/commands/test_config_command.py
index 59d04de860..c1e2924bfc 100644
--- a/tests/cli/commands/test_config_command.py
+++ b/tests/cli/commands/test_config_command.py
@@ -194,9 +194,7 @@ class TestCliConfigList:
)
output = temp_stdout.getvalue()
lines = output.split("\n")
- assert all(
- line.startswith("#") or line.strip() == "" or line.startswith("[")
for line in lines if line
- )
+ assert all(not line.strip() or line.startswith(("#", "[")) for line in
lines if line)
class TestCliConfigGetValue:
diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py
index cd69759995..f1c84721b4 100644
--- a/tests/decorators/test_python.py
+++ b/tests/decorators/test_python.py
@@ -462,7 +462,7 @@ class TestAirflowTaskDecorator(BasePythonTest):
bigger_number.operator.run(start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE)
ret.operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
- ti_add_num = [ti for ti in dr.get_task_instances() if ti.task_id ==
"add_num"][0]
+ ti_add_num = next(ti for ti in dr.get_task_instances() if ti.task_id
== "add_num")
assert ti_add_num.xcom_pull(key=ret.key) == (test_number + 2) * 2
def test_dag_task(self):
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index ed09d9cd18..cffea246e4 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -3123,7 +3123,7 @@ class TestSchedulerJob:
}
for root, _, files in os.walk(TEST_DAG_FOLDER):
for file_name in files:
- if file_name.endswith(".py") or file_name.endswith(".zip"):
+ if file_name.endswith((".py", ".zip")):
if file_name not in ignored_files:
expected_files.add(f"{root}/{file_name}")
for file_path in list_py_file_paths(TEST_DAG_FOLDER,
include_examples=False):
@@ -3136,7 +3136,7 @@ class TestSchedulerJob:
example_dag_folder = airflow.example_dags.__path__[0]
for root, _, files in os.walk(example_dag_folder):
for file_name in files:
- if file_name.endswith(".py") or file_name.endswith(".zip"):
+ if file_name.endswith((".py", ".zip")):
if file_name not in ["__init__.py"] and file_name not in
ignored_files:
expected_files.add(os.path.join(root, file_name))
detected_files.clear()
diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py
index c90ec1aef3..5d809333c8 100644
--- a/tests/models/test_dagrun.py
+++ b/tests/models/test_dagrun.py
@@ -1706,7 +1706,7 @@ def
test_calls_to_verify_integrity_with_mapped_task_zero_length_at_runtime(dag_m
(1, State.NONE),
(2, State.NONE),
]
- ti1 = [i for i in tis if i.map_index == 0][0]
+ ti1 = next(i for i in tis if i.map_index == 0)
# Now "clear" and "reduce" the length to empty list
dag.clear()
Variable.set(key="arg1", value=[])
diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py
index 0dcdac81b5..0e9846c502 100644
--- a/tests/operators/test_python.py
+++ b/tests/operators/test_python.py
@@ -1396,7 +1396,7 @@ class TestShortCircuitWithTeardown:
op1.skip = MagicMock()
dagrun = dag_maker.create_dagrun()
tis = dagrun.get_task_instances()
- ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0]
+ ti: TaskInstance = next(x for x in tis if x.task_id == "op1")
ti._run_raw_task()
expected_tasks = {dag.task_dict[x] for x in expected}
if should_skip:
@@ -1427,7 +1427,7 @@ class TestShortCircuitWithTeardown:
op1.skip = MagicMock()
dagrun = dag_maker.create_dagrun()
tis = dagrun.get_task_instances()
- ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0]
+ ti: TaskInstance = next(x for x in tis if x.task_id == "op1")
ti._run_raw_task()
# we can't use assert_called_with because it's a set and therefore
not ordered
actual_skipped = set(op1.skip.call_args.kwargs["tasks"])
@@ -1454,7 +1454,7 @@ class TestShortCircuitWithTeardown:
op1.skip = MagicMock()
dagrun = dag_maker.create_dagrun()
tis = dagrun.get_task_instances()
- ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0]
+ ti: TaskInstance = next(x for x in tis if x.task_id == "op1")
ti._run_raw_task()
# we can't use assert_called_with because it's a set and therefore
not ordered
actual_kwargs = op1.skip.call_args.kwargs
@@ -1489,7 +1489,7 @@ class TestShortCircuitWithTeardown:
op1.skip = MagicMock()
dagrun = dag_maker.create_dagrun()
tis = dagrun.get_task_instances()
- ti: TaskInstance = [x for x in tis if x.task_id == "op1"][0]
+ ti: TaskInstance = next(x for x in tis if x.task_id == "op1")
ti._run_raw_task()
# we can't use assert_called_with because it's a set and therefore
not ordered
actual_kwargs = op1.skip.call_args.kwargs
diff --git a/tests/serialization/test_dag_serialization.py
b/tests/serialization/test_dag_serialization.py
index 301e54a0ec..26474a8929 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -454,7 +454,7 @@ class TestStringifiedDAGs:
items should not matter but assertEqual would fail if the order of
items changes in the dag dictionary
"""
- dag_dict["dag"]["tasks"] = sorted(dag_dict["dag"]["tasks"],
key=lambda x: sorted(x.keys()))
+ dag_dict["dag"]["tasks"] = sorted(dag_dict["dag"]["tasks"],
key=sorted)
dag_dict["dag"]["_access_control"]["__var"]["test_role"]["__var"]
= sorted(
dag_dict["dag"]["_access_control"]["__var"]["test_role"]["__var"]
)
diff --git a/tests/system/providers/amazon/aws/example_ec2.py
b/tests/system/providers/amazon/aws/example_ec2.py
index 1dd98488c2..dfd69c74bc 100644
--- a/tests/system/providers/amazon/aws/example_ec2.py
+++ b/tests/system/providers/amazon/aws/example_ec2.py
@@ -57,8 +57,7 @@ def get_latest_ami_id():
Owners=["amazon"],
)
# Sort on CreationDate
- sorted_images = sorted(images["Images"], key=itemgetter("CreationDate"),
reverse=True)
- return sorted_images[0]["ImageId"]
+ return max(images["Images"], key=itemgetter("CreationDate"))["ImageId"]
@task
diff --git a/tests/system/providers/amazon/aws/utils/ec2.py
b/tests/system/providers/amazon/aws/utils/ec2.py
index c90418538f..c2d1411374 100644
--- a/tests/system/providers/amazon/aws/utils/ec2.py
+++ b/tests/system/providers/amazon/aws/utils/ec2.py
@@ -40,7 +40,7 @@ def _get_next_available_cidr(vpc_id: str) -> str:
if len({block.prefixlen for block in existing_cidr_blocks}) > 1:
raise ValueError(error_msg_template.format("Subnets do not all use the
same CIDR block size."))
- last_used_block = sorted(existing_cidr_blocks)[-1]
+ last_used_block = max(existing_cidr_blocks)
*_, last_reserved_ip = last_used_block
return f"{last_reserved_ip + 1}/{last_used_block.prefixlen}"
diff --git a/tests/test_utils/providers.py b/tests/test_utils/providers.py
index 7d6770b81a..99f64a6849 100644
--- a/tests/test_utils/providers.py
+++ b/tests/test_utils/providers.py
@@ -53,6 +53,6 @@ def get_provider_min_airflow_version(provider_name):
p = ProvidersManager()
deps = p.providers[provider_name].data["dependencies"]
- airflow_dep = [x for x in deps if x.startswith("apache-airflow")][0]
+ airflow_dep = next(x for x in deps if x.startswith("apache-airflow"))
min_airflow_version = tuple(map(int,
airflow_dep.split(">=")[1].split(".")))
return min_airflow_version
diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py
index 1dd1665a9b..46fa055e8c 100644
--- a/tests/www/test_utils.py
+++ b/tests/www/test_utils.py
@@ -133,7 +133,7 @@ class TestUtils:
def test_params_none_and_zero(self):
query_str = utils.get_params(a=0, b=None, c="true")
# The order won't be consistent, but that doesn't affect behaviour of
a browser
- pairs = list(sorted(query_str.split("&")))
+ pairs = sorted(query_str.split("&"))
assert ["a=0", "c=true"] == pairs
def test_params_all(self):
@@ -429,11 +429,11 @@ def
test_dag_run_custom_sqla_interface_delete_no_collateral_damage(dag_maker, se
assert len(set(x.run_id for x in dag_runs)) == 3
run_id_for_single_delete = "scheduled__2023-01-01T00:00:00+00:00"
# we have 3 runs with this same run_id
- assert len(list(x for x in dag_runs if x.run_id ==
run_id_for_single_delete)) == 3
+ assert sum(1 for x in dag_runs if x.run_id == run_id_for_single_delete) ==
3
# each is a different dag
# if we delete one, it shouldn't delete the others
- one_run = [x for x in dag_runs if x.run_id == run_id_for_single_delete][0]
+ one_run = next(x for x in dag_runs if x.run_id == run_id_for_single_delete)
assert interface.delete(item=one_run) is True
session.commit()
dag_runs = session.query(DagRun).all()
diff --git a/tests/www/views/test_views_acl.py
b/tests/www/views/test_views_acl.py
index 30e2a975c4..ac2821188c 100644
--- a/tests/www/views/test_views_acl.py
+++ b/tests/www/views/test_views_acl.py
@@ -336,7 +336,7 @@ def client_all_dags_dagruns(acl_app, user_all_dags_dagruns):
def test_dag_stats_success(client_all_dags_dagruns):
resp = client_all_dags_dagruns.post("dag_stats", follow_redirects=True)
check_content_in_response("example_bash_operator", resp)
- assert set(list(resp.json.items())[0][1][0].keys()) == {"state", "count"}
+ assert set(next(iter(resp.json.items()))[1][0].keys()) == {"state",
"count"}
def test_task_stats_failure(dag_test_client):
diff --git a/tests/www/views/test_views_home.py
b/tests/www/views/test_views_home.py
index 8525149899..874596b682 100644
--- a/tests/www/views/test_views_home.py
+++ b/tests/www/views/test_views_home.py
@@ -154,7 +154,7 @@ def working_dags(tmpdir):
dag_contents_template = "from airflow import DAG\ndag = DAG('{}',
tags=['{}'])"
with create_session() as session:
- for dag_id, tag in list(zip(TEST_FILTER_DAG_IDS, TEST_TAGS)):
+ for dag_id, tag in zip(TEST_FILTER_DAG_IDS, TEST_TAGS):
filename = os.path.join(tmpdir, f"{dag_id}.py")
with open(filename, "w") as f:
f.writelines(dag_contents_template.format(dag_id, tag))
@@ -169,7 +169,7 @@ def working_dags_with_read_perm(tmpdir):
"access_control={{'role_single_dag':{{'can_read'}}}}) "
)
with create_session() as session:
- for dag_id, tag in list(zip(TEST_FILTER_DAG_IDS, TEST_TAGS)):
+ for dag_id, tag in zip(TEST_FILTER_DAG_IDS, TEST_TAGS):
filename = os.path.join(tmpdir, f"{dag_id}.py")
if dag_id == "filter_test_1":
with open(filename, "w") as f:
@@ -188,7 +188,7 @@ def working_dags_with_edit_perm(tmpdir):
"access_control={{'role_single_dag':{{'can_edit'}}}}) "
)
with create_session() as session:
- for dag_id, tag in list(zip(TEST_FILTER_DAG_IDS, TEST_TAGS)):
+ for dag_id, tag in zip(TEST_FILTER_DAG_IDS, TEST_TAGS):
filename = os.path.join(tmpdir, f"{dag_id}.py")
if dag_id == "filter_test_1":
with open(filename, "w") as f: