This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6d8a58bff74 Exempt Google Cloud link modules from provider test-file
check (#74924)
6d8a58bff74 is described below
commit 6d8a58bff74671eb47c9e32ba386e258e01dc351
Author: Shahar Epstein <[email protected]>
AuthorDate: Tue Oct 20 22:57:23 2026 +0300
Exempt Google Cloud link modules from provider test-file check (#74924)
Link modules only define link classes, so their tests would only assert
constants (name, key, format_str). Move them to a dedicated EXEMPTED_TESTS
list in test_providers_modules_should_have_tests and delete the
constant-only test files. Stale and added-test checks apply to the new list
as they do to OVERLOOKED_TESTS.
---
.../tests/unit/always/test_project_structure.py | 55 +++++++++------
.../tests/unit/google/cloud/links/test_alloy_db.py | 62 -----------------
.../unit/google/cloud/links/test_managed_kafka.py | 79 ----------------------
3 files changed, 36 insertions(+), 160 deletions(-)
diff --git a/airflow-core/tests/unit/always/test_project_structure.py
b/airflow-core/tests/unit/always/test_project_structure.py
index 514e3b16cde..a9d12c05600 100644
--- a/airflow-core/tests/unit/always/test_project_structure.py
+++ b/airflow-core/tests/unit/always/test_project_structure.py
@@ -58,8 +58,8 @@ class TestProjectStructure:
"""
Assert every module in /providers/ has a corresponding test_ file in
providers/providers.
"""
- # The test below had a but for quite a while and we missed a lot of
modules to have tess
- # We should make sure that one goes to 0
+ # A long-standing bug in the test below allowed modules without tests
to go undetected.
+ # This list tracks those modules and should shrink to zero as tests
are added.
OVERLOOKED_TESTS = [
"providers/amazon/tests/unit/amazon/aws/auth_manager/datamodels/test_login.py",
"providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor_config.py",
@@ -90,24 +90,8 @@ class TestProjectStructure:
"providers/fab/tests/unit/fab/www/test_security_appless.py",
"providers/fab/tests/unit/fab/www/test_security_manager.py",
"providers/fab/tests/unit/fab/www/test_session.py",
-
"providers/google/tests/unit/google/cloud/links/test_bigquery_dts.py",
- "providers/google/tests/unit/google/cloud/links/test_bigtable.py",
-
"providers/google/tests/unit/google/cloud/links/test_cloud_build.py",
-
"providers/google/tests/unit/google/cloud/links/test_cloud_functions.py",
-
"providers/google/tests/unit/google/cloud/links/test_cloud_memorystore.py",
- "providers/google/tests/unit/google/cloud/links/test_cloud_sql.py",
- "providers/google/tests/unit/google/cloud/links/test_compute.py",
-
"providers/google/tests/unit/google/cloud/links/test_data_loss_prevention.py",
- "providers/google/tests/unit/google/cloud/links/test_dataflow.py",
- "providers/google/tests/unit/google/cloud/links/test_dataform.py",
-
"providers/google/tests/unit/google/cloud/links/test_datafusion.py",
- "providers/google/tests/unit/google/cloud/links/test_dataprep.py",
"providers/google/tests/unit/google/cloud/links/test_dataproc.py",
- "providers/google/tests/unit/google/cloud/links/test_datastore.py",
"providers/google/tests/unit/google/cloud/links/test_kubernetes_engine.py",
- "providers/google/tests/unit/google/cloud/links/test_pubsub.py",
- "providers/google/tests/unit/google/cloud/links/test_spanner.py",
- "providers/google/tests/unit/google/cloud/links/test_workflows.py",
"providers/google/tests/unit/google/cloud/links/test_translate.py",
"providers/google/tests/unit/google/cloud/operators/vertex_ai/test_auto_ml.py",
"providers/google/tests/unit/google/cloud/operators/vertex_ai/test_batch_prediction_job.py",
@@ -125,6 +109,28 @@ class TestProjectStructure:
"providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_adls.py",
"providers/snowflake/tests/unit/snowflake/triggers/test_snowflake_trigger.py",
]
+ # Deliberately untested: link modules whose tests would only assert
constants.
+ # Remove an entry once its module gains logic worth testing.
+ EXEMPTED_TESTS = [
+ "providers/google/tests/unit/google/cloud/links/test_alloy_db.py",
+
"providers/google/tests/unit/google/cloud/links/test_bigquery_dts.py",
+ "providers/google/tests/unit/google/cloud/links/test_bigtable.py",
+
"providers/google/tests/unit/google/cloud/links/test_cloud_build.py",
+
"providers/google/tests/unit/google/cloud/links/test_cloud_functions.py",
+
"providers/google/tests/unit/google/cloud/links/test_cloud_memorystore.py",
+ "providers/google/tests/unit/google/cloud/links/test_cloud_sql.py",
+ "providers/google/tests/unit/google/cloud/links/test_compute.py",
+
"providers/google/tests/unit/google/cloud/links/test_data_loss_prevention.py",
+ "providers/google/tests/unit/google/cloud/links/test_dataflow.py",
+ "providers/google/tests/unit/google/cloud/links/test_dataform.py",
+
"providers/google/tests/unit/google/cloud/links/test_datafusion.py",
+ "providers/google/tests/unit/google/cloud/links/test_dataprep.py",
+ "providers/google/tests/unit/google/cloud/links/test_datastore.py",
+
"providers/google/tests/unit/google/cloud/links/test_managed_kafka.py",
+ "providers/google/tests/unit/google/cloud/links/test_pubsub.py",
+ "providers/google/tests/unit/google/cloud/links/test_spanner.py",
+ "providers/google/tests/unit/google/cloud/links/test_workflows.py",
+ ]
modules_files: list[pathlib.Path] = list(
AIRFLOW_PROVIDERS_ROOT_PATH.glob("**/src/airflow/providers/**/*.py")
)
@@ -170,12 +176,18 @@ class TestProjectStructure:
]
)
overlooked_test_files = {pathlib.Path(test_file) for test_file in
OVERLOOKED_TESTS}
+ exempted_test_files = {pathlib.Path(test_file) for test_file in
EXEMPTED_TESTS}
stale_overlooked_tests = overlooked_test_files - expected_test_files
assert not stale_overlooked_tests, (
"Detected stale provider test exemptions without corresponding
provider modules - "
f"please remove them from OVERLOOKED_TESTS:
{sorted(stale_overlooked_tests)}"
)
- expected_test_files -= overlooked_test_files
+ stale_exempted_tests = exempted_test_files - expected_test_files
+ assert not stale_exempted_tests, (
+ "Detected stale provider test exemptions without corresponding
provider modules - "
+ f"please remove them from EXEMPTED_TESTS:
{sorted(stale_exempted_tests)}"
+ )
+ expected_test_files -= overlooked_test_files | exempted_test_files
missing_tests_files = [
file.as_posix()
@@ -189,6 +201,11 @@ class TestProjectStructure:
"Detect added tests in providers module - please remove the tests "
"from OVERLOOKED_TESTS list above"
)
+ added_exempted_test_files =
current_test_files.intersection(exempted_test_files)
+ assert set() == added_exempted_test_files, (
+ "Detect added tests in providers module - please remove the tests "
+ f"from EXEMPTED_TESTS list above:
{sorted(added_exempted_test_files)}"
+ )
def filepath_to_module(path: pathlib.Path, src_folder: pathlib.Path):
diff --git a/providers/google/tests/unit/google/cloud/links/test_alloy_db.py
b/providers/google/tests/unit/google/cloud/links/test_alloy_db.py
deleted file mode 100644
index 41c7f22d86d..00000000000
--- a/providers/google/tests/unit/google/cloud/links/test_alloy_db.py
+++ /dev/null
@@ -1,62 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from airflow.providers.google.cloud.links.alloy_db import (
- AlloyDBBackupsLink,
- AlloyDBClusterLink,
- AlloyDBUsersLink,
-)
-
-TEST_LOCATION = "test-location"
-TEST_CLUSTER_ID = "test-cluster-id"
-TEST_PROJECT_ID = "test-project-id"
-EXPECTED_ALLOY_DB_CLUSTER_LINK_NAME = "AlloyDB Cluster"
-EXPECTED_ALLOY_DB_CLUSTER_LINK_KEY = "alloy_db_cluster"
-EXPECTED_ALLOY_DB_CLUSTER_LINK_FORMAT_STR = (
-
"/alloydb/locations/{location_id}/clusters/{cluster_id}?project={project_id}"
-)
-EXPECTED_ALLOY_DB_USERS_LINK_NAME = "AlloyDB Users"
-EXPECTED_ALLOY_DB_USERS_LINK_KEY = "alloy_db_users"
-EXPECTED_ALLOY_DB_USERS_LINK_FORMAT_STR = (
-
"/alloydb/locations/{location_id}/clusters/{cluster_id}/users?project={project_id}"
-)
-EXPECTED_ALLOY_DB_BACKUP_LINK_NAME = "AlloyDB Backups"
-EXPECTED_ALLOY_DB_BACKUP_LINK_KEY = "alloy_db_backups"
-EXPECTED_ALLOY_DB_BACKUP_LINK_FORMAT_STR =
"/alloydb/backups?project={project_id}"
-
-
-class TestAlloyDBClusterLink:
- def test_class_attributes(self):
- assert AlloyDBClusterLink.key == EXPECTED_ALLOY_DB_CLUSTER_LINK_KEY
- assert AlloyDBClusterLink.name == EXPECTED_ALLOY_DB_CLUSTER_LINK_NAME
- assert AlloyDBClusterLink.format_str ==
EXPECTED_ALLOY_DB_CLUSTER_LINK_FORMAT_STR
-
-
-class TestAlloyDBUsersLink:
- def test_class_attributes(self):
- assert AlloyDBUsersLink.key == EXPECTED_ALLOY_DB_USERS_LINK_KEY
- assert AlloyDBUsersLink.name == EXPECTED_ALLOY_DB_USERS_LINK_NAME
- assert AlloyDBUsersLink.format_str ==
EXPECTED_ALLOY_DB_USERS_LINK_FORMAT_STR
-
-
-class TestAlloyDBBackupsLink:
- def test_class_attributes(self):
- assert AlloyDBBackupsLink.key == EXPECTED_ALLOY_DB_BACKUP_LINK_KEY
- assert AlloyDBBackupsLink.name == EXPECTED_ALLOY_DB_BACKUP_LINK_NAME
- assert AlloyDBBackupsLink.format_str ==
EXPECTED_ALLOY_DB_BACKUP_LINK_FORMAT_STR
diff --git
a/providers/google/tests/unit/google/cloud/links/test_managed_kafka.py
b/providers/google/tests/unit/google/cloud/links/test_managed_kafka.py
deleted file mode 100644
index 307be52adfa..00000000000
--- a/providers/google/tests/unit/google/cloud/links/test_managed_kafka.py
+++ /dev/null
@@ -1,79 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from airflow.providers.google.cloud.links.managed_kafka import (
- ApacheKafkaClusterLink,
- ApacheKafkaClusterListLink,
- ApacheKafkaConsumerGroupLink,
- ApacheKafkaTopicLink,
-)
-
-TEST_LOCATION = "test-location"
-TEST_CLUSTER_ID = "test-cluster-id"
-TEST_PROJECT_ID = "test-project-id"
-TEST_TOPIC_ID = "test-topic-id"
-TEST_CONSUMER_GROUP_ID = "test-consumer-group-id"
-EXPECTED_MANAGED_KAFKA_CLUSTER_LINK_NAME = "Apache Kafka Cluster"
-EXPECTED_MANAGED_KAFKA_CLUSTER_LINK_KEY = "cluster_conf"
-EXPECTED_MANAGED_KAFKA_CLUSTER_LINK_FORMAT_STR = (
- "/managedkafka/{location}/clusters/{cluster_id}?project={project_id}"
-)
-EXPECTED_MANAGED_KAFKA_CLUSTER_LIST_LINK_NAME = "Apache Kafka Cluster List"
-EXPECTED_MANAGED_KAFKA_CLUSTER_LIST_LINK_KEY = "cluster_list_conf"
-EXPECTED_MANAGED_KAFKA_CLUSTER_LIST_LINK_FORMAT_STR =
"/managedkafka/clusters?project={project_id}"
-EXPECTED_MANAGED_KAFKA_TOPIC_LINK_NAME = "Apache Kafka Topic"
-EXPECTED_MANAGED_KAFKA_TOPIC_LINK_KEY = "topic_conf"
-EXPECTED_MANAGED_KAFKA_TOPIC_LINK_FORMAT_STR = (
-
"/managedkafka/{location}/clusters/{cluster_id}/topics/{topic_id}?project={project_id}"
-)
-EXPECTED_MANAGED_KAFKA_CONSUMER_GROUP_LINK_NAME = "Apache Kafka Consumer Group"
-EXPECTED_MANAGED_KAFKA_CONSUMER_GROUP_LINK_KEY = "consumer_group_conf"
-EXPECTED_MANAGED_KAFKA_CONSUMER_GROUP_LINK_FORMAT_STR = (
-
"/managedkafka/{location}/clusters/{cluster_id}/consumer_groups/{consumer_group_id}?project={project_id}"
-)
-
-
-class TestApacheKafkaClusterLink:
- def test_class_attributes(self):
- assert ApacheKafkaClusterLink.key ==
EXPECTED_MANAGED_KAFKA_CLUSTER_LINK_KEY
- assert ApacheKafkaClusterLink.name ==
EXPECTED_MANAGED_KAFKA_CLUSTER_LINK_NAME
- assert ApacheKafkaClusterLink.format_str ==
EXPECTED_MANAGED_KAFKA_CLUSTER_LINK_FORMAT_STR
-
-
-class TestApacheKafkaClusterListLink:
- def test_class_attributes(self):
- assert ApacheKafkaClusterListLink.key ==
EXPECTED_MANAGED_KAFKA_CLUSTER_LIST_LINK_KEY
- assert ApacheKafkaClusterListLink.name ==
EXPECTED_MANAGED_KAFKA_CLUSTER_LIST_LINK_NAME
- assert ApacheKafkaClusterListLink.format_str ==
EXPECTED_MANAGED_KAFKA_CLUSTER_LIST_LINK_FORMAT_STR
-
-
-class TestApacheKafkaTopicLink:
- def test_class_attributes(self):
- assert ApacheKafkaTopicLink.key ==
EXPECTED_MANAGED_KAFKA_TOPIC_LINK_KEY
- assert ApacheKafkaTopicLink.name ==
EXPECTED_MANAGED_KAFKA_TOPIC_LINK_NAME
- assert ApacheKafkaTopicLink.format_str ==
EXPECTED_MANAGED_KAFKA_TOPIC_LINK_FORMAT_STR
-
-
-class TestApacheKafkaConsumerGroupLink:
- def test_class_attributes(self):
- assert ApacheKafkaConsumerGroupLink.key ==
EXPECTED_MANAGED_KAFKA_CONSUMER_GROUP_LINK_KEY
- assert ApacheKafkaConsumerGroupLink.name ==
EXPECTED_MANAGED_KAFKA_CONSUMER_GROUP_LINK_NAME
- assert (
- ApacheKafkaConsumerGroupLink.format_str ==
EXPECTED_MANAGED_KAFKA_CONSUMER_GROUP_LINK_FORMAT_STR
- )
