Taragolis commented on code in PR #39468:
URL: https://github.com/apache/airflow/pull/39468#discussion_r1593525214
##########
tests/always/test_example_dags.py:
##########
@@ -19,25 +19,86 @@
import os
import sys
from glob import glob
+from importlib import metadata as importlib_metadata
from pathlib import Path
import pytest
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
from airflow.models import DagBag
from airflow.utils import yaml
from tests.test_utils.asserts import assert_queries_count
AIRFLOW_SOURCES_ROOT = Path(__file__).resolve().parents[2]
AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers"
+CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
+NO_DB_QUERY_EXCEPTION = ("/airflow/example_dags/example_subdag_operator.py",)
+PROVIDERS_PREFIXES = ("airflow/providers/", "tests/system/providers/")
+OPTIONAL_PROVIDERS_DEPENDENCIES: dict[str, dict[str, str | None]] = {
+ # Some certain of examples/system tests might require additional
dependencies,
+ # which are not installed into specific CI check
+ # Format of dictionary:
+ # key: prefix of the file which need to be excluded,
+ # values: dictionary with package distributions and optional specifier,
e.g. >=2.3.4
+}
+IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = (
+ # Some certain of examples/system tests might raise
AirflowProviderDeprecationWarning.
+ # In general, it should be resolved as soon as parameter/operator
deprecated,
+ # however we might postpone change for a while, in this case we should add
it into this tuple
+ # and create the appropriate task in GitHub
+ "tests/system/providers/amazon/aws/example_ecs_fargate.py",
+ "tests/system/providers/amazon/aws/example_eks_with_nodegroups.py",
+ "tests/system/providers/amazon/aws/example_emr.py",
+ "tests/system/providers/amazon/aws/example_emr_notebook_execution.py",
+ "tests/system/providers/dbt/cloud/example_dbt_cloud.py",
+ "tests/system/providers/docker/example_docker_swarm.py",
+
"tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py",
+
"tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py",
+ "tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py",
+ "tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py",
+ "tests/system/providers/google/cloud/gcs/example_gcs_sensor.py",
+ "tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py",
+
"tests/system/providers/google/cloud/life_sciences/example_life_sciences.py",
+ "tests/system/providers/google/marketing_platform/example_analytics.py",
+ "tests/system/providers/weaviate/example_weaviate_cohere.py",
+ "tests/system/providers/weaviate/example_weaviate_openai.py",
+ "tests/system/providers/weaviate/example_weaviate_operator.py",
Review Comment:
We should fix this ones, I will create follow up task once we merge this PR
##########
tests/always/test_example_dags.py:
##########
@@ -19,25 +19,86 @@
import os
import sys
from glob import glob
+from importlib import metadata as importlib_metadata
from pathlib import Path
import pytest
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
from airflow.models import DagBag
from airflow.utils import yaml
from tests.test_utils.asserts import assert_queries_count
AIRFLOW_SOURCES_ROOT = Path(__file__).resolve().parents[2]
AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers"
+CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
+NO_DB_QUERY_EXCEPTION = ("/airflow/example_dags/example_subdag_operator.py",)
+PROVIDERS_PREFIXES = ("airflow/providers/", "tests/system/providers/")
+OPTIONAL_PROVIDERS_DEPENDENCIES: dict[str, dict[str, str | None]] = {
+ # Some certain of examples/system tests might require additional
dependencies,
+ # which are not installed into specific CI check
+ # Format of dictionary:
+ # key: prefix of the file which need to be excluded,
+ # values: dictionary with package distributions and optional specifier,
e.g. >=2.3.4
+}
+IGNORE_AIRFLOW_PROVIDER_DEPRECATION_WARNING: tuple[str, ...] = (
+ # Some certain of examples/system tests might raise
AirflowProviderDeprecationWarning.
+ # In general, it should be resolved as soon as parameter/operator
deprecated,
+ # however we might postpone change for a while, in this case we should add
it into this tuple
+ # and create the appropriate task in GitHub
+ "tests/system/providers/amazon/aws/example_ecs_fargate.py",
+ "tests/system/providers/amazon/aws/example_eks_with_nodegroups.py",
+ "tests/system/providers/amazon/aws/example_emr.py",
+ "tests/system/providers/amazon/aws/example_emr_notebook_execution.py",
+ "tests/system/providers/dbt/cloud/example_dbt_cloud.py",
+ "tests/system/providers/docker/example_docker_swarm.py",
+
"tests/system/providers/google/cloud/azure/example_azure_fileshare_to_gcs.py",
+
"tests/system/providers/google/cloud/bigquery/example_bigquery_operations.py",
+ "tests/system/providers/google/cloud/bigquery/example_bigquery_sensors.py",
+ "tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py",
+ "tests/system/providers/google/cloud/gcs/example_gcs_sensor.py",
+ "tests/system/providers/google/cloud/gcs/example_gcs_to_gcs.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_async.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_job.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_kueue.py",
+
"tests/system/providers/google/cloud/kubernetes_engine/example_kubernetes_engine_resource.py",
+
"tests/system/providers/google/cloud/life_sciences/example_life_sciences.py",
+ "tests/system/providers/google/marketing_platform/example_analytics.py",
+ "tests/system/providers/weaviate/example_weaviate_cohere.py",
+ "tests/system/providers/weaviate/example_weaviate_openai.py",
+ "tests/system/providers/weaviate/example_weaviate_operator.py",
+ # Deprecated Operators/Hooks, which replaced by common.sql Operators/Hooks
+ "tests/system/providers/apache/drill/example_drill_dag.py",
+ "tests/system/providers/jdbc/example_jdbc_queries.py",
+ "tests/system/providers/microsoft/mssql/example_mssql.py",
+ "tests/system/providers/mysql/example_mysql.py",
+ "tests/system/providers/postgres/example_postgres.py",
+ "tests/system/providers/snowflake/example_snowflake.py",
+ "tests/system/providers/sqlite/example_sqlite.py",
+ "tests/system/providers/trino/example_trino.py",
Review Comment:
I'm not sure what we should do with this examples, all of them have a common
replacement, e.g. `SQLExecuteQueryOperator` instead of `PostgresOperator`,
`JdbcOperator`, `SnowflakeOperator` and so on
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
