This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
from ca3f70c063 Add Changelog for 2.5.1
add f786659683 Rewrite kerberos security integration and unit tests
(#28092)
add 350b4a1174 Replace `unittests` in providers tests by pure `pytest`
[Wave-3] (#27696)
add 0b6bdcc234 Replace `unittests` in providers-apache tests by pure
`pytest` (#27948)
add 2873800cbb Move Integration tests to separate package under tests
(#28170)
add 350316f32a Merge redis and rabbitmq integration into celery (#28205)
add 25932757b2 Integration tests are separated into separate command and
CI job (#28207)
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci.yml | 108 +++-
.pre-commit-config.yaml | 1 +
BREEZE.rst | 30 +-
Dockerfile.ci | 15 +-
TESTING.rst | 437 ++++++++--------
.../airflow_breeze/commands/testing_commands.py | 108 ++--
.../commands/testing_commands_config.py | 25 +-
dev/breeze/src/airflow_breeze/global_constants.py | 11 +-
.../src/airflow_breeze/params/shell_params.py | 13 +-
.../airflow_breeze/utils/docker_command_utils.py | 2 -
.../src/airflow_breeze/utils/selective_checks.py | 6 -
dev/breeze/tests/test_selective_checks.py | 26 +-
images/breeze/output-commands-hash.txt | 15 +-
images/breeze/output-commands.svg | 78 ++-
.../output_setup_regenerate-command-images.svg | 28 +-
images/breeze/output_shell.svg | 240 +++++----
images/breeze/output_start-airflow.svg | 264 +++++-----
images/breeze/output_testing.svg | 16 +-
images/breeze/output_testing_integration-tests.svg | 184 +++++++
images/breeze/output_testing_tests.svg | 34 +-
scripts/ci/docker-compose/_docker.env | 2 -
scripts/ci/docker-compose/base.yml | 2 -
scripts/ci/docker-compose/devcontainer.env | 2 -
...ntegration-redis.yml => integration-celery.yml} | 17 +-
scripts/ci/docker-compose/integration-openldap.yml | 46 --
scripts/ci/docker-compose/integration-rabbitmq.yml | 40 --
scripts/ci/openldap/ldif/01-users.example.com.ldif | 48 --
.../ci/openldap/ldif/02-groups.example.com.ldif | 44 --
.../ci/openldap/ldif/03-manager.example.com.ldif | 21 -
scripts/ci/openldap/ldif/04-rootdn.ldif | 24 -
scripts/ci/openldap/slapd.conf | 64 ---
scripts/docker/entrypoint_ci.sh | 15 +-
scripts/in_container/check_environment.sh | 72 ++-
tests/cli/commands/test_celery_command.py | 31 --
tests/conftest.py | 2 +-
tests/executors/test_celery_executor.py | 259 +---------
.../integration}/__init__.py | 0
.../integration/api}/__init__.py | 0
.../integration/api/auth}/__init__.py | 0
.../integration/api/auth/backend}/__init__.py | 0
.../api/auth/backend/test_kerberos_auth.py | 0
.../integration/cli}/__init__.py | 0
.../integration/cli/commands}/__init__.py | 0
.../cli/commands/test_celery_command.py | 56 ++
.../integration/executors}/__init__.py | 0
.../executors/test_celery_executor.py | 311 +----------
.../integration/providers}/__init__.py | 0
.../integration/providers/apache}/__init__.py | 0
.../providers/apache/cassandra}/__init__.py | 0
.../providers/apache/cassandra/hooks}/__init__.py | 0
.../apache/cassandra/hooks/test_cassandra.py | 9 +-
.../providers/apache/pinot}/__init__.py | 0
.../providers/apache/pinot/hooks}/__init__.py | 0
.../providers/apache/pinot/hooks/test_pinot.py} | 25 +-
.../integration/providers/google}/__init__.py | 0
.../providers/google/cloud}/__init__.py | 0
.../providers/google/cloud/transfers}/__init__.py | 0
.../google/cloud/transfers/test_presto_to_gcs.py | 0
.../cloud/transfers/test_presto_to_gcs_system.py | 0
.../google/cloud/transfers/test_trino_to_gcs.py | 0
.../integration/providers/mongo}/__init__.py | 0
.../providers/mongo/sensors}/__init__.py | 0
.../providers/mongo/sensors/test_mongo.py | 8 +-
.../integration/providers/redis}/__init__.py | 0
.../integration/providers/redis/hooks}/__init__.py | 0
.../providers/redis/hooks/test_redis.py | 31 +-
.../providers/redis/operators}/__init__.py | 0
.../redis/operators/test_redis_publish.py | 7 +-
.../providers/redis/sensors}/__init__.py | 0
.../providers/redis/sensors/test_redis_key.py | 8 +-
.../providers/redis/sensors/test_redis_pub_sub.py | 49 +-
.../integration/providers/trino}/__init__.py | 0
.../integration/providers/trino/hooks}/__init__.py | 0
.../providers/trino/hooks/test_trino.py | 48 ++
.../integration/security}/__init__.py | 0
tests/integration/security/test_kerberos.py | 75 +++
tests/providers/airbyte/hooks/test_airbyte.py | 35 +-
tests/providers/airbyte/operators/test_airbyte.py | 3 +-
tests/providers/airbyte/sensors/test_airbyte.py | 3 +-
tests/providers/alibaba/cloud/hooks/test_oss.py | 5 +-
.../alibaba/cloud/log/test_oss_task_handler.py | 5 +-
.../providers/alibaba/cloud/operators/test_oss.py | 13 +-
.../alibaba/cloud/sensors/test_oss_key.py | 5 +-
tests/providers/apache/beam/hooks/test_beam.py | 60 +--
tests/providers/apache/beam/operators/test_beam.py | 49 +-
tests/providers/apache/cassandra/hooks/__init__.py | 16 -
.../apache/cassandra/sensors/test_record.py | 3 +-
.../apache/cassandra/sensors/test_table.py | 3 +-
tests/providers/apache/drill/hooks/test_drill.py | 5 +-
.../providers/apache/drill/operators/test_drill.py | 8 +-
tests/providers/apache/druid/hooks/test_druid.py | 51 +-
.../apache/druid/transfers/test_hive_to_druid.py | 9 +-
tests/providers/apache/hdfs/hooks/test_hdfs.py | 8 +-
tests/providers/apache/hdfs/hooks/test_webhdfs.py | 85 ++-
tests/providers/apache/hdfs/sensors/test_hdfs.py | 56 +-
.../providers/apache/hdfs/sensors/test_web_hdfs.py | 3 +-
tests/providers/apache/hive/__init__.py | 5 +-
tests/providers/apache/hive/hooks/test_hive.py | 56 +-
tests/providers/apache/hive/operators/test_hive.py | 7 +-
.../apache/hive/operators/test_hive_stats.py | 9 +-
tests/providers/apache/hive/sensors/test_hdfs.py | 7 +-
.../apache/hive/sensors/test_hive_partition.py | 7 +-
.../hive/sensors/test_metastore_partition.py | 7 +-
.../hive/sensors/test_named_hive_partition.py | 9 +-
.../apache/hive/transfers/test_hive_to_mysql.py | 11 +-
.../apache/hive/transfers/test_hive_to_samba.py | 11 +-
.../apache/hive/transfers/test_mssql_to_hive.py | 5 +-
.../apache/hive/transfers/test_s3_to_hive.py | 29 +-
.../apache/hive/transfers/test_vertica_to_hive.py | 5 +-
tests/providers/apache/kylin/hooks/test_kylin.py | 5 +-
.../apache/kylin/operators/test_kylin_cube.py | 5 +-
tests/providers/apache/livy/hooks/test_livy.py | 574 +++++++++------------
tests/providers/apache/livy/operators/test_livy.py | 18 +-
tests/providers/apache/livy/sensors/test_livy.py | 29 +-
tests/providers/apache/pig/hooks/test_pig.py | 7 +-
tests/providers/apache/pig/operators/test_pig.py | 3 +-
tests/providers/apache/pinot/hooks/test_pinot.py | 21 +-
.../apache/spark/hooks/test_spark_jdbc.py | 6 +-
.../providers/apache/spark/hooks/test_spark_sql.py | 7 +-
.../apache/spark/hooks/test_spark_submit.py | 13 +-
.../apache/spark/operators/test_spark_jdbc.py | 6 +-
.../apache/spark/operators/test_spark_sql.py | 5 +-
.../apache/spark/operators/test_spark_submit.py | 5 +-
tests/providers/apache/sqoop/hooks/test_sqoop.py | 5 +-
.../providers/apache/sqoop/operators/test_sqoop.py | 5 +-
tests/providers/arangodb/hooks/test_arangodb.py | 6 +-
.../providers/arangodb/operators/test_arangodb.py | 3 +-
tests/providers/arangodb/sensors/test_arangodb.py | 5 +-
.../providers/asana/operators/test_asana_tasks.py | 5 +-
tests/providers/atlassian/jira/hooks/test_jira.py | 5 +-
.../atlassian/jira/operators/test_jira.py | 5 +-
.../providers/atlassian/jira/sensors/test_jira.py | 5 +-
.../providers/celery/sensors/test_celery_queue.py | 5 +-
tests/providers/cloudant/hooks/test_cloudant.py | 5 +-
.../kubernetes/operators/test_spark_kubernetes.py | 5 +-
.../kubernetes/sensors/test_spark_kubernetes.py | 5 +-
tests/providers/common/sql/hooks/test_dbapi.py | 7 +-
tests/providers/common/sql/operators/test_sql.py | 29 +-
tests/providers/common/sql/sensors/test_sql.py | 11 +-
.../providers/databricks/hooks/test_databricks.py | 37 +-
.../databricks/hooks/test_databricks_sql.py | 5 +-
.../databricks/operators/test_databricks.py | 25 +-
.../databricks/operators/test_databricks_repos.py | 7 +-
.../databricks/operators/test_databricks_sql.py | 5 +-
tests/providers/databricks/utils/databricks.py | 6 +-
tests/providers/datadog/hooks/test_datadog.py | 29 +-
tests/providers/datadog/sensors/test_datadog.py | 5 +-
tests/providers/dingding/hooks/test_dingding.py | 5 +-
.../providers/dingding/operators/test_dingding.py | 5 +-
.../discord/hooks/test_discord_webhook.py | 5 +-
.../discord/operators/test_discord_webhook.py | 6 +-
tests/providers/docker/hooks/test_docker.py | 11 +-
tests/providers/docker/operators/test_docker.py | 35 +-
.../docker/operators/test_docker_swarm.py | 8 +-
.../elasticsearch/hooks/test_elasticsearch.py | 20 +-
.../elasticsearch/log/test_es_task_handler.py | 4 +-
tests/providers/exasol/hooks/test_exasol.py | 13 +-
tests/providers/exasol/operators/test_exasol.py | 3 +-
tests/providers/ftp/hooks/test_ftp.py | 14 +-
tests/providers/ftp/sensors/test_ftp.py | 3 +-
tests/providers/grpc/hooks/test_grpc.py | 2 +-
tests/providers/grpc/operators/test_grpc.py | 3 +-
.../_internal_client/test_vault_client.py | 3 +-
tests/providers/hashicorp/hooks/test_vault.py | 25 +-
tests/providers/hashicorp/secrets/test_vault.py | 4 +-
tests/providers/http/hooks/test_http.py | 93 ++--
tests/providers/http/operators/test_http.py | 19 +-
tests/providers/http/sensors/test_http.py | 5 +-
tests/providers/imap/hooks/test_imap.py | 5 +-
.../providers/imap/sensors/test_imap_attachment.py | 11 +-
tests/providers/influxdb/hooks/test_influxdb.py | 6 +-
.../providers/influxdb/operators/test_influxdb.py | 3 +-
tests/providers/jdbc/hooks/test_jdbc.py | 2 +-
tests/providers/jdbc/operators/test_jdbc.py | 5 +-
tests/providers/jenkins/hooks/test_jenkins.py | 9 +-
.../jenkins/operators/test_jenkins_job_trigger.py | 77 +--
tests/providers/jenkins/sensors/test_jenkins.py | 12 +-
tests/providers/mongo/hooks/test_mongo.py | 44 +-
tests/providers/mysql/hooks/test_mysql.py | 56 +-
tests/providers/mysql/operators/test_mysql.py | 29 +-
.../mysql/transfers/test_presto_to_mysql.py | 19 +-
.../providers/mysql/transfers/test_s3_to_mysql.py | 7 +-
.../mysql/transfers/test_trino_to_mysql.py | 19 +-
.../mysql/transfers/test_vertica_to_mysql.py | 5 +-
tests/providers/neo4j/hooks/test_neo4j.py | 28 +-
tests/providers/neo4j/operators/test_neo4j.py | 3 +-
tests/providers/openfaas/hooks/test_openfaas.py | 62 +--
tests/providers/opsgenie/hooks/test_opsgenie.py | 5 +-
.../providers/opsgenie/operators/test_opsgenie.py | 13 +-
tests/providers/oracle/hooks/test_oracle.py | 21 +-
tests/providers/oracle/operators/test_oracle.py | 22 +-
.../oracle/transfers/test_oracle_to_oracle.py | 6 +-
.../papermill/operators/test_papermill.py | 3 +-
tests/providers/postgres/hooks/test_postgres.py | 28 +-
.../providers/postgres/operators/test_postgres.py | 8 +-
tests/providers/presto/hooks/test_presto.py | 15 +-
.../providers/presto/transfers/test_gcs_presto.py | 3 +-
tests/providers/qubole/hooks/test_qubole.py | 4 +-
tests/providers/qubole/hooks/test_qubole_check.py | 4 +-
.../qubole/operators/test_qubole_check.py | 7 +-
tests/providers/qubole/sensors/test_qubole.py | 5 +-
tests/providers/redis/hooks/test_redis.py | 21 +-
.../providers/redis/sensors/test_redis_pub_sub.py | 51 +-
.../providers/salesforce/hooks/test_salesforce.py | 3 +-
.../operators/test_salesforce_apex_rest.py | 3 +-
tests/providers/salesforce/sensors/__init__.py | 16 -
tests/providers/samba/hooks/test_samba.py | 14 +-
tests/providers/segment/hooks/test_segment.py | 7 +-
.../segment/operators/test_segment_track_event.py | 8 +-
tests/providers/sendgrid/utils/test_emailer.py | 5 +-
tests/providers/sftp/hooks/test_sftp.py | 46 +-
tests/providers/sftp/sensors/test_sftp.py | 3 +-
.../singularity/operators/test_singularity.py | 25 +-
tests/providers/slack/operators/test_slack.py | 6 +-
tests/providers/snowflake/hooks/test_snowflake.py | 63 +--
.../snowflake/operators/test_snowflake.py | 6 +-
.../transfers/test_copy_into_snowflake.py | 3 +-
tests/providers/sqlite/hooks/test_sqlite.py | 9 +-
tests/providers/sqlite/operators/test_sqlite.py | 8 +-
tests/providers/ssh/hooks/test_ssh.py | 15 +-
tests/providers/tableau/hooks/test_tableau.py | 23 +-
tests/providers/tableau/operators/test_tableau.py | 9 +-
tests/providers/tableau/sensors/test_tableau.py | 16 +-
tests/providers/telegram/hooks/test_telegram.py | 5 +-
.../providers/telegram/operators/test_telegram.py | 5 +-
tests/providers/trino/hooks/test_trino.py | 40 +-
tests/providers/trino/operators/test_trino.py | 18 +-
tests/providers/trino/transfers/test_gcs_trino.py | 3 +-
tests/providers/vertica/hooks/test_vertica.py | 13 +-
tests/providers/vertica/operators/test_vertica.py | 3 +-
.../yandex/hooks/test_yandexcloud_dataproc.py | 9 +-
.../yandex/operators/test_yandexcloud_dataproc.py | 5 +-
tests/security/test_kerberos.py | 186 ++-----
tests/test_utils/asserts.py | 20 +-
234 files changed, 2470 insertions(+), 3513 deletions(-)
create mode 100644 images/breeze/output_testing_integration-tests.svg
rename scripts/ci/docker-compose/{integration-redis.yml =>
integration-celery.yml} (75%)
delete mode 100644 scripts/ci/docker-compose/integration-openldap.yml
delete mode 100644 scripts/ci/docker-compose/integration-rabbitmq.yml
delete mode 100644 scripts/ci/openldap/ldif/01-users.example.com.ldif
delete mode 100644 scripts/ci/openldap/ldif/02-groups.example.com.ldif
delete mode 100644 scripts/ci/openldap/ldif/03-manager.example.com.ldif
delete mode 100644 scripts/ci/openldap/ldif/04-rootdn.ldif
delete mode 100644 scripts/ci/openldap/slapd.conf
copy {airflow/api_connexion => tests/integration}/__init__.py (100%)
copy {airflow/api_connexion => tests/integration/api}/__init__.py (100%)
copy {airflow/api_connexion => tests/integration/api/auth}/__init__.py (100%)
copy {airflow/api_connexion => tests/integration/api/auth/backend}/__init__.py
(100%)
rename tests/{ => integration}/api/auth/backend/test_kerberos_auth.py (100%)
copy {airflow/api_connexion => tests/integration/cli}/__init__.py (100%)
copy {airflow/api_connexion => tests/integration/cli/commands}/__init__.py
(100%)
create mode 100644 tests/integration/cli/commands/test_celery_command.py
copy {airflow/api_connexion => tests/integration/executors}/__init__.py (100%)
copy tests/{ => integration}/executors/test_celery_executor.py (52%)
copy {airflow/api_connexion => tests/integration/providers}/__init__.py (100%)
copy {airflow/api_connexion => tests/integration/providers/apache}/__init__.py
(100%)
copy {airflow/api_connexion =>
tests/integration/providers/apache/cassandra}/__init__.py (100%)
copy {airflow/api_connexion =>
tests/integration/providers/apache/cassandra/hooks}/__init__.py (100%)
rename tests/{ =>
integration}/providers/apache/cassandra/hooks/test_cassandra.py (98%)
copy {airflow/api_connexion =>
tests/integration/providers/apache/pinot}/__init__.py (100%)
copy {airflow/api_connexion =>
tests/integration/providers/apache/pinot/hooks}/__init__.py (100%)
copy tests/{providers/microsoft/azure/operators/test_adls_delete.py =>
integration/providers/apache/pinot/hooks/test_pinot.py} (60%)
copy {airflow/api_connexion => tests/integration/providers/google}/__init__.py
(100%)
copy {airflow/api_connexion =>
tests/integration/providers/google/cloud}/__init__.py (100%)
copy {airflow/api_connexion =>
tests/integration/providers/google/cloud/transfers}/__init__.py (100%)
rename tests/{ =>
integration}/providers/google/cloud/transfers/test_presto_to_gcs.py (100%)
rename tests/{ =>
integration}/providers/google/cloud/transfers/test_presto_to_gcs_system.py
(100%)
rename tests/{ =>
integration}/providers/google/cloud/transfers/test_trino_to_gcs.py (100%)
copy {airflow/api_connexion => tests/integration/providers/mongo}/__init__.py
(100%)
copy {airflow/api_connexion =>
tests/integration/providers/mongo/sensors}/__init__.py (100%)
rename tests/{ => integration}/providers/mongo/sensors/test_mongo.py (94%)
copy {airflow/api_connexion => tests/integration/providers/redis}/__init__.py
(100%)
copy {airflow/api_connexion =>
tests/integration/providers/redis/hooks}/__init__.py (100%)
copy airflow/providers/redis/sensors/redis_key.py =>
tests/integration/providers/redis/hooks/test_redis.py (55%)
copy {airflow/api_connexion =>
tests/integration/providers/redis/operators}/__init__.py (100%)
rename tests/{ => integration}/providers/redis/operators/test_redis_publish.py
(94%)
copy {airflow/api_connexion =>
tests/integration/providers/redis/sensors}/__init__.py (100%)
rename tests/{ => integration}/providers/redis/sensors/test_redis_key.py (93%)
copy tests/{ => integration}/providers/redis/sensors/test_redis_pub_sub.py
(63%)
copy {airflow/api_connexion => tests/integration/providers/trino}/__init__.py
(100%)
copy {airflow/api_connexion =>
tests/integration/providers/trino/hooks}/__init__.py (100%)
create mode 100644 tests/integration/providers/trino/hooks/test_trino.py
copy {airflow/api_connexion => tests/integration/security}/__init__.py (100%)
create mode 100644 tests/integration/security/test_kerberos.py
delete mode 100644 tests/providers/apache/cassandra/hooks/__init__.py
delete mode 100644 tests/providers/salesforce/sensors/__init__.py
