This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 74a1931c3b Split Celery integration tests to "celery" and "redis"
(#39994)
74a1931c3b is described below
commit 74a1931c3b4131857e3949171738386e7c7d91f3
Author: Shahar Epstein <[email protected]>
AuthorDate: Mon Jun 10 18:56:06 2024 +0300
Split Celery integration tests to "celery" and "redis" (#39994)
---
contributing-docs/testing/integration_tests.rst | 2 ++
dev/breeze/doc/images/output-commands.svg | 2 +-
dev/breeze/doc/images/output_shell.svg | 2 +-
dev/breeze/doc/images/output_shell.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 2 +-
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
.../images/output_testing_integration-tests.svg | 4 ++--
.../images/output_testing_integration-tests.txt | 2 +-
dev/breeze/doc/images/output_testing_tests.svg | 4 ++--
dev/breeze/doc/images/output_testing_tests.txt | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 18 ++++++++++--------
.../src/airflow_breeze/params/shell_params.py | 5 +++--
scripts/ci/docker-compose/integration-celery.yml | 17 ++++++-----------
...ntegration-celery.yml => integration-redis.yml} | 22 +++-------------------
scripts/in_container/check_environment.sh | 3 +++
.../integration/executors/test_celery_executor.py | 5 ++---
.../providers/redis/hooks/test_redis.py | 2 +-
.../redis/operators/test_redis_publish.py | 2 +-
.../providers/redis/sensors/test_redis_key.py | 2 +-
.../providers/redis/sensors/test_redis_pub_sub.py | 2 +-
20 files changed, 44 insertions(+), 58 deletions(-)
diff --git a/contributing-docs/testing/integration_tests.rst
b/contributing-docs/testing/integration_tests.rst
index 80cf169d90..823b4a509c 100644
--- a/contributing-docs/testing/integration_tests.rst
+++ b/contributing-docs/testing/integration_tests.rst
@@ -74,6 +74,8 @@ The following integrations are available:
+--------------+----------------------------------------------------+
| qdrant | Integration required for Qdrant tests. |
+--------------+----------------------------------------------------+
+| redis | Integration required for Redis tests. |
++--------------+----------------------------------------------------+
| statsd | Integration required for Statsd hooks. |
+--------------+----------------------------------------------------+
| trino | Integration required for Trino hooks. |
diff --git a/dev/breeze/doc/images/output-commands.svg
b/dev/breeze/doc/images/output-commands.svg
index df92c06ae3..1381742085 100644
--- a/dev/breeze/doc/images/output-commands.svg
+++ b/dev/breeze/doc/images/output-commands.svg
@@ -300,7 +300,7 @@
</text><text class="breeze-help-r5" x="0" y="166.4" textLength="12.2"
clip-path="url(#breeze-help-line-6)">│</text><text class="breeze-help-r5"
x="414.8" y="166.4" textLength="732"
clip-path="url(#breeze-help-line-6)">[default: 3.8]                                          
[...]
</text><text class="breeze-help-r5" x="0" y="190.8" textLength="12.2"
clip-path="url(#breeze-help-line-7)">│</text><text class="breeze-help-r4"
x="24.4" y="190.8" textLength="158.6"
clip-path="url(#breeze-help-line-7)">--integration</text><text
class="breeze-help-r1" x="414.8" y="190.8" textLength="1024.8"
clip-path="url(#breeze-help-line-7)">Integration(s) to enable when running (can be more than one).       
[...]
</text><text class="breeze-help-r5" x="0" y="215.2" textLength="12.2"
clip-path="url(#breeze-help-line-8)">│</text><text class="breeze-help-r7"
x="414.8" y="215.2" textLength="1024.8"
clip-path="url(#breeze-help-line-8)">(all | all-testable | cassandra | celery | drill | kafka | kerberos | mongo | mssql </text><text
class="breeze-help-r5" x="1451.8" y="215.2" textLength="12.2"
clip-path="url(#breeze-help [...]
-</text><text class="breeze-help-r5" x="0" y="239.6" textLength="12.2"
clip-path="url(#breeze-help-line-9)">│</text><text class="breeze-help-r7"
x="414.8" y="239.6" textLength="1024.8"
clip-path="url(#breeze-help-line-9)">| openlineage | otel | pinot | qdrant | statsd | trino)                           
[...]
+</text><text class="breeze-help-r5" x="0" y="239.6" textLength="12.2"
clip-path="url(#breeze-help-line-9)">│</text><text class="breeze-help-r7"
x="414.8" y="239.6" textLength="1024.8"
clip-path="url(#breeze-help-line-9)">| openlineage | otel | pinot | qdrant | redis | statsd | trino)                     </text><text
clas [...]
</text><text class="breeze-help-r5" x="0" y="264" textLength="12.2"
clip-path="url(#breeze-help-line-10)">│</text><text class="breeze-help-r4"
x="24.4" y="264" textLength="317.2"
clip-path="url(#breeze-help-line-10)">--standalone-dag-processor</text><text
class="breeze-help-r1" x="414.8" y="264" textLength="573.4"
clip-path="url(#breeze-help-line-10)">Run standalone dag processor for start-airflow.</text><text
class="breeze-help-r5" x="1451.8" y="264" textLength= [...]
</text><text class="breeze-help-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-help-line-11)">│</text><text class="breeze-help-r4"
x="24.4" y="288.4" textLength="244"
clip-path="url(#breeze-help-line-11)">--database-isolation</text><text
class="breeze-help-r1" x="414.8" y="288.4" textLength="475.8"
clip-path="url(#breeze-help-line-11)">Run airflow in database isolation mode.</text><text
class="breeze-help-r5" x="1451.8" y="288.4" textLength="12.2" c [...]
</text><text class="breeze-help-r5" x="0" y="312.8" textLength="1464"
clip-path="url(#breeze-help-line-12)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-help-r1" x="1464" y="312.8" textLength="12.2"
clip-path="url(#breeze-help-line-12)">
diff --git a/dev/breeze/doc/images/output_shell.svg
b/dev/breeze/doc/images/output_shell.svg
index c397fe3e38..f1449fefe5 100644
--- a/dev/breeze/doc/images/output_shell.svg
+++ b/dev/breeze/doc/images/output_shell.svg
@@ -521,7 +521,7 @@
</text><text class="breeze-shell-r5" x="0" y="215.2" textLength="12.2"
clip-path="url(#breeze-shell-line-8)">│</text><text class="breeze-shell-r5"
x="414.8" y="215.2" textLength="732"
clip-path="url(#breeze-shell-line-8)">[default: 3.8]                                          
[...]
</text><text class="breeze-shell-r5" x="0" y="239.6" textLength="12.2"
clip-path="url(#breeze-shell-line-9)">│</text><text class="breeze-shell-r4"
x="24.4" y="239.6" textLength="158.6"
clip-path="url(#breeze-shell-line-9)">--integration</text><text
class="breeze-shell-r1" x="414.8" y="239.6" textLength="1024.8"
clip-path="url(#breeze-shell-line-9)">Integration(s) to enable when running (can be more than one).      
[...]
</text><text class="breeze-shell-r5" x="0" y="264" textLength="12.2"
clip-path="url(#breeze-shell-line-10)">│</text><text class="breeze-shell-r7"
x="414.8" y="264" textLength="1024.8"
clip-path="url(#breeze-shell-line-10)">(all | all-testable | cassandra | celery | drill | kafka | kerberos | mongo | mssql </text><text
class="breeze-shell-r5" x="1451.8" y="264" textLength="12.2"
clip-path="url(#breeze-she [...]
-</text><text class="breeze-shell-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-shell-line-11)">│</text><text class="breeze-shell-r7"
x="414.8" y="288.4" textLength="1024.8"
clip-path="url(#breeze-shell-line-11)">| openlineage | otel | pinot | qdrant | statsd | trino)                          
[...]
+</text><text class="breeze-shell-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-shell-line-11)">│</text><text class="breeze-shell-r7"
x="414.8" y="288.4" textLength="1024.8"
clip-path="url(#breeze-shell-line-11)">| openlineage | otel | pinot | qdrant | redis | statsd | trino)                     </text><tex
[...]
</text><text class="breeze-shell-r5" x="0" y="312.8" textLength="12.2"
clip-path="url(#breeze-shell-line-12)">│</text><text class="breeze-shell-r4"
x="24.4" y="312.8" textLength="317.2"
clip-path="url(#breeze-shell-line-12)">--standalone-dag-processor</text><text
class="breeze-shell-r1" x="414.8" y="312.8" textLength="573.4"
clip-path="url(#breeze-shell-line-12)">Run standalone dag processor for start-airflow.</text><text
class="breeze-shell-r5" x="1451.8" y="312 [...]
</text><text class="breeze-shell-r5" x="0" y="337.2" textLength="12.2"
clip-path="url(#breeze-shell-line-13)">│</text><text class="breeze-shell-r4"
x="24.4" y="337.2" textLength="244"
clip-path="url(#breeze-shell-line-13)">--database-isolation</text><text
class="breeze-shell-r1" x="414.8" y="337.2" textLength="475.8"
clip-path="url(#breeze-shell-line-13)">Run airflow in database isolation mode.</text><text
class="breeze-shell-r5" x="1451.8" y="337.2" textLength=" [...]
</text><text class="breeze-shell-r5" x="0" y="361.6" textLength="1464"
clip-path="url(#breeze-shell-line-14)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-shell-r1" x="1464" y="361.6" textLength="12.2"
clip-path="url(#breeze-shell-line-14)">
diff --git a/dev/breeze/doc/images/output_shell.txt
b/dev/breeze/doc/images/output_shell.txt
index 8c0c62dc49..ea3e916b67 100644
--- a/dev/breeze/doc/images/output_shell.txt
+++ b/dev/breeze/doc/images/output_shell.txt
@@ -1 +1 @@
-24a11835dbadb243b380175681db6827
+84c9572359eef66076ea6af1355fdda7
diff --git a/dev/breeze/doc/images/output_start-airflow.svg
b/dev/breeze/doc/images/output_start-airflow.svg
index 46142fa6cc..9ff84f31fb 100644
--- a/dev/breeze/doc/images/output_start-airflow.svg
+++ b/dev/breeze/doc/images/output_start-airflow.svg
@@ -400,7 +400,7 @@
</text><text class="breeze-start-airflow-r5" x="0" y="264" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-10)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="264" textLength="122"
clip-path="url(#breeze-start-airflow-line-10)">--platform</text><text
class="breeze-start-airflow-r1" x="414.8" y="264" textLength="329.4"
clip-path="url(#breeze-start-airflow-line-10)">Platform for Airflow image.</text><text
class="breeze-start-airflow-r7" x="756.4" y="26 [...]
</text><text class="breeze-start-airflow-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-11)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="288.4" textLength="158.6"
clip-path="url(#breeze-start-airflow-line-11)">--integration</text><text
class="breeze-start-airflow-r1" x="414.8" y="288.4" textLength="1024.8"
clip-path="url(#breeze-start-airflow-line-11)">Integration(s) to enable when running (can be more 
[...]
</text><text class="breeze-start-airflow-r5" x="0" y="312.8" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-12)">│</text><text
class="breeze-start-airflow-r7" x="414.8" y="312.8" textLength="1024.8"
clip-path="url(#breeze-start-airflow-line-12)">(all | all-testable | cassandra | celery | drill | kafka | kerberos | mongo | mssql </text><text
class="breeze-start-airflow-r5" x="1451.8" y="312.8 [...]
-</text><text class="breeze-start-airflow-r5" x="0" y="337.2" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-13)">│</text><text
class="breeze-start-airflow-r7" x="414.8" y="337.2" textLength="1024.8"
clip-path="url(#breeze-start-airflow-line-13)">| openlineage | otel | pinot | qdrant | statsd | trino)                    
[...]
+</text><text class="breeze-start-airflow-r5" x="0" y="337.2" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-13)">│</text><text
class="breeze-start-airflow-r7" x="414.8" y="337.2" textLength="1024.8"
clip-path="url(#breeze-start-airflow-line-13)">| openlineage | otel | pinot | qdrant | redis | statsd | trino)                 
[...]
</text><text class="breeze-start-airflow-r5" x="0" y="361.6" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-14)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="361.6" textLength="317.2"
clip-path="url(#breeze-start-airflow-line-14)">--standalone-dag-processor</text><text
class="breeze-start-airflow-r1" x="414.8" y="361.6" textLength="573.4"
clip-path="url(#breeze-start-airflow-line-14)">Run standalone dag processor for start-airflow.</text
[...]
</text><text class="breeze-start-airflow-r5" x="0" y="386" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-15)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="386" textLength="244"
clip-path="url(#breeze-start-airflow-line-15)">--database-isolation</text><text
class="breeze-start-airflow-r1" x="414.8" y="386" textLength="475.8"
clip-path="url(#breeze-start-airflow-line-15)">Run airflow in database isolation mode.</text><text
class="breeze-s [...]
</text><text class="breeze-start-airflow-r5" x="0" y="410.4" textLength="12.2"
clip-path="url(#breeze-start-airflow-line-16)">│</text><text
class="breeze-start-airflow-r4" x="24.4" y="410.4" textLength="231.8"
clip-path="url(#breeze-start-airflow-line-16)">--load-example-dags</text><text
class="breeze-start-airflow-r6" x="366" y="410.4" textLength="24.4"
clip-path="url(#breeze-start-airflow-line-16)">-e</text><text
class="breeze-start-airflow-r1" x="414.8" y="410.4" textLength="780.8" cl [...]
diff --git a/dev/breeze/doc/images/output_start-airflow.txt
b/dev/breeze/doc/images/output_start-airflow.txt
index 91963233a2..7f9086008d 100644
--- a/dev/breeze/doc/images/output_start-airflow.txt
+++ b/dev/breeze/doc/images/output_start-airflow.txt
@@ -1 +1 @@
-df6894b42ff874ea9b414d61d4a7c8a0
+ae3a2480bdd0cce8f42234fb0b7804e6
diff --git a/dev/breeze/doc/images/output_testing_integration-tests.svg
b/dev/breeze/doc/images/output_testing_integration-tests.svg
index d94317705c..6801d5d67f 100644
--- a/dev/breeze/doc/images/output_testing_integration-tests.svg
+++ b/dev/breeze/doc/images/output_testing_integration-tests.svg
@@ -212,8 +212,8 @@
</text><text class="breeze-testing-integration-tests-r5" x="0" y="361.6"
textLength="24.4"
clip-path="url(#breeze-testing-integration-tests-line-14)">╭─</text><text
class="breeze-testing-integration-tests-r5" x="24.4" y="361.6"
textLength="219.6"
clip-path="url(#breeze-testing-integration-tests-line-14)"> Test environment </text><text
class="breeze-testing-integration-tests-r5" x="244" y="361.6"
textLength="1195.6" clip-path="url(#breeze-testing-integration-tests-line-14)"
[...]
</text><text class="breeze-testing-integration-tests-r5" x="0" y="386"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-15)">│</text><text
class="breeze-testing-integration-tests-r4" x="24.4" y="386" textLength="158.6"
clip-path="url(#breeze-testing-integration-tests-line-15)">--integration</text><text
class="breeze-testing-integration-tests-r1" x="610" y="386" textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-15)">Integration(s) to 
[...]
</text><text class="breeze-testing-integration-tests-r5" x="0" y="410.4"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-16)">│</text><text
class="breeze-testing-integration-tests-r6" x="610" y="410.4"
textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-16)">(all | all-testable | cassandra | celery | drill | kafka | kerberos </text><text
class="breeze-testing-integration-tests-r5 [...]
-</text><text class="breeze-testing-integration-tests-r5" x="0" y="434.8"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-17)">│</text><text
class="breeze-testing-integration-tests-r6" x="610" y="434.8"
textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-17)">| mongo | mssql | openlineage | otel | pinot | qdrant | statsd |    </text><text
class="breeze-tes [...]
-</text><text class="breeze-testing-integration-tests-r5" x="0" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-18)">│</text><text
class="breeze-testing-integration-tests-r6" x="610" y="459.2"
textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-18)">trino)                              &#
[...]
+</text><text class="breeze-testing-integration-tests-r5" x="0" y="434.8"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-17)">│</text><text
class="breeze-testing-integration-tests-r6" x="610" y="434.8"
textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-17)">| mongo | mssql | openlineage | otel | pinot | qdrant | redis |     </text><text
class="breez [...]
+</text><text class="breeze-testing-integration-tests-r5" x="0" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-18)">│</text><text
class="breeze-testing-integration-tests-r6" x="610" y="459.2"
textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-18)">statsd | trino)                           &
[...]
</text><text class="breeze-testing-integration-tests-r5" x="0" y="483.6"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-19)">│</text><text
class="breeze-testing-integration-tests-r4" x="24.4" y="483.6"
textLength="109.8"
clip-path="url(#breeze-testing-integration-tests-line-19)">--backend</text><text
class="breeze-testing-integration-tests-r7" x="561.2" y="483.6"
textLength="24.4"
clip-path="url(#breeze-testing-integration-tests-line-19)">-b</text><text
class="br [...]
</text><text class="breeze-testing-integration-tests-r5" x="0" y="508"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-20)">│</text><text
class="breeze-testing-integration-tests-r1" x="610" y="508" textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-20)">invalid DB configuration and no database and any attempts to connect</text><text
class="breeze-testing-integration-tests-r5" x="1451.8" y="508 [...]
</text><text class="breeze-testing-integration-tests-r5" x="0" y="532.4"
textLength="12.2"
clip-path="url(#breeze-testing-integration-tests-line-21)">│</text><text
class="breeze-testing-integration-tests-r1" x="610" y="532.4"
textLength="829.6"
clip-path="url(#breeze-testing-integration-tests-line-21)">to Airflow DB will fail.                        
[...]
diff --git a/dev/breeze/doc/images/output_testing_integration-tests.txt
b/dev/breeze/doc/images/output_testing_integration-tests.txt
index 1ce1ec9dcb..4e797be366 100644
--- a/dev/breeze/doc/images/output_testing_integration-tests.txt
+++ b/dev/breeze/doc/images/output_testing_integration-tests.txt
@@ -1 +1 @@
-1fcc3bf55f3f8368264a998f68ed3c8f
+29357a3838bf8ff90520c3bfbf818bbe
diff --git a/dev/breeze/doc/images/output_testing_tests.svg
b/dev/breeze/doc/images/output_testing_tests.svg
index f7d17bff54..7876ef08a1 100644
--- a/dev/breeze/doc/images/output_testing_tests.svg
+++ b/dev/breeze/doc/images/output_testing_tests.svg
@@ -467,8 +467,8 @@
</text><text class="breeze-testing-tests-r5" x="0" y="1020.4"
textLength="24.4" clip-path="url(#breeze-testing-tests-line-41)">╭─</text><text
class="breeze-testing-tests-r5" x="24.4" y="1020.4" textLength="219.6"
clip-path="url(#breeze-testing-tests-line-41)"> Test environment </text><text
class="breeze-testing-tests-r5" x="244" y="1020.4" textLength="1195.6"
clip-path="url(#breeze-testing-tests-line-41)">────────────────────────────────────────────────────────────────────
[...]
</text><text class="breeze-testing-tests-r5" x="0" y="1044.8"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-42)">│</text><text
class="breeze-testing-tests-r4" x="24.4" y="1044.8" textLength="158.6"
clip-path="url(#breeze-testing-tests-line-42)">--integration</text><text
class="breeze-testing-tests-r1" x="610" y="1044.8" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-42)">Integration(s) to enable when running (can be more 
[...]
</text><text class="breeze-testing-tests-r5" x="0" y="1069.2"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-43)">│</text><text
class="breeze-testing-tests-r7" x="610" y="1069.2" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-43)">(all | all-testable | cassandra | celery | drill | kafka | kerberos </text><text
class="breeze-testing-tests-r5" x="1451.8" y="1069.2" textLength="12.2"
clip-path="url(# [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1093.6"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-44)">│</text><text
class="breeze-testing-tests-r7" x="610" y="1093.6" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-44)">| mongo | mssql | openlineage | otel | pinot | qdrant | statsd |    </text><text
class="breeze-testing-tests-r5" x="1451.8" y="1093.6" textLengt [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-testing-tests-line-45)">│</text><text
class="breeze-testing-tests-r7" x="610" y="1118" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-45)">trino)                                      
[...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1093.6"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-44)">│</text><text
class="breeze-testing-tests-r7" x="610" y="1093.6" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-44)">| mongo | mssql | openlineage | otel | pinot | qdrant | redis |     </text><text
class="breeze-testing-tests-r5" x="1451.8" y="1093.6" text [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-testing-tests-line-45)">│</text><text
class="breeze-testing-tests-r7" x="610" y="1118" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-45)">statsd | trino)                                   
[...]
</text><text class="breeze-testing-tests-r5" x="0" y="1142.4"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-46)">│</text><text
class="breeze-testing-tests-r4" x="24.4" y="1142.4" textLength="109.8"
clip-path="url(#breeze-testing-tests-line-46)">--backend</text><text
class="breeze-testing-tests-r6" x="561.2" y="1142.4" textLength="24.4"
clip-path="url(#breeze-testing-tests-line-46)">-b</text><text
class="breeze-testing-tests-r1" x="610" y="1142.4" textLength="829.6" clip-pat
[...]
</text><text class="breeze-testing-tests-r5" x="0" y="1166.8"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-47)">│</text><text
class="breeze-testing-tests-r1" x="610" y="1166.8" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-47)">invalid DB configuration and no database and any attempts to connect</text><text
class="breeze-testing-tests-r5" x="1451.8" y="1166.8" textLength="12.2"
clip-path="url(#breeze-testing- [...]
</text><text class="breeze-testing-tests-r5" x="0" y="1191.2"
textLength="12.2" clip-path="url(#breeze-testing-tests-line-48)">│</text><text
class="breeze-testing-tests-r1" x="610" y="1191.2" textLength="829.6"
clip-path="url(#breeze-testing-tests-line-48)">to Airflow DB will fail.                               
[...]
diff --git a/dev/breeze/doc/images/output_testing_tests.txt
b/dev/breeze/doc/images/output_testing_tests.txt
index e5e54e6eff..731a581153 100644
--- a/dev/breeze/doc/images/output_testing_tests.txt
+++ b/dev/breeze/doc/images/output_testing_tests.txt
@@ -1 +1 @@
-db5149f207af5cb6903b8213597539d9
+45c2dd92b2f83548aa9256e529d9da7e
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 6f80ada99a..b672f8afa3 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -59,8 +59,9 @@ TESTABLE_INTEGRATIONS = [
"mongo",
"mssql",
"pinot",
- "trino",
"qdrant",
+ "redis",
+ "trino",
]
OTHER_INTEGRATIONS = ["statsd", "otel", "openlineage"]
ALLOWED_DEBIAN_VERSIONS = ["bookworm", "bullseye"]
@@ -232,16 +233,17 @@ def get_default_platform_machine() -> str:
DOCKER_DEFAULT_PLATFORM = f"linux/{get_default_platform_machine()}"
DOCKER_BUILDKIT = 1
-SSH_PORT = "12322"
-WEBSERVER_HOST_PORT = "28080"
-POSTGRES_HOST_PORT = "25433"
-MYSQL_HOST_PORT = "23306"
+DRILL_HOST_PORT = "28047"
FLOWER_HOST_PORT = "25555"
-REDIS_HOST_PORT = "26379"
-CELERY_BROKER_URLS_MAP = {"rabbitmq": "amqp://guest:guest@rabbitmq:5672",
"redis": "redis://redis:6379/0"}
MSSQL_HOST_PORT = "21433"
-DRILL_HOST_PORT = "28047"
+MYSQL_HOST_PORT = "23306"
+POSTGRES_HOST_PORT = "25433"
+RABBITMQ_HOST_PORT = "25672"
+REDIS_HOST_PORT = "26379"
+SSH_PORT = "12322"
+WEBSERVER_HOST_PORT = "28080"
+CELERY_BROKER_URLS_MAP = {"rabbitmq": "amqp://guest:guest@rabbitmq:5672",
"redis": "redis://redis:6379/0"}
SQLITE_URL = "sqlite:////root/airflow/sqlite/airflow.db"
PYTHONDONTWRITEBYTECODE = True
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py
b/dev/breeze/src/airflow_breeze/params/shell_params.py
index f67aa99131..ec18ccda76 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -489,6 +489,7 @@ class ShellParams:
_set_var(_env, "BASE_BRANCH", self.base_branch, "main")
_set_var(_env, "BREEZE", "true")
_set_var(_env, "BREEZE_INIT_COMMAND", None, "")
+ _set_var(_env, "CELERY_BROKER_URLS_MAP", CELERY_BROKER_URLS_MAP)
_set_var(_env, "CELERY_FLOWER", self.celery_flower)
_set_var(_env, "CHICKEN_EGG_PROVIDERS", self.chicken_egg_providers)
_set_var(_env, "CI", None, "false")
@@ -508,6 +509,7 @@ class ShellParams:
_set_var(_env, "DOCKER_IS_ROOTLESS", self.rootless_docker)
_set_var(_env, "DOWNGRADE_SQLALCHEMY", self.downgrade_sqlalchemy)
_set_var(_env, "DOWNGRADE_PENDULUM", self.downgrade_pendulum)
+ _set_var(_env, "DRILL_HOST_PORT", None, DRILL_HOST_PORT)
_set_var(_env, "ENABLED_SYSTEMS", None, "")
_set_var(_env, "ENABLE_COVERAGE", self.enable_coverage)
_set_var(_env, "FLOWER_HOST_PORT", None, FLOWER_HOST_PORT)
@@ -525,6 +527,7 @@ class ShellParams:
_set_var(_env, "ISSUE_ID", self.issue_id)
_set_var(_env, "LOAD_DEFAULT_CONNECTIONS",
self.load_default_connections)
_set_var(_env, "LOAD_EXAMPLES", self.load_example_dags)
+ _set_var(_env, "MSSQL_HOST_PORT", None, MSSQL_HOST_PORT)
_set_var(_env, "MYSQL_HOST_PORT", None, MYSQL_HOST_PORT)
_set_var(_env, "MYSQL_VERSION", self.mysql_version)
_set_var(_env, "MOUNT_SOURCES", self.mount_sources)
@@ -542,8 +545,6 @@ class ShellParams:
_set_var(_env, "PYTHON_MAJOR_MINOR_VERSION", self.python)
_set_var(_env, "QUIET", self.quiet)
_set_var(_env, "REDIS_HOST_PORT", None, REDIS_HOST_PORT)
- _set_var(_env, "MSSQL_HOST_PORT", None, MSSQL_HOST_PORT)
- _set_var(_env, "DRILL_HOST_PORT", None, DRILL_HOST_PORT)
_set_var(_env, "REGENERATE_MISSING_DOCS", self.regenerate_missing_docs)
_set_var(_env, "REMOVE_ARM_PACKAGES", self.remove_arm_packages)
_set_var(_env, "RUN_SYSTEM_TESTS", self.run_system_tests)
diff --git a/scripts/ci/docker-compose/integration-celery.yml
b/scripts/ci/docker-compose/integration-celery.yml
index 02129cd096..16ba20ce7c 100644
--- a/scripts/ci/docker-compose/integration-celery.yml
+++ b/scripts/ci/docker-compose/integration-celery.yml
@@ -17,11 +17,14 @@
---
services:
rabbitmq:
+ container_name: rabbitmq
image: rabbitmq:3.7
labels:
breeze.description: "Integration required for Celery executor tests."
volumes:
- rabbitmq-db-volume:/var/lib/rabbitmq
+ ports:
+ - "${RABBITMQ_HOST_PORT}:5672"
healthcheck:
test: rabbitmq-diagnostics -q ping
interval: 5s
@@ -29,17 +32,9 @@ services:
retries: 50
restart: "on-failure"
redis:
- image: redis:5.0.1
- volumes:
- - redis-db-volume:/data/redis
- ports:
- - "${REDIS_HOST_PORT}:6379"
- healthcheck:
- test: ["CMD", "redis-cli", "ping"]
- interval: 5s
- timeout: 30s
- retries: 50
- restart: "on-failure"
+ extends:
+ file: integration-redis.yml
+ service: redis
airflow:
environment:
- INTEGRATION_CELERY=true
diff --git a/scripts/ci/docker-compose/integration-celery.yml
b/scripts/ci/docker-compose/integration-redis.yml
similarity index 69%
copy from scripts/ci/docker-compose/integration-celery.yml
copy to scripts/ci/docker-compose/integration-redis.yml
index 02129cd096..29dd95912d 100644
--- a/scripts/ci/docker-compose/integration-celery.yml
+++ b/scripts/ci/docker-compose/integration-redis.yml
@@ -16,20 +16,10 @@
# under the License.
---
services:
- rabbitmq:
- image: rabbitmq:3.7
- labels:
- breeze.description: "Integration required for Celery executor tests."
- volumes:
- - rabbitmq-db-volume:/var/lib/rabbitmq
- healthcheck:
- test: rabbitmq-diagnostics -q ping
- interval: 5s
- timeout: 30s
- retries: 50
- restart: "on-failure"
redis:
image: redis:5.0.1
+ labels:
+ breeze.description: "Integration required for Redis tests."
volumes:
- redis-db-volume:/data/redis
ports:
@@ -42,15 +32,9 @@ services:
restart: "on-failure"
airflow:
environment:
- - INTEGRATION_CELERY=true
- - CELERY_FLOWER=${CELERY_FLOWER}
- - AIRFLOW__CELERY__BROKER_URL=${AIRFLOW__CELERY__BROKER_URL}
+ - INTEGRATION_REDIS=true
depends_on:
redis:
condition: service_healthy
- rabbitmq:
- condition: service_healthy
-
volumes:
- rabbitmq-db-volume:
redis-db-volume:
diff --git a/scripts/in_container/check_environment.sh
b/scripts/in_container/check_environment.sh
index c2a2384dde..8ddcaa9fca 100755
--- a/scripts/in_container/check_environment.sh
+++ b/scripts/in_container/check_environment.sh
@@ -156,6 +156,9 @@ fi
if [[ ${INTEGRATION_MONGO} == "true" ]]; then
check_service "MongoDB" "run_nc mongo 27017" 50
fi
+if [[ ${INTEGRATION_REDIS} == "true" ]]; then
+ check_service "Redis" "run_nc redis 6379" 50
+fi
if [[ ${INTEGRATION_CELERY} == "true" ]]; then
check_service "Redis" "run_nc redis 6379" 50
check_service "RabbitMQ" "run_nc rabbitmq 5672" 50
diff --git a/tests/integration/executors/test_celery_executor.py
b/tests/integration/executors/test_celery_executor.py
index 5893438d42..03a43cc5eb 100644
--- a/tests/integration/executors/test_celery_executor.py
+++ b/tests/integration/executors/test_celery_executor.py
@@ -22,6 +22,7 @@ import json
import logging
import os
import sys
+from ast import literal_eval
from datetime import datetime
from importlib import reload
from time import sleep
@@ -50,9 +51,7 @@ logger = logging.getLogger(__name__)
def _prepare_test_bodies():
- if "CELERY_BROKER_URLS" in os.environ:
- return os.environ["CELERY_BROKER_URLS"].split(",")
- return [conf.get("celery", "BROKER_URL")]
+ return literal_eval(os.environ["CELERY_BROKER_URLS_MAP"]).values()
class FakeCeleryResult:
diff --git a/tests/integration/providers/redis/hooks/test_redis.py
b/tests/integration/providers/redis/hooks/test_redis.py
index d7760933f2..eac17ee676 100644
--- a/tests/integration/providers/redis/hooks/test_redis.py
+++ b/tests/integration/providers/redis/hooks/test_redis.py
@@ -22,7 +22,7 @@ import pytest
from airflow.providers.redis.hooks.redis import RedisHook
[email protected]("celery")
[email protected]("redis")
class TestRedisHook:
def test_real_ping(self):
hook = RedisHook(redis_conn_id="redis_default")
diff --git a/tests/integration/providers/redis/operators/test_redis_publish.py
b/tests/integration/providers/redis/operators/test_redis_publish.py
index 4467836912..c5ea8a65bd 100644
--- a/tests/integration/providers/redis/operators/test_redis_publish.py
+++ b/tests/integration/providers/redis/operators/test_redis_publish.py
@@ -29,7 +29,7 @@ from airflow.utils import timezone
DEFAULT_DATE = timezone.datetime(2017, 1, 1)
[email protected]("celery")
[email protected]("redis")
class TestRedisPublishOperator:
def setup_method(self):
args = {"owner": "airflow", "start_date": DEFAULT_DATE}
diff --git a/tests/integration/providers/redis/sensors/test_redis_key.py
b/tests/integration/providers/redis/sensors/test_redis_key.py
index 76034a1be7..f54e816c29 100644
--- a/tests/integration/providers/redis/sensors/test_redis_key.py
+++ b/tests/integration/providers/redis/sensors/test_redis_key.py
@@ -27,7 +27,7 @@ from airflow.utils import timezone
DEFAULT_DATE = timezone.datetime(2017, 1, 1)
[email protected]("celery")
[email protected]("redis")
class TestRedisSensor:
def setup_method(self):
args = {"owner": "airflow", "start_date": DEFAULT_DATE}
diff --git a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py
b/tests/integration/providers/redis/sensors/test_redis_pub_sub.py
index f452871bdb..55bc9845ba 100644
--- a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py
+++ b/tests/integration/providers/redis/sensors/test_redis_pub_sub.py
@@ -30,7 +30,7 @@ from airflow.utils import timezone
DEFAULT_DATE = timezone.datetime(2017, 1, 1)
[email protected]("celery")
[email protected]("redis")
class TestRedisPubSubSensor:
def setup_method(self):
args = {"owner": "airflow", "start_date": DEFAULT_DATE}
