This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v3-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 69a0fbac7a048474ded75159363ac5eb7d519c73 Author: LIU ZHE YOU <[email protected]> AuthorDate: Sun Nov 2 02:18:47 2025 +0800 Add `localstack` Breeze integration (#54050) (#57668) * Add localstack breeze integration * Fix Breeze unit test * Disbale test for localstack and export envs * Add docs for Running External System Integrations with Breeze * Fix output-commands.svg (cherry picked from commit 84bd22c3d78aafc5939e1256bacc4918dc666829) --- Dockerfile.ci | 21 ++++++++++++ contributing-docs/testing/integration_tests.rst | 2 ++ dev/breeze/doc/03_developer_tasks.rst | 23 ++++++++++++++ dev/breeze/doc/images/output-commands.svg | 10 +++--- dev/breeze/doc/images/output_shell.svg | 14 ++++---- dev/breeze/doc/images/output_shell.txt | 2 +- dev/breeze/doc/images/output_start-airflow.svg | 14 ++++---- dev/breeze/doc/images/output_start-airflow.txt | 2 +- .../output_testing_providers-integration-tests.svg | 4 +-- .../output_testing_providers-integration-tests.txt | 2 +- dev/breeze/src/airflow_breeze/global_constants.py | 2 ++ .../src/airflow_breeze/params/shell_params.py | 1 + .../ci/docker-compose/integration-localstack.yml | 37 ++++++++++++++++++++++ scripts/docker/entrypoint_ci.sh | 21 ++++++++++++ 14 files changed, 131 insertions(+), 24 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 0927b49eb7b..4e39ccf966f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1208,6 +1208,27 @@ function environment_initialization() { ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null fi + if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then + echo + echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" + echo + + # Define LocalStack AWS configuration + declare -A localstack_config=( + ["AWS_ENDPOINT_URL"]="http://localstack:4566"; + ["AWS_ACCESS_KEY_ID"]="test" + ["AWS_SECRET_ACCESS_KEY"]="test" + ["AWS_DEFAULT_REGION"]="us-east-1" + ) + + # Export each configuration variable and log it + for key in "${!localstack_config[@]}"; do + export "$key"="${localstack_config[$key]}" + echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" + done + echo + fi + cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index a5d6c6deb87..af0554c48ab 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -70,6 +70,8 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | keycloak | Integration for manual testing of multi-team Airflow. | +--------------+-------------------------------------------------------+ +| localstack | Integration that emulates AWS services locally. | ++--------------+-------------------------------------------------------+ | mongo | Integration required for MongoDB hooks. | +--------------+-------------------------------------------------------+ | mssql | Integration required for mssql hooks. | diff --git a/dev/breeze/doc/03_developer_tasks.rst b/dev/breeze/doc/03_developer_tasks.rst index 38d9db39d83..4d0d4c6905b 100644 --- a/dev/breeze/doc/03_developer_tasks.rst +++ b/dev/breeze/doc/03_developer_tasks.rst @@ -392,6 +392,29 @@ These are all available flags of ``start-airflow`` command: :width: 100% :alt: Breeze start-airflow +Running External System Integrations with Breeze +------------------------------------------------ + +You can run Airflow alongside external systems in Breeze, such as Kafka, Cassandra, MongoDB, and more. + +To start Airflow with an integration, use the following command: + +.. code-block:: bash + + breeze --python 3.10 --backend postgres --integration <integration_name> + +For example, to run Airflow with Kafka: + +.. code-block:: bash + + breeze --python 3.10 --backend postgres --integration kafka + +Check the available integrations by running: + +.. code-block:: bash + + breeze --integration --help + Launching multiple terminals in the same environment ---------------------------------------------------- diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index d3656bc530f..4df859dfffe 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -338,11 +338,11 @@ </text><text class="breeze-help-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-help-line-8)">│</text><text class="breeze-help-r4" x="24.4" y="215.2" textLength="158.6" clip-path="url(#breeze-help-line-8)">--integration</text><text class="breeze-help-r1" x="768.6" y="215.2" textLength="671" clip-path="url(#breeze-help-line-8)">Core Integrations to enable when running (can be more  </text><text class="breeze-help-r5" x="1451.8 [...] </text><text class="breeze-help-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-help-line-9)">│</text><text class="breeze-help-r1" x="768.6" y="239.6" textLength="671" clip-path="url(#breeze-help-line-9)">than one).                                           & [...] </text><text class="breeze-help-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-help-line-10)">│</text><text class="breeze-help-r7" x="768.6" y="264" textLength="671" clip-path="url(#breeze-help-line-10)">(all | all-testable | cassandra | celery | drill |     </text><text class="breeze-help-r5" x="1451.8" y="264" textLength="12.2" clip-path="url(#breeze-help-line-10)">│</text><text class="breeze-help-r1" x=" [...] -</text><text class="breeze-help-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-help-line-11)">│</text><text class="breeze-help-r7" x="768.6" y="288.4" textLength="671" clip-path="url(#breeze-help-line-11)">kafka | kerberos | keycloak | mongo | mssql |          </text><text class="breeze-help-r5" x="1451.8" y="288.4" textLength="12.2" clip-path="url(#breeze-help-line-11)">│</text>< [...] -</text><text class="breeze-help-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-help-line-12)">│</text><text class="breeze-help-r7" x="768.6" y="312.8" textLength="671" clip-path="url(#breeze-help-line-12)">openlineage | otel | pinot | qdrant | redis | redis |  </text><text class="breeze-help-r5" x="1451.8" y="312.8" textLength="12.2" clip-path="url(#breeze-help-line-12)">│</text><text class="breeze-help-r1" x= [...] -</text><text class="breeze-help-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-help-line-13)">│</text><text class="breeze-help-r7" x="768.6" y="337.2" textLength="671" clip-path="url(#breeze-help-line-13)">statsd | tinkerpop | trino | ydb)                      </text><text class="breeze-help-r5" x="1451.8" y="337.2" textLength="12.2" cl [...] -</text><text class="breeze-help-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-help-line-14)">│</text><text class="breeze-help-r4" x="24.4" y="361.6" textLength="317.2" clip-path="url(#breeze-help-line-14)">--standalone-dag-processor</text><text class="breeze-help-r1" x="341.6" y="361.6" textLength="12.2" clip-path="url(#breeze-help-line-14)">/</text><text class="breeze-help-r4" x="353.8" y="361.6" textLength="341.6" clip-path="url(#breeze-help-line-14)">--no-standalone-dag [...] -</text><text class="breeze-help-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-help-line-15)">│</text><text class="breeze-help-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-help-line-15)">r</text><text class="breeze-help-r1" x="768.6" y="386" textLength="671" clip-path="url(#breeze-help-line-15)">(required for Airflow 3).                      [...] +</text><text class="breeze-help-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-help-line-11)">│</text><text class="breeze-help-r7" x="768.6" y="288.4" textLength="671" clip-path="url(#breeze-help-line-11)">kafka | kerberos | keycloak | localstack | mongo |     </text><text class="breeze-help-r5" x="1451.8" y="288.4" textLength="12.2" clip-path="url(#breeze-help-line-11)">│</text><text class="breeze-help-r [...] +</text><text class="breeze-help-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-help-line-12)">│</text><text class="breeze-help-r7" x="768.6" y="312.8" textLength="671" clip-path="url(#breeze-help-line-12)">mssql | openlineage | otel | pinot | qdrant | redis |  </text><text class="breeze-help-r5" x="1451.8" y="312.8" textLength="12.2" clip-path="url(#breeze-help-line-12)">│</text><text class="breeze-help-r1" x= [...] +</text><text class="breeze-help-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-help-line-13)">│</text><text class="breeze-help-r7" x="768.6" y="337.2" textLength="671" clip-path="url(#breeze-help-line-13)">redis | statsd | tinkerpop | trino | ydb)              </text><text class="breeze-help-r5" x="1451.8" y="337.2" textLength="12.2" clip-path="url(#breeze-help-line [...] +</text><text class="breeze-help-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-help-line-14)">│</text><text class="breeze-help-r4" x="24.4" y="361.6" textLength="317.2" clip-path="url(#breeze-help-line-14)">--standalone-dag-processor</text><text class="breeze-help-r1" x="341.6" y="361.6" textLength="12.2" clip-path="url(#breeze-help-line-14)">/</text><text class="breeze-help-r4" x="353.8" y="361.6" textLength="341.6" clip-path="url(#breeze-help-line-14)">--no-standalone-dag [...] +</text><text class="breeze-help-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-help-line-15)">│</text><text class="breeze-help-r1" x="768.6" y="386" textLength="671" clip-path="url(#breeze-help-line-15)">(required for Airflow 3).                              </text><text class="breeze-help-r5" x="1451.8" y="386" t [...] </text><text class="breeze-help-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-help-line-16)">│</text><text class="breeze-help-r5" x="768.6" y="410.4" textLength="671" clip-path="url(#breeze-help-line-16)">[default: standalone-dag-processor]                    </text><text class="breeze-help-r5" x="1451.8" y="410.4" textLength="12.2" clip-path="url(#breeze-help-line-16)" [...] </text><text class="breeze-help-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-help-line-17)">│</text><text class="breeze-help-r4" x="24.4" y="434.8" textLength="170.8" clip-path="url(#breeze-help-line-17)">--auth-manager</text><text class="breeze-help-r1" x="768.6" y="434.8" textLength="463.6" clip-path="url(#breeze-help-line-17)">Specify the auth manager to set       </text><text class="breeze-help-r5" x="1451.8" [...] </text><text class="breeze-help-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-help-line-18)">│</text><text class="breeze-help-r7" x="768.6" y="459.2" textLength="463.6" clip-path="url(#breeze-help-line-18)">(>SimpleAuthManager< | FabAuthManager)</text><text class="breeze-help-r5" x="1451.8" y="459.2" textLength="12.2" clip-path="url(#breeze-help-line-18)">│</text><text class="breeze-help-r1" x="1464" y="459.2" textLength="12.2" clip-path="url(#breeze-help-l [...] diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 6dbf70093a2..89e7b8ddcc0 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -604,15 +604,15 @@ </text><text class="breeze-shell-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-shell-line-10)">│</text><text class="breeze-shell-r4" x="24.4" y="264" textLength="158.6" clip-path="url(#breeze-shell-line-10)">--integration</text><text class="breeze-shell-r1" x="768.6" y="264" textLength="671" clip-path="url(#breeze-shell-line-10)">Core Integrations to enable when running (can be more  </text><text class="breeze-shell-r5" x="14 [...] </text><text class="breeze-shell-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-shell-line-11)">│</text><text class="breeze-shell-r1" x="768.6" y="288.4" textLength="671" clip-path="url(#breeze-shell-line-11)">than one).                                          & [...] </text><text class="breeze-shell-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-shell-line-12)">│</text><text class="breeze-shell-r7" x="768.6" y="312.8" textLength="671" clip-path="url(#breeze-shell-line-12)">(all | all-testable | cassandra | celery | drill |     </text><text class="breeze-shell-r5" x="1451.8" y="312.8" textLength="12.2" clip-path="url(#breeze-shell-line-12)">│</text><text class="breeze- [...] -</text><text class="breeze-shell-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-shell-line-13)">│</text><text class="breeze-shell-r7" x="768.6" y="337.2" textLength="671" clip-path="url(#breeze-shell-line-13)">kafka | kerberos | keycloak | mongo | mssql |          </text><text class="breeze-shell-r5" x="1451.8" y="337.2" textLength="12.2" clip-path="url(#breeze-shell-line-13)">│</ [...] -</text><text class="breeze-shell-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-shell-line-14)">│</text><text class="breeze-shell-r7" x="768.6" y="361.6" textLength="671" clip-path="url(#breeze-shell-line-14)">openlineage | otel | pinot | qdrant | redis | redis |  </text><text class="breeze-shell-r5" x="1451.8" y="361.6" textLength="12.2" clip-path="url(#breeze-shell-line-14)">│</text><text class="breeze-shell [...] -</text><text class="breeze-shell-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-shell-line-15)">│</text><text class="breeze-shell-r7" x="768.6" y="386" textLength="671" clip-path="url(#breeze-shell-line-15)">statsd | tinkerpop | trino | ydb)                      </text><text class="breeze-shell-r5" x="1451.8" y="386" textLength="12.2" cli [...] +</text><text class="breeze-shell-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-shell-line-13)">│</text><text class="breeze-shell-r7" x="768.6" y="337.2" textLength="671" clip-path="url(#breeze-shell-line-13)">kafka | kerberos | keycloak | localstack | mongo |     </text><text class="breeze-shell-r5" x="1451.8" y="337.2" textLength="12.2" clip-path="url(#breeze-shell-line-13)">│</text><text class="breeze- [...] +</text><text class="breeze-shell-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-shell-line-14)">│</text><text class="breeze-shell-r7" x="768.6" y="361.6" textLength="671" clip-path="url(#breeze-shell-line-14)">mssql | openlineage | otel | pinot | qdrant | redis |  </text><text class="breeze-shell-r5" x="1451.8" y="361.6" textLength="12.2" clip-path="url(#breeze-shell-line-14)">│</text><text class="breeze-shell [...] +</text><text class="breeze-shell-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-shell-line-15)">│</text><text class="breeze-shell-r7" x="768.6" y="386" textLength="671" clip-path="url(#breeze-shell-line-15)">redis | statsd | tinkerpop | trino | ydb)              </text><text class="breeze-shell-r5" x="1451.8" y="386" textLength="12.2" clip-path="url(#breeze-shell-line [...] </text><text class="breeze-shell-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-shell-line-16)">│</text><text class="breeze-shell-r4" x="24.4" y="410.4" textLength="231.8" clip-path="url(#breeze-shell-line-16)">--load-example-dags</text><text class="breeze-shell-r6" x="719.8" y="410.4" textLength="24.4" clip-path="url(#breeze-shell-line-16)">-e</text><text class="breeze-shell-r1" x="768.6" y="410.4" textLength="671" clip-path="url(#breeze-shell-line-16)">Enable configu [...] </text><text class="breeze-shell-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-shell-line-17)">│</text><text class="breeze-shell-r1" x="768.6" y="434.8" textLength="671" clip-path="url(#breeze-shell-line-17)">Airflow.                                           &# [...] </text><text class="breeze-shell-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-shell-line-18)">│</text><text class="breeze-shell-r4" x="24.4" y="459.2" textLength="317.2" clip-path="url(#breeze-shell-line-18)">--load-default-connections</text><text class="breeze-shell-r6" x="719.8" y="459.2" textLength="24.4" clip-path="url(#breeze-shell-line-18)">-c</text><text class="breeze-shell-r1" x="768.6" y="459.2" textLength="671" clip-path="url(#breeze-shell-line-18)">Enable  [...] </text><text class="breeze-shell-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-shell-line-19)">│</text><text class="breeze-shell-r1" x="768.6" y="483.6" textLength="671" clip-path="url(#breeze-shell-line-19)">starting Airflow.                                      </text><text class [...] -</text><text class="breeze-shell-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-shell-line-20)">│</text><text class="breeze-shell-r4" x="24.4" y="508" textLength="317.2" clip-path="url(#breeze-shell-line-20)">--standalone-dag-processor</text><text class="breeze-shell-r1" x="341.6" y="508" textLength="12.2" clip-path="url(#breeze-shell-line-20)">/</text><text class="breeze-shell-r4" x="353.8" y="508" textLength="341.6" clip-path="url(#breeze-shell-line-20)">--no-standalone-dag [...] -</text><text class="breeze-shell-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-shell-line-21)">│</text><text class="breeze-shell-r4" x="24.4" y="532.4" textLength="12.2" clip-path="url(#breeze-shell-line-21)">r</text><text class="breeze-shell-r1" x="768.6" y="532.4" textLength="671" clip-path="url(#breeze-shell-line-21)">(required for Airflow 3).                    [...] +</text><text class="breeze-shell-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-shell-line-20)">│</text><text class="breeze-shell-r4" x="24.4" y="508" textLength="317.2" clip-path="url(#breeze-shell-line-20)">--standalone-dag-processor</text><text class="breeze-shell-r1" x="341.6" y="508" textLength="12.2" clip-path="url(#breeze-shell-line-20)">/</text><text class="breeze-shell-r4" x="353.8" y="508" textLength="341.6" clip-path="url(#breeze-shell-line-20)">--no-standalone-dag [...] +</text><text class="breeze-shell-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-shell-line-21)">│</text><text class="breeze-shell-r1" x="768.6" y="532.4" textLength="671" clip-path="url(#breeze-shell-line-21)">(required for Airflow 3).                              </text><text class="breeze-shell-r5" x="1451.8" [...] </text><text class="breeze-shell-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-shell-line-22)">│</text><text class="breeze-shell-r5" x="768.6" y="556.8" textLength="671" clip-path="url(#breeze-shell-line-22)">[default: standalone-dag-processor]                    </text><text class="breeze-shell-r5" x="1451.8" y="556.8" textLength="12.2" clip-path="url(#breeze-shell-lin [...] </text><text class="breeze-shell-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-shell-line-23)">│</text><text class="breeze-shell-r4" x="24.4" y="581.2" textLength="390.4" clip-path="url(#breeze-shell-line-23)">--start-api-server-with-examples</text><text class="breeze-shell-r1" x="768.6" y="581.2" textLength="671" clip-path="url(#breeze-shell-line-23)">Start minimal airflow api-server with examples (for    </text><text clas [...] </text><text class="breeze-shell-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-shell-line-24)">│</text><text class="breeze-shell-r1" x="768.6" y="605.6" textLength="671" clip-path="url(#breeze-shell-line-24)">testing purposes) when entering breeze.                </text><text class="breeze-shell-r5" x="1451.8" y="605.6" textLength="12.2" clip-path="url(#breeze-shell-line-24) [...] @@ -706,8 +706,8 @@ </text><text class="breeze-shell-r5" x="0" y="2752.8" textLength="12.2" clip-path="url(#breeze-shell-line-112)">│</text><text class="breeze-shell-r4" x="24.4" y="2752.8" textLength="244" clip-path="url(#breeze-shell-line-112)">--excluded-providers</text><text class="breeze-shell-r1" x="756.4" y="2752.8" textLength="683.2" clip-path="url(#breeze-shell-line-112)">JSON-string of dictionary containing excluded providers </text><text class="breeze-shell-r5" x="14 [...] </text><text class="breeze-shell-r5" x="0" y="2777.2" textLength="12.2" clip-path="url(#breeze-shell-line-113)">│</text><text class="breeze-shell-r1" x="756.4" y="2777.2" textLength="683.2" clip-path="url(#breeze-shell-line-113)">per python version ({'3.12': ['provider']})             </text><text class="breeze-shell-r5" x="1451.8" y="2777.2" textLength="12.2" clip-path="url(#breeze-s [...] </text><text class="breeze-shell-r5" x="0" y="2801.6" textLength="12.2" clip-path="url(#breeze-shell-line-114)">│</text><text class="breeze-shell-r7" x="756.4" y="2801.6" textLength="683.2" clip-path="url(#breeze-shell-line-114)">(TEXT)                                           [...] -</text><text class="breeze-shell-r5" x="0" y="2826" textLength="12.2" clip-path="url(#breeze-shell-line-115)">│</text><text class="breeze-shell-r4" x="24.4" y="2826" textLength="414.8" clip-path="url(#breeze-shell-line-115)">--install-airflow-with-constraints</text><text class="breeze-shell-r1" x="439.2" y="2826" textLength="12.2" clip-path="url(#breeze-shell-line-115)">/</text><text class="breeze-shell-r4" x="451.4" y="2826" textLength="256.2" clip-path="url(#breeze-shell-line-115)">--n [...] -</text><text class="breeze-shell-r5" x="0" y="2850.4" textLength="12.2" clip-path="url(#breeze-shell-line-116)">│</text><text class="breeze-shell-r4" x="24.4" y="2850.4" textLength="195.2" clip-path="url(#breeze-shell-line-116)">with-constraints</text><text class="breeze-shell-r1" x="756.4" y="2850.4" textLength="683.2" clip-path="url(#breeze-shell-line-116)">determined from package or airflow version.          &# [...] +</text><text class="breeze-shell-r5" x="0" y="2826" textLength="12.2" clip-path="url(#breeze-shell-line-115)">│</text><text class="breeze-shell-r4" x="24.4" y="2826" textLength="414.8" clip-path="url(#breeze-shell-line-115)">--install-airflow-with-constraints</text><text class="breeze-shell-r1" x="439.2" y="2826" textLength="12.2" clip-path="url(#breeze-shell-line-115)">/</text><text class="breeze-shell-r4" x="451.4" y="2826" textLength="256.2" clip-path="url(#breeze-shell-line-115)">--n [...] +</text><text class="breeze-shell-r5" x="0" y="2850.4" textLength="12.2" clip-path="url(#breeze-shell-line-116)">│</text><text class="breeze-shell-r1" x="756.4" y="2850.4" textLength="683.2" clip-path="url(#breeze-shell-line-116)">determined from package or airflow version.             </text><text class="breeze-shell-r5" x="1451.8" y="2850.4" textLength="12.2" clip-path="url(#breeze-shell-line-116)" [...] </text><text class="breeze-shell-r5" x="0" y="2874.8" textLength="12.2" clip-path="url(#breeze-shell-line-117)">│</text><text class="breeze-shell-r5" x="756.4" y="2874.8" textLength="683.2" clip-path="url(#breeze-shell-line-117)">[default: install-airflow-with-constraints]             </text><text class="breeze-shell-r5" x="1451.8" y="2874.8" textLength="12.2" clip-path="url(#breeze-shell-line-117)">│</text><text class [...] </text><text class="breeze-shell-r5" x="0" y="2899.2" textLength="12.2" clip-path="url(#breeze-shell-line-118)">│</text><text class="breeze-shell-r4" x="24.4" y="2899.2" textLength="341.6" clip-path="url(#breeze-shell-line-118)">--install-selected-providers</text><text class="breeze-shell-r1" x="756.4" y="2899.2" textLength="683.2" clip-path="url(#breeze-shell-line-118)">Comma-separated list of providers selected to be       [...] </text><text class="breeze-shell-r5" x="0" y="2923.6" textLength="12.2" clip-path="url(#breeze-shell-line-119)">│</text><text class="breeze-shell-r1" x="756.4" y="2923.6" textLength="231.8" clip-path="url(#breeze-shell-line-119)">installed (implies </text><text class="breeze-shell-r4" x="988.2" y="2923.6" textLength="353.8" clip-path="url(#breeze-shell-line-119)">--use-distributions-from-dist</text><text class="breeze-shell-r1" x="1342" y="2923.6" textLength="97.6" clip-path="u [...] diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index a2bca2d863a..6ca99f63129 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -c23fcef0bfb6f2bbb46937d673e4cddb +11c2a41c8c4dd97becbe2af203fb9baa diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index e4f8058584b..7bd5899d022 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -518,11 +518,11 @@ </text><text class="breeze-start-airflow-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-14)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="361.6" textLength="158.6" clip-path="url(#breeze-start-airflow-line-14)">--integration</text><text class="breeze-start-airflow-r1" x="768.6" y="361.6" textLength="671" clip-path="url(#breeze-start-airflow-line-14)">Core Integrations to enable when running (can be more [...] </text><text class="breeze-start-airflow-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-start-airflow-line-15)">│</text><text class="breeze-start-airflow-r1" x="768.6" y="386" textLength="671" clip-path="url(#breeze-start-airflow-line-15)">than one).                                      [...] </text><text class="breeze-start-airflow-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-start-airflow-line-16)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="410.4" textLength="671" clip-path="url(#breeze-start-airflow-line-16)">(all | all-testable | cassandra | celery | drill |     </text><text class="breeze-start-airflow-r5" x="1451.8" y="410.4" textLength="12.2" clip-path="url(#breeze-star [...] -</text><text class="breeze-start-airflow-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-17)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="434.8" textLength="671" clip-path="url(#breeze-start-airflow-line-17)">kafka | kerberos | keycloak | mongo | mssql |          </text><text class="breeze-start-airflow-r5" x="1451.8" y="434.8" textLength="12.2" cl [...] -</text><text class="breeze-start-airflow-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-start-airflow-line-18)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="459.2" textLength="671" clip-path="url(#breeze-start-airflow-line-18)">openlineage | otel | pinot | qdrant | redis | redis |  </text><text class="breeze-start-airflow-r5" x="1451.8" y="459.2" textLength="12.2" clip-path="url(#breeze-start-air [...] -</text><text class="breeze-start-airflow-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-19)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="483.6" textLength="671" clip-path="url(#breeze-start-airflow-line-19)">statsd | tinkerpop | trino | ydb)                      </text><text class="breeze-start-airflow- [...] -</text><text class="breeze-start-airflow-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-start-airflow-line-20)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="508" textLength="317.2" clip-path="url(#breeze-start-airflow-line-20)">--standalone-dag-processor</text><text class="breeze-start-airflow-r1" x="341.6" y="508" textLength="12.2" clip-path="url(#breeze-start-airflow-line-20)">/</text><text class="breeze-start-airflow-r4" x="353.8" y="508" textLength="341.6" cl [...] -</text><text class="breeze-start-airflow-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-start-airflow-line-21)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="532.4" textLength="12.2" clip-path="url(#breeze-start-airflow-line-21)">r</text><text class="breeze-start-airflow-r1" x="768.6" y="532.4" textLength="671" clip-path="url(#breeze-start-airflow-line-21)">(required for Airflow 3).            [...] +</text><text class="breeze-start-airflow-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-17)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="434.8" textLength="671" clip-path="url(#breeze-start-airflow-line-17)">kafka | kerberos | keycloak | localstack | mongo |     </text><text class="breeze-start-airflow-r5" x="1451.8" y="434.8" textLength="12.2" clip-path="url(#breeze-star [...] +</text><text class="breeze-start-airflow-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-start-airflow-line-18)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="459.2" textLength="671" clip-path="url(#breeze-start-airflow-line-18)">mssql | openlineage | otel | pinot | qdrant | redis |  </text><text class="breeze-start-airflow-r5" x="1451.8" y="459.2" textLength="12.2" clip-path="url(#breeze-start-air [...] +</text><text class="breeze-start-airflow-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-19)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="483.6" textLength="671" clip-path="url(#breeze-start-airflow-line-19)">redis | statsd | tinkerpop | trino | ydb)              </text><text class="breeze-start-airflow-r5" x="1451.8" y="483.6" textL [...] +</text><text class="breeze-start-airflow-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-start-airflow-line-20)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="508" textLength="317.2" clip-path="url(#breeze-start-airflow-line-20)">--standalone-dag-processor</text><text class="breeze-start-airflow-r1" x="341.6" y="508" textLength="12.2" clip-path="url(#breeze-start-airflow-line-20)">/</text><text class="breeze-start-airflow-r4" x="353.8" y="508" textLength="341.6" cl [...] +</text><text class="breeze-start-airflow-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-start-airflow-line-21)">│</text><text class="breeze-start-airflow-r1" x="768.6" y="532.4" textLength="671" clip-path="url(#breeze-start-airflow-line-21)">(required for Airflow 3).                              </text><text cla [...] </text><text class="breeze-start-airflow-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-22)">│</text><text class="breeze-start-airflow-r5" x="768.6" y="556.8" textLength="671" clip-path="url(#breeze-start-airflow-line-22)">[default: standalone-dag-processor]                    </text><text class="breeze-start-airflow-r5" x="1451.8" y="556.8" textLength [...] </text><text class="breeze-start-airflow-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-start-airflow-line-23)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="581.2" textLength="170.8" clip-path="url(#breeze-start-airflow-line-23)">--auth-manager</text><text class="breeze-start-airflow-r1" x="768.6" y="581.2" textLength="463.6" clip-path="url(#breeze-start-airflow-line-23)">Specify the auth manager to set      [...] </text><text class="breeze-start-airflow-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-24)">│</text><text class="breeze-start-airflow-r7" x="768.6" y="605.6" textLength="463.6" clip-path="url(#breeze-start-airflow-line-24)">(>SimpleAuthManager< | FabAuthManager)</text><text class="breeze-start-airflow-r5" x="1451.8" y="605.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-24)">│</text><text class="breeze-start-airflow-r1" x=" [...] @@ -598,8 +598,8 @@ </text><text class="breeze-start-airflow-r5" x="0" y="2313.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-94)">│</text><text class="breeze-start-airflow-r5" x="756.4" y="2313.6" textLength="683.2" clip-path="url(#breeze-start-airflow-line-94)">[default: ""]                                   [...] </text><text class="breeze-start-airflow-r5" x="0" y="2338" textLength="12.2" clip-path="url(#breeze-start-airflow-line-95)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="2338" textLength="341.6" clip-path="url(#breeze-start-airflow-line-95)">--clean-airflow-installation</text><text class="breeze-start-airflow-r1" x="756.4" y="2338" textLength="683.2" clip-path="url(#breeze-start-airflow-line-95)">Clean the airflow installation before installing  [...] </text><text class="breeze-start-airflow-r5" x="0" y="2362.4" textLength="12.2" clip-path="url(#breeze-start-airflow-line-96)">│</text><text class="breeze-start-airflow-r1" x="756.4" y="2362.4" textLength="158.6" clip-path="url(#breeze-start-airflow-line-96)">specified by </text><text class="breeze-start-airflow-r4" x="915" y="2362.4" textLength="256.2" clip-path="url(#breeze-start-airflow-line-96)">--use-airflow-version</text><text class="breeze-start-airflow-r1" x="1171.2" y= [...] -</text><text class="breeze-start-airflow-r5" x="0" y="2386.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-97)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="2386.8" textLength="414.8" clip-path="url(#breeze-start-airflow-line-97)">--install-airflow-with-constraints</text><text class="breeze-start-airflow-r1" x="439.2" y="2386.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-97)">/</text><text class="breeze-start-airflow-r4" x="451.4" y="2386.8" t [...] -</text><text class="breeze-start-airflow-r5" x="0" y="2411.2" textLength="12.2" clip-path="url(#breeze-start-airflow-line-98)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="2411.2" textLength="195.2" clip-path="url(#breeze-start-airflow-line-98)">with-constraints</text><text class="breeze-start-airflow-r1" x="756.4" y="2411.2" textLength="683.2" clip-path="url(#breeze-start-airflow-line-98)">determined from package or airflow version.    [...] +</text><text class="breeze-start-airflow-r5" x="0" y="2386.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-97)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="2386.8" textLength="414.8" clip-path="url(#breeze-start-airflow-line-97)">--install-airflow-with-constraints</text><text class="breeze-start-airflow-r1" x="439.2" y="2386.8" textLength="12.2" clip-path="url(#breeze-start-airflow-line-97)">/</text><text class="breeze-start-airflow-r4" x="451.4" y="2386.8" t [...] +</text><text class="breeze-start-airflow-r5" x="0" y="2411.2" textLength="12.2" clip-path="url(#breeze-start-airflow-line-98)">│</text><text class="breeze-start-airflow-r1" x="756.4" y="2411.2" textLength="683.2" clip-path="url(#breeze-start-airflow-line-98)">determined from package or airflow version.             </text><text class="breeze-start-airflow-r5" x="1451.8" y="2411.2" textLength="12.2" c [...] </text><text class="breeze-start-airflow-r5" x="0" y="2435.6" textLength="12.2" clip-path="url(#breeze-start-airflow-line-99)">│</text><text class="breeze-start-airflow-r5" x="756.4" y="2435.6" textLength="683.2" clip-path="url(#breeze-start-airflow-line-99)">[default: install-airflow-with-constraints]             </text><text class="breeze-start-airflow-r5" x="1451.8" y="2435.6" textLength="12.2" clip-path="url(#breez [...] </text><text class="breeze-start-airflow-r5" x="0" y="2460" textLength="12.2" clip-path="url(#breeze-start-airflow-line-100)">│</text><text class="breeze-start-airflow-r4" x="24.4" y="2460" textLength="341.6" clip-path="url(#breeze-start-airflow-line-100)">--install-selected-providers</text><text class="breeze-start-airflow-r1" x="756.4" y="2460" textLength="683.2" clip-path="url(#breeze-start-airflow-line-100)">Comma-separated list of providers selected to  [...] </text><text class="breeze-start-airflow-r5" x="0" y="2484.4" textLength="12.2" clip-path="url(#breeze-start-airflow-line-101)">│</text><text class="breeze-start-airflow-r1" x="756.4" y="2484.4" textLength="231.8" clip-path="url(#breeze-start-airflow-line-101)">installed (implies </text><text class="breeze-start-airflow-r4" x="988.2" y="2484.4" textLength="353.8" clip-path="url(#breeze-start-airflow-line-101)">--use-distributions-from-dist</text><text class="breeze-start-airflo [...] diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 70645b7317e..44878609bd0 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -b2b6b3d14961062524f862a86299ccce +9da141373d346a15dd576fa386acbb3b diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg index 6bee01df22d..d023b849a3f 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg @@ -237,8 +237,8 @@ </text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="752" textLength="1464" clip-path="url(#breeze-testing-providers-integration-tests-line-30)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-providers-integration-tests-r1" x="1464" y="752" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-30)"> </text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="776.4" textLength="24.4" clip-path="url(#breeze-testing-providers-integration-tests-line-31)">╭─</text><text class="breeze-testing-providers-integration-tests-r5" x="24.4" y="776.4" textLength="231.8" clip-path="url(#breeze-testing-providers-integration-tests-line-31)"> Integration tests </text><text class="breeze-testing-providers-integration-tests-r5" x="256.2" y="776.4" textLength="1183.4" clip- [...] </text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-32)">│</text><text class="breeze-testing-providers-integration-tests-r4" x="24.4" y="800.8" textLength="158.6" clip-path="url(#breeze-testing-providers-integration-tests-line-32)">--integration</text><text class="breeze-testing-providers-integration-tests-r1" x="231.8" y="800.8" textLength="1207.8" clip-path="url(#breeze-test [...] -</text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-33)">│</text><text class="breeze-testing-providers-integration-tests-r6" x="231.8" y="825.2" textLength="1207.8" clip-path="url(#breeze-testing-providers-integration-tests-line-33)">(all | all-testable | cassandra | celery | drill | kafka | mongo | mssql [...] -</text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-34)">│</text><text class="breeze-testing-providers-integration-tests-r6" x="231.8" y="849.6" textLength="1207.8" clip-path="url(#breeze-testing-providers-integration-tests-line-34)">qdrant | redis | tinkerpop | trino | ydb)           [...] +</text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-33)">│</text><text class="breeze-testing-providers-integration-tests-r6" x="231.8" y="825.2" textLength="1207.8" clip-path="url(#breeze-testing-providers-integration-tests-line-33)">(all | all-testable | cassandra | celery | drill | kafka | localstack | mon [...] +</text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-34)">│</text><text class="breeze-testing-providers-integration-tests-r6" x="231.8" y="849.6" textLength="1207.8" clip-path="url(#breeze-testing-providers-integration-tests-line-34)">| pinot | qdrant | redis | tinkerpop | trino | ydb)       [...] </text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="874" textLength="1464" clip-path="url(#breeze-testing-providers-integration-tests-line-35)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-providers-integration-tests-r1" x="1464" y="874" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-35)"> </text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="898.4" textLength="24.4" clip-path="url(#breeze-testing-providers-integration-tests-line-36)">╭─</text><text class="breeze-testing-providers-integration-tests-r5" x="24.4" y="898.4" textLength="402.6" clip-path="url(#breeze-testing-providers-integration-tests-line-36)"> Advanced flag for tests command </text><text class="breeze-testing-providers-integration-tests-r5" x="427" y="898.4 [...] </text><text class="breeze-testing-providers-integration-tests-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-testing-providers-integration-tests-line-37)">│</text><text class="breeze-testing-providers-integration-tests-r4" x="24.4" y="922.8" textLength="231.8" clip-path="url(#breeze-testing-providers-integration-tests-line-37)">--github-repository</text><text class="breeze-testing-providers-integration-tests-r7" x="366" y="922.8" textLength="24.4" clip-path="url(#breeze-te [...] diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt index 7a3452bc580..f8e31d84496 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt @@ -1 +1 @@ -66a5d51390e696c06b63d6ad296c7784 +c01e190daa309e88d3021a63f3099b52 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index b4dc7002df5..3327b13c042 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -78,6 +78,7 @@ TESTABLE_PROVIDERS_INTEGRATIONS = [ "drill", "tinkerpop", "kafka", + "localstack", "mongo", "mssql", "pinot", @@ -88,6 +89,7 @@ TESTABLE_PROVIDERS_INTEGRATIONS = [ ] DISABLE_TESTABLE_INTEGRATIONS_FROM_CI = [ "mssql", + "localstack", # just for local integration testing for now ] DISABLE_TESTABLE_INTEGRATIONS_FROM_ARM = [ "kerberos", diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6c43d29d766..082b3838f75 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -429,6 +429,7 @@ class ShellParams: else: integrations = self.integration for integration in integrations: + get_console().print(f"[info]Adding integration compose file for {integration}[/]") compose_file_list.append(DOCKER_COMPOSE_DIR / f"integration-{integration}.yml") if "trino" in integrations and "kerberos" not in integrations: get_console().print( diff --git a/scripts/ci/docker-compose/integration-localstack.yml b/scripts/ci/docker-compose/integration-localstack.yml new file mode 100644 index 00000000000..68560bb0476 --- /dev/null +++ b/scripts/ci/docker-compose/integration-localstack.yml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" + image: localstack/localstack:4.7 + labels: + breeze.description: "Integration that emulates AWS services locally." + ports: + - "4566:4566" # LocalStack Gateway + - "4510-4559:4510-4559" # external services port range + environment: + # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ + - DEBUG=${DEBUG:-0} + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + + airflow: + environment: + - INTEGRATION_LOCALSTACK=true + depends_on: + - localstack diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index af93ccaa026..bf64cb795e9 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -184,6 +184,27 @@ function environment_initialization() { ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null fi + if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then + echo + echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" + echo + + # Define LocalStack AWS configuration + declare -A localstack_config=( + ["AWS_ENDPOINT_URL"]="http://localstack:4566"; + ["AWS_ACCESS_KEY_ID"]="test" + ["AWS_SECRET_ACCESS_KEY"]="test" + ["AWS_DEFAULT_REGION"]="us-east-1" + ) + + # Export each configuration variable and log it + for key in "${!localstack_config[@]}"; do + export "$key"="${localstack_config[$key]}" + echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" + done + echo + fi + cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags
