This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 954a20280de603dfd195e2d89068c64226921376 Author: Timon Viola <[email protected]> AuthorDate: Wed Jan 3 19:21:47 2024 +0100 Add pre-commit check to check integrations table. (#36497) (cherry picked from commit 9d1eba087b488d473f8e3a3b12df63d83c7364e8) --- .pre-commit-config.yaml | 8 + STATIC_CODE_CHECKS.rst | 2 + TESTING.rst | 45 +++-- dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 + images/breeze/output_static-checks.svg | 132 +++++++------- images/breeze/output_static-checks.txt | 2 +- .../ci/docker-compose/integration-cassandra.yml | 3 + scripts/ci/docker-compose/integration-celery.yml | 3 + scripts/ci/docker-compose/integration-kafka.yml | 3 + scripts/ci/docker-compose/integration-kerberos.yml | 3 + scripts/ci/docker-compose/integration-mongo.yml | 3 + .../ci/docker-compose/integration-openlineage.yml | 4 +- scripts/ci/docker-compose/integration-otel.yml | 3 + scripts/ci/docker-compose/integration-pinot.yml | 3 + scripts/ci/docker-compose/integration-statsd.yml | 3 + scripts/ci/docker-compose/integration-trino.yml | 3 + .../pre_commit_check_integrations_list.py | 202 +++++++++++++++++++++ 17 files changed, 339 insertions(+), 84 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a7b2a64e5..d7f44643b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -776,6 +776,14 @@ repos: additional_dependencies: ['pyyaml', 'jinja2', 'black==23.10.0', 'tabulate', 'rich>=12.4.4'] require_serial: true pass_filenames: false + - id: check-integrations-list-consistent + name: Sync integrations list with docs + entry: ./scripts/ci/pre_commit/pre_commit_check_integrations_list.py + language: python + files: ^scripts/ci/docker-compose/integration-.*\.yml$|^TESTING.rst$ + additional_dependencies: ['black==23.10.0', 'tabulate', 'rich>=12.4.4', 'pyyaml'] + require_serial: true + pass_filenames: false - id: update-breeze-readme-config-hash name: Update Breeze README.md with config files hash language: python diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 1bed2e8421..29a70e533f 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -202,6 +202,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-init-decorator-arguments | Check model __init__ and decorator arguments are in sync | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-integrations-list-consistent | Sync integrations list with docs | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-lazy-logging | Check that all logging methods are lazy | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-links-to-example-dags-do-not-use-hardcoded-versions | Verify example dags do not use hard-coded version numbers | | diff --git a/TESTING.rst b/TESTING.rst index fba97b870a..3316f8581b 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -1181,24 +1181,33 @@ until you stop the Breeze environment with the ``stop`` command and started with The following integrations are available: -.. list-table:: Airflow Test Integrations - :widths: 15 80 - :header-rows: 1 - - * - Integration - - Description - * - cassandra - - Integration required for Cassandra hooks - * - kerberos - - Integration that provides Kerberos authentication - * - mongo - - Integration required for MongoDB hooks - * - pinot - - Integration required for Apache Pinot hooks - * - celery - - Integration required for Celery executor tests - * - trino - - Integration required for Trino hooks +.. BEGIN AUTO-GENERATED INTEGRATION LIST + ++--------------+----------------------------------------------------+ +| Identifier | Description | ++==============+====================================================+ +| cassandra | Integration required for Cassandra hooks. | ++--------------+----------------------------------------------------+ +| celery | Integration required for Celery executor tests. | ++--------------+----------------------------------------------------+ +| kafka | Integration required for Kafka hooks. | ++--------------+----------------------------------------------------+ +| kerberos | Integration that provides Kerberos authentication. | ++--------------+----------------------------------------------------+ +| mongo | Integration required for MongoDB hooks. | ++--------------+----------------------------------------------------+ +| openlineage | Integration required for Openlineage hooks. | ++--------------+----------------------------------------------------+ +| otel | Integration required for OTEL/opentelemetry hooks. | ++--------------+----------------------------------------------------+ +| pinot | Integration required for Apache Pinot hooks. | ++--------------+----------------------------------------------------+ +| statsd | Integration required for Satsd hooks. | ++--------------+----------------------------------------------------+ +| trino | Integration required for Trino hooks. | ++--------------+----------------------------------------------------+ + +.. END AUTO-GENERATED INTEGRATION LIST' To start the ``mongo`` integration only, enter: diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 1297448cab..dd9f639596 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -54,6 +54,7 @@ PRE_COMMIT_LIST = [ "check-hooks-apply", "check-incorrect-use-of-LoggingMixin", "check-init-decorator-arguments", + "check-integrations-list-consistent", "check-lazy-logging", "check-links-to-example-dags-do-not-use-hardcoded-versions", "check-merge-conflict", diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index 290f6a3264..e601050d09 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 2050.8" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 2075.2" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-static-checks-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="1999.8" /> + <rect x="0" y="0" width="1463.0" height="2024.1999999999998" /> </clipPath> <clipPath id="breeze-static-checks-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -288,9 +288,12 @@ <clipPath id="breeze-static-checks-line-80"> <rect x="0" y="1953.5" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-static-checks-line-81"> + <rect x="0" y="1977.9" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="2048.8" rx="8"/><text class="breeze-static-checks-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: static-checks</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="2073.2" rx="8"/><text class="breeze-static-checks-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: static-checks</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -321,67 +324,68 @@ </text><text class="breeze-static-checks-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-18)">│</text><text class="breeze-static-checks-r7" x="451.4" y="459.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-18)">check-extras-order | check-fab-migrations | check-for-inclusive-language |       </text><text class="breeze-static-checks-r5" x="1451.8" y="459.2" textLength="12.2" clip-path [...] </text><text class="breeze-static-checks-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-19)">│</text><text class="breeze-static-checks-r7" x="451.4" y="483.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-19)">check-google-re2-as-dependency | check-hooks-apply |                             [...] </text><text class="breeze-static-checks-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-static-checks-line-20)">│</text><text class="breeze-static-checks-r7" x="451.4" y="508" textLength="988.2" clip-path="url(#breeze-static-checks-line-20)">check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           </text><text class="breeze-static-checks-r5" x="1451.8" y="508" textLength="12.2" clip- [...] -</text><text class="breeze-static-checks-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-21)">│</text><text class="breeze-static-checks-r7" x="451.4" y="532.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-21)">check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | </text><text class="breeze-static-checks-r5" x="1451.8" y="532.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-21)">│< [...] -</text><text class="breeze-static-checks-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-22)">│</text><text class="breeze-static-checks-r7" x="451.4" y="556.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-22)">check-merge-conflict | check-newsfragments-are-valid |                           </t [...] -</text><text class="breeze-static-checks-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-23)">│</text><text class="breeze-static-checks-r7" x="451.4" y="581.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-23)">check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |</text><text class="breeze-static-checks-r5" x="1451.8" y="581.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-23)">│</text [...] -</text><text class="breeze-static-checks-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-24)">│</text><text class="breeze-static-checks-r7" x="451.4" y="605.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-24)">check-no-relative-imports | check-only-new-session-with-provide-session |        </text><text class="breeze-static-checks-r5" x="1451.8" y="605.6" textLength="12.2" clip-path="url [...] -</text><text class="breeze-static-checks-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-static-checks-line-25)">│</text><text class="breeze-static-checks-r7" x="451.4" y="630" textLength="988.2" clip-path="url(#breeze-static-checks-line-25)">check-persist-credentials-disabled-in-github-workflows |                         </text><text class="breeze- [...] -</text><text class="breeze-static-checks-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-26)">│</text><text class="breeze-static-checks-r7" x="451.4" y="654.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-26)">check-pre-commit-information-consistent | check-provide-create-sessions-imports |</text><text class="breeze-static-checks-r5" x="1451.8" y="654.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-26)">│</text [...] -</text><text class="breeze-static-checks-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-27)">│</text><text class="breeze-static-checks-r7" x="451.4" y="678.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-27)">check-provider-docs-valid | check-provider-yaml-valid |                          </text>< [...] -</text><text class="breeze-static-checks-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-28)">│</text><text class="breeze-static-checks-r7" x="451.4" y="703.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-28)">check-providers-init-file-missing | check-providers-subpackages-init-file-exist |</text><text class="breeze-static-checks-r5" x="1451.8" y="703.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-28)">│</text [...] -</text><text class="breeze-static-checks-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-29)">│</text><text class="breeze-static-checks-r7" x="451.4" y="727.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-29)">check-pydevd-left-in-code | check-revision-heads-map |                           </t [...] -</text><text class="breeze-static-checks-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-static-checks-line-30)">│</text><text class="breeze-static-checks-r7" x="451.4" y="752" textLength="988.2" clip-path="url(#breeze-static-checks-line-30)">check-safe-filter-usage-in-html | check-setup-order |                            </ [...] -</text><text class="breeze-static-checks-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-31)">│</text><text class="breeze-static-checks-r7" x="451.4" y="776.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-31)">check-start-date-not-used-in-defaults | check-system-tests-present |             </text><text class="breeze-static-checks-r5" x="1451.8" y="776.4" textLen [...] -</text><text class="breeze-static-checks-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-32)">│</text><text class="breeze-static-checks-r7" x="451.4" y="800.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-32)">check-system-tests-tocs | check-tests-unittest-testcase |                        </text><text class [...] -</text><text class="breeze-static-checks-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-33)">│</text><text class="breeze-static-checks-r7" x="451.4" y="825.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-33)">check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell</text><text class="breeze-static-checks-r5" x="1451.8" y="825.2" textLength="12.2" clip-path="url(#breeze-static-checks-li [...] -</text><text class="breeze-static-checks-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-34)">│</text><text class="breeze-static-checks-r7" x="451.4" y="849.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-34)">| compile-www-assets | compile-www-assets-dev |                             [...] -</text><text class="breeze-static-checks-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-static-checks-line-35)">│</text><text class="breeze-static-checks-r7" x="451.4" y="874" textLength="988.2" clip-path="url(#breeze-static-checks-line-35)">create-missing-init-py-files-tests | debug-statements | detect-private-key |     </text><text class="breeze-static-checks-r5" x="1451.8" y="874" textLength="12.2" clip-path="url(#breeze-st [...] -</text><text class="breeze-static-checks-r5" x="0" y="898.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-36)">│</text><text class="breeze-static-checks-r7" x="451.4" y="898.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-36)">doctoc | end-of-file-fixer | fix-encoding-pragma | flynt |                       </t [...] -</text><text class="breeze-static-checks-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-37)">│</text><text class="breeze-static-checks-r7" x="451.4" y="922.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-37)">generate-airflow-diagrams | generate-pypi-readme | identity | insert-license |   </text><text class="breeze-static-checks-r5" x="1451.8" y="922.8" textLength="12.2" clip-path="url(#bre [...] -</text><text class="breeze-static-checks-r5" x="0" y="947.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-38)">│</text><text class="breeze-static-checks-r7" x="451.4" y="947.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-38)">lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart |               </text><text class="breeze-static-checks-r5 [...] -</text><text class="breeze-static-checks-r5" x="0" y="971.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-39)">│</text><text class="breeze-static-checks-r7" x="451.4" y="971.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-39)">lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core |</text><text class="breeze-static-checks-r5" x="1451.8" y="971.6" textLength="12.2" clip-path="url(#breeze-s [...] -</text><text class="breeze-static-checks-r5" x="0" y="996" textLength="12.2" clip-path="url(#breeze-static-checks-line-40)">│</text><text class="breeze-static-checks-r7" x="451.4" y="996" textLength="988.2" clip-path="url(#breeze-static-checks-line-40)">mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn |</text><text class="breeze-static-checks-r5" x="1451.8" y="996" textLength="12.2" clip-path="url(#breeze-static- [...] -</text><text class="breeze-static-checks-r5" x="0" y="1020.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-41)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1020.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-41)">replace-bad-characters | rst-backticks | ruff | ruff-format | shellcheck |       </text><text class="breeze-static-checks-r5" x="1451.8" y="1020.4" text [...] -</text><text class="breeze-static-checks-r5" x="0" y="1044.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-42)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1044.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-42)">trailing-whitespace | ts-compile-format-lint-www | update-black-version |        </text><text class="breeze-static-checks-r5" x="1451.8" y="1044.8" textLength="12.2" c [...] -</text><text class="breeze-static-checks-r5" x="0" y="1069.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-43)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1069.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-43)">update-breeze-cmd-output | update-breeze-readme-config-hash |                    </text><text class="breeze-static-ch [...] -</text><text class="breeze-static-checks-r5" x="0" y="1093.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-44)">update-common-sql-api-stubs | update-er-diagram | update-extras |                </text><text class="breeze-static-checks-r5" x [...] -</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2" clip-path="url(#breeze-static-checks-line-45)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2" clip-path="url(#breeze-static-checks-line-45)">update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            </text><text class="breeze-static-checks-r5" x="1451.8" y="1118" textLength="12. [...] -</text><text class="breeze-static-checks-r5" x="0" y="1142.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-46)">update-installed-providers-to-be-sorted | update-local-yml-file |                </text><text class="breeze-static-checks-r5" x="1451.8" [...] -</text><text class="breeze-static-checks-r5" x="0" y="1166.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-47)">update-migration-references | update-providers-dependencies |                    </text><text class="breeze-static-ch [...] -</text><text class="breeze-static-checks-r5" x="0" y="1191.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-48)">update-spelling-wordlist-to-be-sorted | update-supported-versions |              </text><text class="breeze-static-checks-r5" x="1451.8" y="1191.2" [...] -</text><text class="breeze-static-checks-r5" x="0" y="1215.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-49)">update-vendored-in-k8s-json-schema | update-version | validate-pyproject |       </text><text class="breeze-static-checks-r5" x="1451.8" y="1215.6" textLength="12.2" clip-p [...] -</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2" clip-path="url(#breeze-static-checks-line-50)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2" clip-path="url(#breeze-static-checks-line-50)">yamllint)                                       [...] -</text><text class="breeze-static-checks-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1264.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1264.4" textLength="61" clip-path="url(#breeze-static-checks-line-51)">-show</text><text class="breeze-static-checks-r4" x="97.6" y="1264.4" textLength="195.2" clip-path="url(# [...] -</text><text class="breeze-static-checks-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1288.8" textLength="134.2" clip-path="url(#breeze-static-checks-line-52)">-initialize</text><text class="breeze-static-checks-r4" x="170.8" y="1288.8" textLength="146.4" clip-p [...] -</text><text class="breeze-static-checks-r5" x="0" y="1313.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1313.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1313.2" textLength="48.8" clip-path="url(#breeze-static-checks-line-53)">-max</text><text class="breeze-static-checks-r4" x="85.4" y="1313.2" textLength="292.8" clip-path="url( [...] -</text><text class="breeze-static-checks-r5" x="0" y="1337.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1337.6" textLength="854" clip-path="url(#breeze-static-checks-line-54)">(INTEGER RANGE)                                    [...] -</text><text class="breeze-static-checks-r5" x="0" y="1362" textLength="12.2" clip-path="url(#breeze-static-checks-line-55)">│</text><text class="breeze-static-checks-r5" x="451.4" y="1362" textLength="854" clip-path="url(#breeze-static-checks-line-55)">[default: 3; 1<=x<=10]                                 &# [...] -</text><text class="breeze-static-checks-r5" x="0" y="1386.4" textLength="1464" clip-path="url(#breeze-static-checks-line-56)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1386.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-56)"> -</text><text class="breeze-static-checks-r5" x="0" y="1410.8" textLength="24.4" clip-path="url(#breeze-static-checks-line-57)">╭─</text><text class="breeze-static-checks-r5" x="24.4" y="1410.8" textLength="463.6" clip-path="url(#breeze-static-checks-line-57)"> Selecting files to run the checks on </text><text class="breeze-static-checks-r5" x="488" y="1410.8" textLength="951.6" clip-path="url(#breeze-static-checks-line-57)">──────────────────────── [...] -</text><text class="breeze-static-checks-r5" x="0" y="1435.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-58)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1435.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-58)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1435.2" textLength="61" clip-path="url(#breeze-static-checks-line-58)">-file</text><text class="breeze-static-checks-r6" x="256.2" y="1435.2" textLength="24.4" clip-path="url(# [...] -</text><text class="breeze-static-checks-r5" x="0" y="1459.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-59)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1459.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-59)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1459.6" textLength="48.8" clip-path="url(#breeze-static-checks-line-59)">-all</text><text class="breeze-static-checks-r4" x="85.4" y="1459.6" textLength="73.2" clip-path="url(# [...] -</text><text class="breeze-static-checks-r5" x="0" y="1484" textLength="12.2" clip-path="url(#breeze-static-checks-line-60)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1484" textLength="12.2" clip-path="url(#breeze-static-checks-line-60)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1484" textLength="85.4" clip-path="url(#breeze-static-checks-line-60)">-commit</text><text class="breeze-static-checks-r4" x="122" y="1484" textLength="48.8" clip-path="url(#breeze [...] -</text><text class="breeze-static-checks-r5" x="0" y="1508.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-61)">│</text><text class="breeze-static-checks-r1" x="305" y="1508.4" textLength="183" clip-path="url(#breeze-static-checks-line-61)">exclusive with </text><text class="breeze-static-checks-r4" x="488" y="1508.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-61)">-</text><text class="breeze-static-checks-r4" x="500.2" y="1508.4" textLength="61" [...] -</text><text class="breeze-static-checks-r5" x="0" y="1532.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-62)">│</text><text class="breeze-static-checks-r7" x="305" y="1532.8" textLength="1134.6" clip-path="url(#breeze-static-checks-line-62)">(TEXT)                                       [...] -</text><text class="breeze-static-checks-r5" x="0" y="1557.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-63)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1557.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-63)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1557.2" textLength="61" clip-path="url(#breeze-static-checks-line-63)">-last</text><text class="breeze-static-checks-r4" x="97.6" y="1557.2" textLength="85.4" clip-path="url(#b [...] -</text><text class="breeze-static-checks-r5" x="0" y="1581.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-64)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1581.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-64)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1581.6" textLength="61" clip-path="url(#breeze-static-checks-line-64)">-only</text><text class="breeze-static-checks-r4" x="97.6" y="1581.6" textLength="134.2" clip-path="url(# [...] -</text><text class="breeze-static-checks-r5" x="0" y="1606" textLength="12.2" clip-path="url(#breeze-static-checks-line-65)">│</text><text class="breeze-static-checks-r1" x="305" y="1606" textLength="1134.6" clip-path="url(#breeze-static-checks-line-65)">branch and HEAD of your branch.                              & [...] -</text><text class="breeze-static-checks-r5" x="0" y="1630.4" textLength="1464" clip-path="url(#breeze-static-checks-line-66)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1630.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-66)"> -</text><text class="breeze-static-checks-r5" x="0" y="1654.8" textLength="24.4" clip-path="url(#breeze-static-checks-line-67)">╭─</text><text class="breeze-static-checks-r5" x="24.4" y="1654.8" textLength="463.6" clip-path="url(#breeze-static-checks-line-67)"> Building image before running checks </text><text class="breeze-static-checks-r5" x="488" y="1654.8" textLength="951.6" clip-path="url(#breeze-static-checks-line-67)">────────────────────────────────── [...] -</text><text class="breeze-static-checks-r5" x="0" y="1679.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-68)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1679.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-68)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1679.2" textLength="61" clip-path="url(#breeze-static-checks-line-68)">-skip</text><text class="breeze-static-checks-r4" x="97.6" y="1679.2" textLength="244" clip-path="url(#br [...] -</text><text class="breeze-static-checks-r5" x="0" y="1703.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-69)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1703.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-69)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1703.6" textLength="73.2" clip-path="url(#breeze-static-checks-line-69)">-force</text><text class="breeze-static-checks-r4" x="109.8" y="1703.6" textLength="73.2" clip-path="ur [...] -</text><text class="breeze-static-checks-r5" x="0" y="1728" textLength="12.2" clip-path="url(#breeze-static-checks-line-70)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1728" textLength="12.2" clip-path="url(#breeze-static-checks-line-70)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1728" textLength="73.2" clip-path="url(#breeze-static-checks-line-70)">-image</text><text class="breeze-static-checks-r4" x="109.8" y="1728" textLength="48.8" clip-path="url(#breez [...] -</text><text class="breeze-static-checks-r5" x="0" y="1752.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-71)">│</text><text class="breeze-static-checks-r7" x="414.8" y="1752.4" textLength="963.8" clip-path="url(#breeze-static-checks-line-71)">(TEXT)                                      [...] -</text><text class="breeze-static-checks-r5" x="0" y="1776.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-72)">│</text><text class="breeze-static-checks-r5" x="414.8" y="1776.8" textLength="963.8" clip-path="url(#breeze-static-checks-line-72)">[default: latest]                                    [...] -</text><text class="breeze-static-checks-r5" x="0" y="1801.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-73)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1801.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-73)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1801.2" textLength="85.4" clip-path="url(#breeze-static-checks-line-73)">-github</text><text class="breeze-static-checks-r4" x="122" y="1801.2" textLength="134.2" clip-path="ur [...] -</text><text class="breeze-static-checks-r5" x="0" y="1825.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-74)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1825.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-74)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1825.6" textLength="97.6" clip-path="url(#breeze-static-checks-line-74)">-builder</text><text class="breeze-static-checks-r1" x="414.8" y="1825.6" textLength="756.4" clip-path= [...] -</text><text class="breeze-static-checks-r5" x="0" y="1850" textLength="12.2" clip-path="url(#breeze-static-checks-line-75)">│</text><text class="breeze-static-checks-r5" x="414.8" y="1850" textLength="756.4" clip-path="url(#breeze-static-checks-line-75)">[default: autodetect]                                    [...] -</text><text class="breeze-static-checks-r5" x="0" y="1874.4" textLength="1464" clip-path="url(#breeze-static-checks-line-76)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1874.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-76)"> -</text><text class="breeze-static-checks-r5" x="0" y="1898.8" textLength="24.4" clip-path="url(#breeze-static-checks-line-77)">╭─</text><text class="breeze-static-checks-r5" x="24.4" y="1898.8" textLength="195.2" clip-path="url(#breeze-static-checks-line-77)"> Common options </text><text class="breeze-static-checks-r5" x="219.6" y="1898.8" textLength="1220" clip-path="url(#breeze-static-checks-line-77)">────────────────────────────────────────────────────────────────────── [...] -</text><text class="breeze-static-checks-r5" x="0" y="1923.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-78)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1923.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-78)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1923.2" textLength="48.8" clip-path="url(#breeze-static-checks-line-78)">-dry</text><text class="breeze-static-checks-r4" x="85.4" y="1923.2" textLength="48.8" clip-path="url(# [...] -</text><text class="breeze-static-checks-r5" x="0" y="1947.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-79)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1947.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-79)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1947.6" textLength="97.6" clip-path="url(#breeze-static-checks-line-79)">-verbose</text><text class="breeze-static-checks-r6" x="158.6" y="1947.6" textLength="24.4" clip-path=" [...] -</text><text class="breeze-static-checks-r5" x="0" y="1972" textLength="12.2" clip-path="url(#breeze-static-checks-line-80)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1972" textLength="12.2" clip-path="url(#breeze-static-checks-line-80)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1972" textLength="61" clip-path="url(#breeze-static-checks-line-80)">-help</text><text class="breeze-static-checks-r6" x="158.6" y="1972" textLength="24.4" clip-path="url(#breeze-s [...] -</text><text class="breeze-static-checks-r5" x="0" y="1996.4" textLength="1464" clip-path="url(#breeze-static-checks-line-81)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1996.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-81)"> +</text><text class="breeze-static-checks-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-21)">│</text><text class="breeze-static-checks-r7" x="451.4" y="532.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-21)">check-integrations-list-consistent | check-lazy-logging |                        </text><text class [...] +</text><text class="breeze-static-checks-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-22)">│</text><text class="breeze-static-checks-r7" x="451.4" y="556.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-22)">check-links-to-example-dags-do-not-use-hardcoded-versions | check-merge-conflict </text><text class="breeze-static-checks-r5" x="1451.8" y="556.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-22)">│</text [...] +</text><text class="breeze-static-checks-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-23)">│</text><text class="breeze-static-checks-r7" x="451.4" y="581.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-23)">| check-newsfragments-are-valid | check-no-airflow-deprecation-in-providers |    </text><text class="breeze-static-checks-r5" x="1451.8" y="581.2" textLength="12.2" clip-path="url(#breeze-static [...] +</text><text class="breeze-static-checks-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-24)">│</text><text class="breeze-static-checks-r7" x="451.4" y="605.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-24)">check-no-providers-in-core-examples | check-no-relative-imports |                </text><text class="breeze-static-checks-r5" x="1451.8" y= [...] +</text><text class="breeze-static-checks-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-static-checks-line-25)">│</text><text class="breeze-static-checks-r7" x="451.4" y="630" textLength="988.2" clip-path="url(#breeze-static-checks-line-25)">check-only-new-session-with-provide-session |                               &# [...] +</text><text class="breeze-static-checks-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-26)">│</text><text class="breeze-static-checks-r7" x="451.4" y="654.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-26)">check-persist-credentials-disabled-in-github-workflows |                         </text><text class="bre [...] +</text><text class="breeze-static-checks-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-27)">│</text><text class="breeze-static-checks-r7" x="451.4" y="678.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-27)">check-pre-commit-information-consistent | check-provide-create-sessions-imports |</text><text class="breeze-static-checks-r5" x="1451.8" y="678.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-27)">│</text [...] +</text><text class="breeze-static-checks-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-28)">│</text><text class="breeze-static-checks-r7" x="451.4" y="703.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-28)">check-provider-docs-valid | check-provider-yaml-valid |                          </text>< [...] +</text><text class="breeze-static-checks-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-29)">│</text><text class="breeze-static-checks-r7" x="451.4" y="727.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-29)">check-providers-init-file-missing | check-providers-subpackages-init-file-exist |</text><text class="breeze-static-checks-r5" x="1451.8" y="727.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-29)">│</text [...] +</text><text class="breeze-static-checks-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-static-checks-line-30)">│</text><text class="breeze-static-checks-r7" x="451.4" y="752" textLength="988.2" clip-path="url(#breeze-static-checks-line-30)">check-pydevd-left-in-code | check-revision-heads-map |                           </text> [...] +</text><text class="breeze-static-checks-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-31)">│</text><text class="breeze-static-checks-r7" x="451.4" y="776.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-31)">check-safe-filter-usage-in-html | check-setup-order |                            [...] +</text><text class="breeze-static-checks-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-32)">│</text><text class="breeze-static-checks-r7" x="451.4" y="800.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-32)">check-start-date-not-used-in-defaults | check-system-tests-present |             </text><text class="breeze-static-checks-r5" x="1451.8" y="800.8" textLen [...] +</text><text class="breeze-static-checks-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-33)">│</text><text class="breeze-static-checks-r7" x="451.4" y="825.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-33)">check-system-tests-tocs | check-tests-unittest-testcase |                        </text><text class [...] +</text><text class="breeze-static-checks-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-34)">│</text><text class="breeze-static-checks-r7" x="451.4" y="849.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-34)">check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell</text><text class="breeze-static-checks-r5" x="1451.8" y="849.6" textLength="12.2" clip-path="url(#breeze-static-checks-li [...] +</text><text class="breeze-static-checks-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-static-checks-line-35)">│</text><text class="breeze-static-checks-r7" x="451.4" y="874" textLength="988.2" clip-path="url(#breeze-static-checks-line-35)">| compile-www-assets | compile-www-assets-dev |                             [...] +</text><text class="breeze-static-checks-r5" x="0" y="898.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-36)">│</text><text class="breeze-static-checks-r7" x="451.4" y="898.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-36)">create-missing-init-py-files-tests | debug-statements | detect-private-key |     </text><text class="breeze-static-checks-r5" x="1451.8" y="898.4" textLength="12.2" clip-path="url(#bre [...] +</text><text class="breeze-static-checks-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-37)">│</text><text class="breeze-static-checks-r7" x="451.4" y="922.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-37)">doctoc | end-of-file-fixer | fix-encoding-pragma | flynt |                       </t [...] +</text><text class="breeze-static-checks-r5" x="0" y="947.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-38)">│</text><text class="breeze-static-checks-r7" x="451.4" y="947.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-38)">generate-airflow-diagrams | generate-pypi-readme | identity | insert-license |   </text><text class="breeze-static-checks-r5" x="1451.8" y="947.2" textLength="12.2" clip-path="url(#bre [...] +</text><text class="breeze-static-checks-r5" x="0" y="971.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-39)">│</text><text class="breeze-static-checks-r7" x="451.4" y="971.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-39)">lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart |               </text><text class="breeze-static-checks-r5 [...] +</text><text class="breeze-static-checks-r5" x="0" y="996" textLength="12.2" clip-path="url(#breeze-static-checks-line-40)">│</text><text class="breeze-static-checks-r7" x="451.4" y="996" textLength="988.2" clip-path="url(#breeze-static-checks-line-40)">lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core |</text><text class="breeze-static-checks-r5" x="1451.8" y="996" textLength="12.2" clip-path="url(#breeze-static- [...] +</text><text class="breeze-static-checks-r5" x="0" y="1020.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-41)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1020.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-41)">mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn |</text><text class="breeze-static-checks-r5" x="1451.8" y="1020.4" textLength="12.2" clip-path="url(#breez [...] +</text><text class="breeze-static-checks-r5" x="0" y="1044.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-42)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1044.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-42)">replace-bad-characters | rst-backticks | ruff | ruff-format | shellcheck |       </text><text class="breeze-static-checks-r5" x="1451.8" y="1044.8" text [...] +</text><text class="breeze-static-checks-r5" x="0" y="1069.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-43)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1069.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-43)">trailing-whitespace | ts-compile-format-lint-www | update-black-version |        </text><text class="breeze-static-checks-r5" x="1451.8" y="1069.2" textLength="12.2" c [...] +</text><text class="breeze-static-checks-r5" x="0" y="1093.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-44)">update-breeze-cmd-output | update-breeze-readme-config-hash |                    </text><text class="breeze-static-ch [...] +</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2" clip-path="url(#breeze-static-checks-line-45)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2" clip-path="url(#breeze-static-checks-line-45)">update-common-sql-api-stubs | update-er-diagram | update-extras |                </text><text class="breeze-static-checks-r5" x="14 [...] +</text><text class="breeze-static-checks-r5" x="0" y="1142.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-46)">update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            </text><text class="breeze-static-checks-r5" x="1451.8" y="1142.4" textLengt [...] +</text><text class="breeze-static-checks-r5" x="0" y="1166.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2" clip-path="url(#breeze-static-checks-line-47)">update-installed-providers-to-be-sorted | update-local-yml-file |                </text><text class="breeze-static-checks-r5" x="1451.8" [...] +</text><text class="breeze-static-checks-r5" x="0" y="1191.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2" clip-path="url(#breeze-static-checks-line-48)">update-migration-references | update-providers-dependencies |                    </text><text class="breeze-static-ch [...] +</text><text class="breeze-static-checks-r5" x="0" y="1215.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2" clip-path="url(#breeze-static-checks-line-49)">update-spelling-wordlist-to-be-sorted | update-supported-versions |              </text><text class="breeze-static-checks-r5" x="1451.8" y="1215.6" [...] +</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2" clip-path="url(#breeze-static-checks-line-50)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2" clip-path="url(#breeze-static-checks-line-50)">update-vendored-in-k8s-json-schema | update-version | validate-pyproject |       </text><text class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2" clip-path="u [...] +</text><text class="breeze-static-checks-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2" clip-path="url(#breeze-static-checks-line-51)">yamllint)                                     & [...] +</text><text class="breeze-static-checks-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1288.8" textLength="61" clip-path="url(#breeze-static-checks-line-52)">-show</text><text class="breeze-static-checks-r4" x="97.6" y="1288.8" textLength="195.2" clip-path="url(# [...] +</text><text class="breeze-static-checks-r5" x="0" y="1313.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1313.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1313.2" textLength="134.2" clip-path="url(#breeze-static-checks-line-53)">-initialize</text><text class="breeze-static-checks-r4" x="170.8" y="1313.2" textLength="146.4" clip-p [...] +</text><text class="breeze-static-checks-r5" x="0" y="1337.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1337.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1337.6" textLength="48.8" clip-path="url(#breeze-static-checks-line-54)">-max</text><text class="breeze-static-checks-r4" x="85.4" y="1337.6" textLength="292.8" clip-path="url( [...] +</text><text class="breeze-static-checks-r5" x="0" y="1362" textLength="12.2" clip-path="url(#breeze-static-checks-line-55)">│</text><text class="breeze-static-checks-r7" x="451.4" y="1362" textLength="854" clip-path="url(#breeze-static-checks-line-55)">(INTEGER RANGE)                                    &# [...] +</text><text class="breeze-static-checks-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-56)">│</text><text class="breeze-static-checks-r5" x="451.4" y="1386.4" textLength="854" clip-path="url(#breeze-static-checks-line-56)">[default: 3; 1<=x<=10]                                 [...] +</text><text class="breeze-static-checks-r5" x="0" y="1410.8" textLength="1464" clip-path="url(#breeze-static-checks-line-57)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1410.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-57)"> +</text><text class="breeze-static-checks-r5" x="0" y="1435.2" textLength="24.4" clip-path="url(#breeze-static-checks-line-58)">╭─</text><text class="breeze-static-checks-r5" x="24.4" y="1435.2" textLength="463.6" clip-path="url(#breeze-static-checks-line-58)"> Selecting files to run the checks on </text><text class="breeze-static-checks-r5" x="488" y="1435.2" textLength="951.6" clip-path="url(#breeze-static-checks-line-58)">──────────────────────── [...] +</text><text class="breeze-static-checks-r5" x="0" y="1459.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-59)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1459.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-59)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1459.6" textLength="61" clip-path="url(#breeze-static-checks-line-59)">-file</text><text class="breeze-static-checks-r6" x="256.2" y="1459.6" textLength="24.4" clip-path="url(# [...] +</text><text class="breeze-static-checks-r5" x="0" y="1484" textLength="12.2" clip-path="url(#breeze-static-checks-line-60)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1484" textLength="12.2" clip-path="url(#breeze-static-checks-line-60)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1484" textLength="48.8" clip-path="url(#breeze-static-checks-line-60)">-all</text><text class="breeze-static-checks-r4" x="85.4" y="1484" textLength="73.2" clip-path="url(#breeze-s [...] +</text><text class="breeze-static-checks-r5" x="0" y="1508.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-61)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1508.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-61)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1508.4" textLength="85.4" clip-path="url(#breeze-static-checks-line-61)">-commit</text><text class="breeze-static-checks-r4" x="122" y="1508.4" textLength="48.8" clip-path="url [...] +</text><text class="breeze-static-checks-r5" x="0" y="1532.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-62)">│</text><text class="breeze-static-checks-r1" x="305" y="1532.8" textLength="183" clip-path="url(#breeze-static-checks-line-62)">exclusive with </text><text class="breeze-static-checks-r4" x="488" y="1532.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-62)">-</text><text class="breeze-static-checks-r4" x="500.2" y="1532.8" textLength="61" [...] +</text><text class="breeze-static-checks-r5" x="0" y="1557.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-63)">│</text><text class="breeze-static-checks-r7" x="305" y="1557.2" textLength="1134.6" clip-path="url(#breeze-static-checks-line-63)">(TEXT)                                       [...] +</text><text class="breeze-static-checks-r5" x="0" y="1581.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-64)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1581.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-64)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1581.6" textLength="61" clip-path="url(#breeze-static-checks-line-64)">-last</text><text class="breeze-static-checks-r4" x="97.6" y="1581.6" textLength="85.4" clip-path="url(#b [...] +</text><text class="breeze-static-checks-r5" x="0" y="1606" textLength="12.2" clip-path="url(#breeze-static-checks-line-65)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1606" textLength="12.2" clip-path="url(#breeze-static-checks-line-65)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1606" textLength="61" clip-path="url(#breeze-static-checks-line-65)">-only</text><text class="breeze-static-checks-r4" x="97.6" y="1606" textLength="134.2" clip-path="url(#breeze-s [...] +</text><text class="breeze-static-checks-r5" x="0" y="1630.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-66)">│</text><text class="breeze-static-checks-r1" x="305" y="1630.4" textLength="1134.6" clip-path="url(#breeze-static-checks-line-66)">branch and HEAD of your branch.                              [...] +</text><text class="breeze-static-checks-r5" x="0" y="1654.8" textLength="1464" clip-path="url(#breeze-static-checks-line-67)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1654.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-67)"> +</text><text class="breeze-static-checks-r5" x="0" y="1679.2" textLength="24.4" clip-path="url(#breeze-static-checks-line-68)">╭─</text><text class="breeze-static-checks-r5" x="24.4" y="1679.2" textLength="463.6" clip-path="url(#breeze-static-checks-line-68)"> Building image before running checks </text><text class="breeze-static-checks-r5" x="488" y="1679.2" textLength="951.6" clip-path="url(#breeze-static-checks-line-68)">────────────────────────────────── [...] +</text><text class="breeze-static-checks-r5" x="0" y="1703.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-69)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1703.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-69)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1703.6" textLength="61" clip-path="url(#breeze-static-checks-line-69)">-skip</text><text class="breeze-static-checks-r4" x="97.6" y="1703.6" textLength="244" clip-path="url(#br [...] +</text><text class="breeze-static-checks-r5" x="0" y="1728" textLength="12.2" clip-path="url(#breeze-static-checks-line-70)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1728" textLength="12.2" clip-path="url(#breeze-static-checks-line-70)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1728" textLength="73.2" clip-path="url(#breeze-static-checks-line-70)">-force</text><text class="breeze-static-checks-r4" x="109.8" y="1728" textLength="73.2" clip-path="url(#breez [...] +</text><text class="breeze-static-checks-r5" x="0" y="1752.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-71)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1752.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-71)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1752.4" textLength="73.2" clip-path="url(#breeze-static-checks-line-71)">-image</text><text class="breeze-static-checks-r4" x="109.8" y="1752.4" textLength="48.8" clip-path="ur [...] +</text><text class="breeze-static-checks-r5" x="0" y="1776.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-72)">│</text><text class="breeze-static-checks-r7" x="414.8" y="1776.8" textLength="963.8" clip-path="url(#breeze-static-checks-line-72)">(TEXT)                                      [...] +</text><text class="breeze-static-checks-r5" x="0" y="1801.2" textLength="12.2" clip-path="url(#breeze-static-checks-line-73)">│</text><text class="breeze-static-checks-r5" x="414.8" y="1801.2" textLength="963.8" clip-path="url(#breeze-static-checks-line-73)">[default: latest]                                    [...] +</text><text class="breeze-static-checks-r5" x="0" y="1825.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-74)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1825.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-74)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1825.6" textLength="85.4" clip-path="url(#breeze-static-checks-line-74)">-github</text><text class="breeze-static-checks-r4" x="122" y="1825.6" textLength="134.2" clip-path="ur [...] +</text><text class="breeze-static-checks-r5" x="0" y="1850" textLength="12.2" clip-path="url(#breeze-static-checks-line-75)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1850" textLength="12.2" clip-path="url(#breeze-static-checks-line-75)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1850" textLength="97.6" clip-path="url(#breeze-static-checks-line-75)">-builder</text><text class="breeze-static-checks-r1" x="414.8" y="1850" textLength="756.4" clip-path="url(#br [...] +</text><text class="breeze-static-checks-r5" x="0" y="1874.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-76)">│</text><text class="breeze-static-checks-r5" x="414.8" y="1874.4" textLength="756.4" clip-path="url(#breeze-static-checks-line-76)">[default: autodetect]                                  &# [...] +</text><text class="breeze-static-checks-r5" x="0" y="1898.8" textLength="1464" clip-path="url(#breeze-static-checks-line-77)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="1898.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-77)"> +</text><text class="breeze-static-checks-r5" x="0" y="1923.2" textLength="24.4" clip-path="url(#breeze-static-checks-line-78)">╭─</text><text class="breeze-static-checks-r5" x="24.4" y="1923.2" textLength="195.2" clip-path="url(#breeze-static-checks-line-78)"> Common options </text><text class="breeze-static-checks-r5" x="219.6" y="1923.2" textLength="1220" clip-path="url(#breeze-static-checks-line-78)">────────────────────────────────────────────────────────────────────── [...] +</text><text class="breeze-static-checks-r5" x="0" y="1947.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-79)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1947.6" textLength="12.2" clip-path="url(#breeze-static-checks-line-79)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1947.6" textLength="48.8" clip-path="url(#breeze-static-checks-line-79)">-dry</text><text class="breeze-static-checks-r4" x="85.4" y="1947.6" textLength="48.8" clip-path="url(# [...] +</text><text class="breeze-static-checks-r5" x="0" y="1972" textLength="12.2" clip-path="url(#breeze-static-checks-line-80)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1972" textLength="12.2" clip-path="url(#breeze-static-checks-line-80)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1972" textLength="97.6" clip-path="url(#breeze-static-checks-line-80)">-verbose</text><text class="breeze-static-checks-r6" x="158.6" y="1972" textLength="24.4" clip-path="url(#bre [...] +</text><text class="breeze-static-checks-r5" x="0" y="1996.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-81)">│</text><text class="breeze-static-checks-r4" x="24.4" y="1996.4" textLength="12.2" clip-path="url(#breeze-static-checks-line-81)">-</text><text class="breeze-static-checks-r4" x="36.6" y="1996.4" textLength="61" clip-path="url(#breeze-static-checks-line-81)">-help</text><text class="breeze-static-checks-r6" x="158.6" y="1996.4" textLength="24.4" clip-path="url(# [...] +</text><text class="breeze-static-checks-r5" x="0" y="2020.8" textLength="1464" clip-path="url(#breeze-static-checks-line-82)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-static-checks-r1" x="1464" y="2020.8" textLength="12.2" clip-path="url(#breeze-static-checks-line-82)"> </text> </g> </g> diff --git a/images/breeze/output_static-checks.txt b/images/breeze/output_static-checks.txt index 5b32905ea3..00157e7cd8 100644 --- a/images/breeze/output_static-checks.txt +++ b/images/breeze/output_static-checks.txt @@ -1 +1 @@ -1197108ac5d3038067e599375d5130dd +65040ded8dc5fc072fa49219f88feb36 diff --git a/scripts/ci/docker-compose/integration-cassandra.yml b/scripts/ci/docker-compose/integration-cassandra.yml index 10569ee8ed..665637217c 100644 --- a/scripts/ci/docker-compose/integration-cassandra.yml +++ b/scripts/ci/docker-compose/integration-cassandra.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: cassandra: image: cassandra:3.0 + labels: + breeze.description: "Integration required for Cassandra hooks." environment: HEAP_NEWSIZE: 128M MAX_HEAP_SIZE: 256M diff --git a/scripts/ci/docker-compose/integration-celery.yml b/scripts/ci/docker-compose/integration-celery.yml index 918c9cdcf4..11383198b8 100644 --- a/scripts/ci/docker-compose/integration-celery.yml +++ b/scripts/ci/docker-compose/integration-celery.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: rabbitmq: image: rabbitmq:3.7 + labels: + breeze.description: "Integration required for Celery executor tests." volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source - rabbitmq-db-volume:/var/lib/rabbitmq diff --git a/scripts/ci/docker-compose/integration-kafka.yml b/scripts/ci/docker-compose/integration-kafka.yml index cc0daca1d7..1b1377ac4d 100644 --- a/scripts/ci/docker-compose/integration-kafka.yml +++ b/scripts/ci/docker-compose/integration-kafka.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: broker: image: confluentinc/cp-kafka:7.3.0 + labels: + breeze.description: "Integration required for Kafka hooks." hostname: broker container_name: broker ports: diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml index ec27d952cd..8f32d65074 100644 --- a/scripts/ci/docker-compose/integration-kerberos.yml +++ b/scripts/ci/docker-compose/integration-kerberos.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: kdc-server-example-com: image: ghcr.io/apache/airflow-krb5-kdc-server:2021.07.04 + labels: + breeze.description: "Integration that provides Kerberos authentication." hostname: krb5-kdc-server-example-com domainname: example.com networks: diff --git a/scripts/ci/docker-compose/integration-mongo.yml b/scripts/ci/docker-compose/integration-mongo.yml index 33e6b66b55..207e727f1a 100644 --- a/scripts/ci/docker-compose/integration-mongo.yml +++ b/scripts/ci/docker-compose/integration-mongo.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: mongo: image: mongo:3 + labels: + breeze.description: "Integration required for MongoDB hooks." volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source - mongo-db-volume:/data/db diff --git a/scripts/ci/docker-compose/integration-openlineage.yml b/scripts/ci/docker-compose/integration-openlineage.yml index fdacdec52d..8d6281de04 100644 --- a/scripts/ci/docker-compose/integration-openlineage.yml +++ b/scripts/ci/docker-compose/integration-openlineage.yml @@ -15,10 +15,12 @@ # specific language governing permissions and limitations # under the License. --- -version: "3.7" +version: "3.8" services: marquez: image: marquezproject/marquez:0.40.0 + labels: + breeze.description: "Integration required for Openlineage hooks." environment: - POSTGRES_USER=marquez - POSTGRES_PASSWORD=marquez diff --git a/scripts/ci/docker-compose/integration-otel.yml b/scripts/ci/docker-compose/integration-otel.yml index ccc2be22fe..3df68a2cef 100644 --- a/scripts/ci/docker-compose/integration-otel.yml +++ b/scripts/ci/docker-compose/integration-otel.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: otel-collector: image: otel/opentelemetry-collector-contrib:0.70.0 + labels: + breeze.description: "Integration required for OTEL/opentelemetry hooks." container_name: "breeze-otel-collector" command: [--config=/etc/otel-collector-config.yml] volumes: diff --git a/scripts/ci/docker-compose/integration-pinot.yml b/scripts/ci/docker-compose/integration-pinot.yml index 510f6b5cea..b0d1964eab 100644 --- a/scripts/ci/docker-compose/integration-pinot.yml +++ b/scripts/ci/docker-compose/integration-pinot.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: pinot: image: apachepinot/pinot:0.8.0 + labels: + breeze.description: "Integration required for Apache Pinot hooks." ports: - "9080:9080" volumes: diff --git a/scripts/ci/docker-compose/integration-statsd.yml b/scripts/ci/docker-compose/integration-statsd.yml index f4e6eadb4c..cfe3b75f7e 100644 --- a/scripts/ci/docker-compose/integration-statsd.yml +++ b/scripts/ci/docker-compose/integration-statsd.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: statsd-exporter: image: apache/airflow:airflow-statsd-exporter-2020.09.05-v0.17.0 + labels: + breeze.description: "Integration required for Satsd hooks." container_name: "breeze-statsd-exporter" ports: - "9125:9125" diff --git a/scripts/ci/docker-compose/integration-trino.yml b/scripts/ci/docker-compose/integration-trino.yml index 626769b730..3c57f62224 100644 --- a/scripts/ci/docker-compose/integration-trino.yml +++ b/scripts/ci/docker-compose/integration-trino.yml @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. --- +version: "3.8" services: trino: image: ghcr.io/apache/airflow-trino:359-2021.07.04 + labels: + breeze.description: "Integration required for Trino hooks." hostname: trino domainname: example.com diff --git a/scripts/ci/pre_commit/pre_commit_check_integrations_list.py b/scripts/ci/pre_commit/pre_commit_check_integrations_list.py new file mode 100755 index 0000000000..f965ec3ea4 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_check_integrations_list.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Module to check integration tests are listed in documentation. + +Compare the contents of the integrations table and the docker-compose +integration files, if there is a mismatch, the table is generated. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any + +import yaml + +# make sure common_precommit_utils is imported +sys.path.insert(0, str(Path(__file__).parent.resolve())) +from common_precommit_utils import ( + AIRFLOW_SOURCES_ROOT_PATH, + console, + insert_documentation, +) +from tabulate import tabulate + +DOCUMENTATION_PATH = AIRFLOW_SOURCES_ROOT_PATH / "TESTING.rst" +INTEGRATION_TESTS_PATH = AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "docker-compose" +INTEGRATION_TEST_PREFIX = "integration-*.yml" +DOCS_MARKER_START = ".. BEGIN AUTO-GENERATED INTEGRATION LIST" +DOCS_MARKER_END = ".. END AUTO-GENERATED INTEGRATION LIST" +_LIST_MATCH = r"\|[^|\n]+" + + +def get_ci_integrations( + tests_path: Path = INTEGRATION_TESTS_PATH, + integration_prefix: str = INTEGRATION_TEST_PREFIX, +) -> dict[str, Path]: + """Get list of integrations from matching filenames.""" + if not tests_path.is_dir() and tests_path.exists(): + console.print(f"[red]Bad tests path: {tests_path}. [/]") + sys.exit(1) + + integrations_files = [_i for _i in tests_path.glob(integration_prefix)] + + if len(integrations_files) == 0: + console.print( + f"[red]No integrations found." + f"Pattern '{integration_prefix}' did not match any files under {tests_path}. [/]" + ) + sys.exit(1) + + # parse into list of ids + integrations = {} + for _i in integrations_files: + try: + _key = _i.stem.split("-")[1] + integrations[_key] = _i + except IndexError: + console.print(f"[red]Tried to parse {_i.stem}, but did not contain '-' separator. [/]") + continue + + return integrations + + +def get_docs_integrations(docs_path: Path = DOCUMENTATION_PATH): + """Get integrations listed in docs.""" + table_lines = [] + _list_start_line = None + with open(docs_path, encoding="utf8") as f: + for line_n, line in enumerate(f): + if DOCS_MARKER_END in line: + break + if DOCS_MARKER_START in line: + _list_start_line = line_n + if _list_start_line is None: + continue + if line_n > _list_start_line: + table_lines.append(line) + + if len(table_lines) == 0: + console.print("[red]No integrations table in docs.[/]") + sys.exit(1) + + table_cells = [] + for line in table_lines: + m = re.findall(_LIST_MATCH, line) + if len(m) == 0: + continue + table_cells.append(m[0].strip("|").strip()) + + def _list_matcher(j): + """Filter callable to exclude header and empty cells.""" + if len(j) == 0: + return False + elif j in ["Description", "Identifier"]: + return False + else: + return True + + table_cells = list(filter(_list_matcher, table_cells)) + return table_cells + + +def update_integration_tests_array(contents: dict[str, list[str]]): + """Generate docs table.""" + rows = [] + sorted_contents = dict(sorted(contents.items())) + for integration, description in sorted_contents.items(): + formatted_hook_description = ( + description[0] if len(description) == 1 else "* " + "\n* ".join(description) + ) + rows.append((integration, formatted_hook_description)) + formatted_table = "\n" + tabulate(rows, tablefmt="grid", headers=("Identifier", "Description")) + "\n\n" + insert_documentation( + file_path=AIRFLOW_SOURCES_ROOT_PATH / "TESTING.rst", + content=formatted_table.splitlines(keepends=True), + header=DOCS_MARKER_START, + footer=DOCS_MARKER_END, + ) + + +def print_diff(source, target, msg): + difference = source - target + if difference: + console.print(msg) + for i in difference: + console.print(f"[red]\t- {i}[/]") + return list(difference) + + +def _get_breeze_description(parsed_compose: dict[str, Any], label_key: str = "breeze.description"): + """Extract all breeze.description labels per image.""" + image_label_map = {} + # possible key error handled outside + for _img_name, img in parsed_compose["services"].items(): + try: + for _label_name, label in img["labels"].items(): + if _label_name == label_key: + image_label_map[_img_name] = label + except KeyError: + # service has no 'lables' entry + continue + return image_label_map + + +def get_integration_descriptions(integrations: dict[str, Path]) -> dict[str, list[Any]]: + """Pull breeze description from docker-compose files.""" + table = {} + for integration, path in integrations.items(): + with open(path) as f: + _compose = yaml.safe_load(f) + + try: + _labels = _get_breeze_description(_compose) + except KeyError: + console.print(f"[red]No 'services' entry in compose file {path}.[/]") + sys.exit(1) + table[integration] = list(_labels.values()) + return table + + +def main(): + docs_integrations = get_docs_integrations() + ci_integrations = get_ci_integrations() + + if len(ci_integrations) == 0: + console.print("[red]No integrations found.[/]") + sys.exit(1) + + _ci_items = set(ci_integrations) + _docs_items = set(docs_integrations) + diff = [] + diff.append(print_diff(_ci_items, _docs_items, "[red]Found in ci files, but not in docs: [/]")) + diff.append(print_diff(_docs_items, _ci_items, "[red]Found in docs, but not in ci files: [/]")) + if diff: + console.print( + "[yellow]Regenerating documentation table. Don't forget to review and commit possible changes.[/]" + ) + + table_contents = get_integration_descriptions(ci_integrations) + update_integration_tests_array(table_contents) + + +if __name__ == "__main__": + main()
