This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7e5c827472 Improve breeze command image generation and consistency
check (#35169)
7e5c827472 is described below
commit 7e5c827472732c6dba2bb55a80b21617c59d88a5
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Oct 25 11:47:17 2023 +0200
Improve breeze command image generation and consistency check (#35169)
* Improve breeze command image generation and consistency check
There is no more need to regenerate the images manually via
`breeze setup regenerate-command-images`. Also check for
consistency of command configuration vs command parameters in Breeze
is now done in pre-commit. This means that anyone making changes
to Breeze and having pre-commit installed will have two steps less
to keep the --help images generated and to keep consistency of
flags in breeze commands.
* Update .pre-commit-config.yaml
---
.github/workflows/ci.yml | 1 -
.../commands/setup_commands_config.py | 1 +
images/breeze/output-commands-hash.txt | 2 +-
images/breeze/output_setup.svg | 28 ++++++---------
.../ci/pre_commit/pre_commit_breeze_cmd_line.py | 41 +++++++++++++++++++---
5 files changed, 48 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 779b3a46bb..3dd00e2094 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -455,7 +455,6 @@ jobs:
echo "Please run 'breeze setup version' and commit the changes." &&
false)
- run: python -m pytest -n auto --color=yes
working-directory: ./dev/breeze/
- - run: breeze setup check-all-params-in-groups
tests-www:
timeout-minutes: 10
diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py
index 463305a716..1da86bb56a 100644
--- a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py
@@ -23,6 +23,7 @@ SETUP_COMMANDS: dict[str, str | list[str]] = {
"self-upgrade",
"cleanup",
"config",
+ "check-all-params-in-groups",
"regenerate-command-images",
"command-hash-export",
"version",
diff --git a/images/breeze/output-commands-hash.txt
b/images/breeze/output-commands-hash.txt
index 790bab4e7a..283ca94693 100644
--- a/images/breeze/output-commands-hash.txt
+++ b/images/breeze/output-commands-hash.txt
@@ -62,7 +62,7 @@ setup:config:0b95b129703e11a99b8ddd05a0dd6b35
setup:regenerate-command-images:ea2fba3440bd4e84311a53abe6e8dc56
setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b
setup:version:be116d90a21c2afe01087f7609774e1e
-setup:000b1d18baa7f1a740a1afb6a6bfe246
+setup:9dff88972f7c2ba4da2ad47f77ba2d06
shell:619e9de6bb9e6eb26acab06c8aedb183
start-airflow:c238330495a57851030415e34042173b
static-checks:19926b8fcea5784b28d4a0d99865363c
diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg
index 0864a65f00..9b165e4524 100644
--- a/images/breeze/output_setup.svg
+++ b/images/breeze/output_setup.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 489.2"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 440.4"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -42,7 +42,7 @@
<defs>
<clipPath id="breeze-setup-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="438.2" />
+ <rect x="0" y="0" width="1463.0" height="389.4" />
</clipPath>
<clipPath id="breeze-setup-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -89,15 +89,9 @@
<clipPath id="breeze-setup-line-14">
<rect x="0" y="343.1" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="breeze-setup-line-15">
- <rect x="0" y="367.5" width="1464" height="24.65"/>
- </clipPath>
-<clipPath id="breeze-setup-line-16">
- <rect x="0" y="391.9" width="1464" height="24.65"/>
- </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="487.2" rx="8"/><text
class="breeze-setup-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: setup</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="438.4" rx="8"/><text
class="breeze-setup-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: setup</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -116,15 +110,13 @@
</text><text class="breeze-setup-r5" x="0" y="166.4" textLength="12.2"
clip-path="url(#breeze-setup-line-6)">│</text><text class="breeze-setup-r4"
x="24.4" y="166.4" textLength="12.2"
clip-path="url(#breeze-setup-line-6)">-</text><text class="breeze-setup-r4"
x="36.6" y="166.4" textLength="61"
clip-path="url(#breeze-setup-line-6)">-help</text><text class="breeze-setup-r6"
x="122" y="166.4" textLength="24.4"
clip-path="url(#breeze-setup-line-6)">-h</text><text class="breeze-setup-r1"
x="1 [...]
</text><text class="breeze-setup-r5" x="0" y="190.8" textLength="1464"
clip-path="url(#breeze-setup-line-7)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-setup-r1" x="1464" y="190.8" textLength="12.2"
clip-path="url(#breeze-setup-line-7)">
</text><text class="breeze-setup-r5" x="0" y="215.2" textLength="24.4"
clip-path="url(#breeze-setup-line-8)">╭─</text><text class="breeze-setup-r5"
x="24.4" y="215.2" textLength="85.4"
clip-path="url(#breeze-setup-line-8)"> Setup </text><text
class="breeze-setup-r5" x="109.8" y="215.2" textLength="1329.8"
clip-path="url(#breeze-setup-line-8)">─────────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text
class="breeze-s [...]
-</text><text class="breeze-setup-r5" x="0" y="239.6" textLength="12.2"
clip-path="url(#breeze-setup-line-9)">│</text><text class="breeze-setup-r4"
x="24.4" y="239.6" textLength="390.4"
clip-path="url(#breeze-setup-line-9)">autocomplete                    </text><text
class="breeze-setup-r1" x="439.2" y="239.6" textLength="1000.4"
clip-path="url(#breeze-setup-line-9)">Enables autocompl [...]
-</text><text class="breeze-setup-r5" x="0" y="264" textLength="12.2"
clip-path="url(#breeze-setup-line-10)">│</text><text class="breeze-setup-r4"
x="24.4" y="264" textLength="390.4"
clip-path="url(#breeze-setup-line-10)">self-upgrade                    </text><text
class="breeze-setup-r1" x="439.2" y="264" textLength="1000.4"
clip-path="url(#breeze-setup-line-10)">Self upgrade Br [...]
-</text><text class="breeze-setup-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-setup-line-11)">│</text><text class="breeze-setup-r4"
x="24.4" y="288.4" textLength="390.4"
clip-path="url(#breeze-setup-line-11)">config                          </text><text
class="breeze-setup-r1" x="439.2" y="288.4" textLength="1000.4"
clip-path="url(#breeze-setup [...]
-</text><text class="breeze-setup-r5" x="0" y="312.8" textLength="12.2"
clip-path="url(#breeze-setup-line-12)">│</text><text class="breeze-setup-r4"
x="24.4" y="312.8" textLength="390.4"
clip-path="url(#breeze-setup-line-12)">regenerate-command-images       </text><text
class="breeze-setup-r1" x="439.2" y="312.8" textLength="1000.4"
clip-path="url(#breeze-setup-line-12)">Regenerate breeze command images.      
[...]
-</text><text class="breeze-setup-r5" x="0" y="337.2" textLength="12.2"
clip-path="url(#breeze-setup-line-13)">│</text><text class="breeze-setup-r4"
x="24.4" y="337.2" textLength="390.4"
clip-path="url(#breeze-setup-line-13)">version                         </text><text
class="breeze-setup-r1" x="439.2" y="337.2" textLength="1000.4"
clip-path="url(#breeze-setup-line [...]
-</text><text class="breeze-setup-r5" x="0" y="361.6" textLength="1464"
clip-path="url(#breeze-setup-line-14)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-setup-r1" x="1464" y="361.6" textLength="12.2"
clip-path="url(#breeze-setup-line-14)">
-</text><text class="breeze-setup-r5" x="0" y="386" textLength="24.4"
clip-path="url(#breeze-setup-line-15)">╭─</text><text class="breeze-setup-r5"
x="24.4" y="386" textLength="122"
clip-path="url(#breeze-setup-line-15)"> Commands </text><text
class="breeze-setup-r5" x="146.4" y="386" textLength="1293.2"
clip-path="url(#breeze-setup-line-15)">──────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text
class="breeze-setup [...]
-</text><text class="breeze-setup-r5" x="0" y="410.4" textLength="12.2"
clip-path="url(#breeze-setup-line-16)">│</text><text class="breeze-setup-r4"
x="24.4" y="410.4" textLength="524.6"
clip-path="url(#breeze-setup-line-16)">check-all-params-in-groups                 </text><text
class="breeze-setup-r1" x="573.4" y="410.4" textLength="866.2"
clip-path="url(#breeze-setup-line-16)">Check that all [...]
-</text><text class="breeze-setup-r5" x="0" y="434.8" textLength="1464"
clip-path="url(#breeze-setup-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-setup-r1" x="1464" y="434.8" textLength="12.2"
clip-path="url(#breeze-setup-line-17)">
+</text><text class="breeze-setup-r5" x="0" y="239.6" textLength="12.2"
clip-path="url(#breeze-setup-line-9)">│</text><text class="breeze-setup-r4"
x="24.4" y="239.6" textLength="402.6"
clip-path="url(#breeze-setup-line-9)">autocomplete                     </text><text
class="breeze-setup-r1" x="451.4" y="239.6" textLength="988.2"
clip-path="url(#breeze-setup-line-9)">Enables auto [...]
+</text><text class="breeze-setup-r5" x="0" y="264" textLength="12.2"
clip-path="url(#breeze-setup-line-10)">│</text><text class="breeze-setup-r4"
x="24.4" y="264" textLength="402.6"
clip-path="url(#breeze-setup-line-10)">self-upgrade                     </text><text
class="breeze-setup-r1" x="451.4" y="264" textLength="988.2"
clip-path="url(#breeze-setup-line-10)">Self upgrade [...]
+</text><text class="breeze-setup-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-setup-line-11)">│</text><text class="breeze-setup-r4"
x="24.4" y="288.4" textLength="402.6"
clip-path="url(#breeze-setup-line-11)">config                           </text><text
class="breeze-setup-r1" x="451.4" y="288.4" textLength="988.2"
clip-path="url(#breeze- [...]
+</text><text class="breeze-setup-r5" x="0" y="312.8" textLength="12.2"
clip-path="url(#breeze-setup-line-12)">│</text><text class="breeze-setup-r4"
x="24.4" y="312.8" textLength="402.6"
clip-path="url(#breeze-setup-line-12)">check-all-params-in-groups       </text><text
class="breeze-setup-r1" x="451.4" y="312.8" textLength="988.2"
clip-path="url(#breeze-setup-line-12)">Check that all parameters are put in groups. 
[...]
+</text><text class="breeze-setup-r5" x="0" y="337.2" textLength="12.2"
clip-path="url(#breeze-setup-line-13)">│</text><text class="breeze-setup-r4"
x="24.4" y="337.2" textLength="402.6"
clip-path="url(#breeze-setup-line-13)">regenerate-command-images        </text><text
class="breeze-setup-r1" x="451.4" y="337.2" textLength="988.2"
clip-path="url(#breeze-setup-line-13)">Regenerate breeze command images.     &
[...]
+</text><text class="breeze-setup-r5" x="0" y="361.6" textLength="12.2"
clip-path="url(#breeze-setup-line-14)">│</text><text class="breeze-setup-r4"
x="24.4" y="361.6" textLength="402.6"
clip-path="url(#breeze-setup-line-14)">version                          </text><text
class="breeze-setup-r1" x="451.4" y="361.6" textLength="988.2"
clip-path="url(#breeze-setup [...]
+</text><text class="breeze-setup-r5" x="0" y="386" textLength="1464"
clip-path="url(#breeze-setup-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-setup-r1" x="1464" y="386" textLength="12.2"
clip-path="url(#breeze-setup-line-15)">
</text>
</g>
</g>
diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
index 198488a119..63c9429189 100755
--- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
@@ -19,6 +19,8 @@
from __future__ import annotations
import os
+import shutil
+import subprocess
import sys
from pathlib import Path
from subprocess import call
@@ -76,11 +78,40 @@ def is_regeneration_needed() -> bool:
if __name__ == "__main__":
+ return_code = 0
verify_all_commands_described_in_docs()
if is_regeneration_needed():
- console.print("\n[bright_blue]Some of the commands changed since last
time images were generated.\n")
- console.print(
- "\n[red]Image generation is needed. Please run this command:\n\n"
- "[magenta]breeze setup regenerate-command-images\n"
+ return_code = 1
+ if shutil.which("breeze") is None:
+ console.print("\n[red]Breeze command configuration has changed.\n")
+ console.print(
+ "[yellow]The `breeze` command is not on path. "
+ "Skipping regeneration of images and consistency check."
+ )
+ console.print(
+ "\n[bright_blue]Some of the commands changed since last time
images were generated.\n"
+ )
+ console.print("\n[yellow]You should install it and run those
commands manually:\n")
+ console.print("\n[magenta]breeze setup regenerate-command
images\n")
+ console.print("\n[magenta]breeze setup
check-all-params-in-groups\n")
+ sys.exit(return_code)
+ res = subprocess.run(
+ ["breeze", "setup", "regenerate-command-images"],
+ check=False,
)
- sys.exit(1)
+ if res.returncode != 0:
+ console.print("\n[red]Breeze command configuration has changed.\n")
+ console.print("\n[bright_blue]Images have been regenerated.\n")
+ console.print("\n[bright_blue]You might want to run it
manually:\n")
+ console.print("\n[magenta]]breeze setup regenerate-command
images\n")
+ res = subprocess.run(
+ ["breeze", "setup", "check-all-params-in-groups"],
+ check=False,
+ )
+ if res.returncode != 0:
+ return_code = 1
+ console.print("\n[red]Breeze command configuration has changed.\n")
+ console.print("\n[yellow]Please fix it in the appropriate
command_*_config.py file\n")
+ console.print("\n[bright_blue]You can run consistency check manually
by running:\n")
+ console.print("\nbreeze setup check-all-params-in-groups\n")
+ sys.exit(return_code)
