potiuk commented on code in PR #35830:
URL: https://github.com/apache/airflow/pull/35830#discussion_r1405499699
##########
scripts/ci/pre_commit/common_precommit_utils.py:
##########
@@ -71,3 +80,74 @@ def get_directory_hash(directory: Path, skip_path_regexp:
str | None = None) ->
if file.is_file() and not file.name.startswith("."):
sha.update(file.read_bytes())
return sha.hexdigest()
+
+
+def initialize_breeze_precommit(name: str, file: str):
+ if name not in ("__main__", "__mp_main__"):
+ raise SystemExit(
+ "This file is intended to be executed as an executable program.
You cannot use it as a module."
+ f"To run this script, run the ./{file} command"
+ )
+
+ if os.environ.get("SKIP_BREEZE_PRE_COMMITS"):
+ console.print("[yellow]Skipping breeze pre-commit as
SKIP_BREEZE_PRE_COMMIT is set")
+ sys.exit(1)
+ if shutil.which("breeze") is None:
+ console.print(
+ "[red]The `breeze` command is not on path.[/]\n\n"
+ "[yellow]Please install breeze with `pipx install -e ./dev/breeze`
from Airflow sources "
+ "and make sure you run `pipx ensurepath`[/]\n\n"
+ "[bright_blue]You can also set SKIP_BREEZE_PRE_COMMITS env
variable to non-empty "
+ "value to skip all breeze tests."
+ )
+ sys.exit(1)
+
+
+def run_command_via_breeze_shell(
+ cmd: list[str],
+ python_version: str = DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ backend: str = "none",
+ executor: str = "SequentialExecutor",
+ extra_env: dict[str, str] | None = None,
+ project_name: str = "pre-commit",
+ skip_environment_initialization: bool = True,
+ **other_popen_kwargs,
+) -> subprocess.CompletedProcess:
+ extra_env = extra_env or {}
+ subprocess_cmd: list[str] = [
+ "breeze",
+ "shell",
+ "--python",
+ python_version,
+ "--backend",
+ backend,
+ "--executor",
+ executor,
+ "--quiet",
+ "--skip-image-upgrade-check",
+ "--tty",
+ "disabled",
+ ]
+ if skip_environment_initialization:
+ subprocess_cmd.append("--skip-environment-initialization")
+ if project_name:
+ subprocess_cmd.extend(["--project-name", project_name])
+ subprocess_cmd.append(" ".join([shlex.quote(arg) for arg in cmd]))
+ if os.environ.get("VERBOSE_COMMANDS"):
+ console.print(
+ f"[magenta]Running command: {' '.join([shlex.quote(item) for item
in subprocess_cmd])}[/]"
+ )
+ return subprocess.run(
Review Comment:
Oh yeah... Actually I got it in now in my local PR:
<img width="1556" alt="Screenshot 2023-11-26 at 23 37 22"
src="https://github.com/apache/airflow/assets/595491/bcc3a9eb-89e3-455d-93bd-ef51fc3c1adf";>
Unfortunattely `mypy` checks are not fully deterministic. Depending onn the
set of files passed to mypy, mypy sometimes produces different results when
some "small" things change. This is because of Python's lack of full
static-typing and dynamic Python nature. MyPy tries to figure out which types
are there by looking into other Python code and tries to look at the ways how
things are used/initialized and **sometimes** it figures out things wrongly. We
cannot do much about it - other than periodically updating to newer versions of
`mypy` which will get - hopefully - better and better in this
heuristics/guessing.
##########
scripts/ci/pre_commit/common_precommit_utils.py:
##########
@@ -71,3 +80,74 @@ def get_directory_hash(directory: Path, skip_path_regexp:
str | None = None) ->
if file.is_file() and not file.name.startswith("."):
sha.update(file.read_bytes())
return sha.hexdigest()
+
+
+def initialize_breeze_precommit(name: str, file: str):
+ if name not in ("__main__", "__mp_main__"):
+ raise SystemExit(
+ "This file is intended to be executed as an executable program.
You cannot use it as a module."
+ f"To run this script, run the ./{file} command"
+ )
+
+ if os.environ.get("SKIP_BREEZE_PRE_COMMITS"):
+ console.print("[yellow]Skipping breeze pre-commit as
SKIP_BREEZE_PRE_COMMIT is set")
+ sys.exit(1)
+ if shutil.which("breeze") is None:
+ console.print(
+ "[red]The `breeze` command is not on path.[/]\n\n"
+ "[yellow]Please install breeze with `pipx install -e ./dev/breeze`
from Airflow sources "
+ "and make sure you run `pipx ensurepath`[/]\n\n"
+ "[bright_blue]You can also set SKIP_BREEZE_PRE_COMMITS env
variable to non-empty "
+ "value to skip all breeze tests."
+ )
+ sys.exit(1)
+
+
+def run_command_via_breeze_shell(
+ cmd: list[str],
+ python_version: str = DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ backend: str = "none",
+ executor: str = "SequentialExecutor",
+ extra_env: dict[str, str] | None = None,
+ project_name: str = "pre-commit",
+ skip_environment_initialization: bool = True,
+ **other_popen_kwargs,
+) -> subprocess.CompletedProcess:
+ extra_env = extra_env or {}
+ subprocess_cmd: list[str] = [
+ "breeze",
+ "shell",
+ "--python",
+ python_version,
+ "--backend",
+ backend,
+ "--executor",
+ executor,
+ "--quiet",
+ "--skip-image-upgrade-check",
+ "--tty",
+ "disabled",
+ ]
+ if skip_environment_initialization:
+ subprocess_cmd.append("--skip-environment-initialization")
+ if project_name:
+ subprocess_cmd.extend(["--project-name", project_name])
+ subprocess_cmd.append(" ".join([shlex.quote(arg) for arg in cmd]))
+ if os.environ.get("VERBOSE_COMMANDS"):
+ console.print(
+ f"[magenta]Running command: {' '.join([shlex.quote(item) for item
in subprocess_cmd])}[/]"
+ )
+ return subprocess.run(
Review Comment:
Oh yeah... Actually I got it in now in my local (separated) PR:
<img width="1556" alt="Screenshot 2023-11-26 at 23 37 22"
src="https://github.com/apache/airflow/assets/595491/bcc3a9eb-89e3-455d-93bd-ef51fc3c1adf";>
Unfortunattely `mypy` checks are not fully deterministic. Depending onn the
set of files passed to mypy, mypy sometimes produces different results when
some "small" things change. This is because of Python's lack of full
static-typing and dynamic Python nature. MyPy tries to figure out which types
are there by looking into other Python code and tries to look at the ways how
things are used/initialized and **sometimes** it figures out things wrongly. We
cannot do much about it - other than periodically updating to newer versions of
`mypy` which will get - hopefully - better and better in this
heuristics/guessing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
