This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8a67e15f1f Finish refactoring of breeze-dependent pre-commits to use
breeze shell (#35830)
8a67e15f1f is described below
commit 8a67e15f1f5e5d92e2cf7ec0e03ceca21155300c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Nov 27 23:28:43 2023 +0100
Finish refactoring of breeze-dependent pre-commits to use breeze shell
(#35830)
Some of our pre-commits were using code from Breeze in rather complex
way - by inserting PYTHONPATH and importing code from there. That was
complex and brittle and with recent changes of ShelParam #35801 those
precommits required more and more dependencies to be added to their
pre-commit virtualenvs.
The reason that it was done this way was the assumption that someone
might want to run pre-commits locally without having breeze installed,
but this assumption and use case is rather unlikely, becasue breeze
becomes more and more useful and used so we can safely assume that
anyone who wants to do pre-commits will also have breeze installed and
on path. And anyway to run those pre-commits you need to have breeze
CI image pulled and built, so you should generally have breeze to run
them.
This PR finishes the series of PRs implementing the refactor and
completes the refactor.
---
.pre-commit-config.yaml | 6 +--
dev/breeze/src/airflow_breeze/utils/run_utils.py | 24 ----------
.../pre_commit/pre_commit_migration_reference.py | 53 +++++-----------------
scripts/in_container/run_migration_reference.py | 8 ++++
4 files changed, 23 insertions(+), 68 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5b87ecdb70..77b8b60abc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1050,8 +1050,8 @@ repos:
files: ^docs/.*\.py$
exclude: ^docs/rtd-deprecation
require_serial: true
- additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml',
'jsonschema', 'filelock', 'markdown-it-py']
- - id: check-provider-yaml-valid
+ additional_dependencies: ['rich>=12.4.4']
+ - id: check-provider-yaml-valid
name: Validate provider.yaml files
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
language: python
@@ -1064,7 +1064,7 @@ repos:
entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.py
pass_filenames: false
files:
^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
- additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml',
'jsonschema', 'filelock', 'markdown-it-py']
+ additional_dependencies: ['rich>=12.4.4']
- id: update-er-diagram
name: Update ER diagram
language: python
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py
b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 874ae954d0..0707cdad11 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -33,8 +33,6 @@ from typing import Mapping, Union
from rich.markup import escape
-from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
-from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import Output, get_console
from airflow_breeze.utils.path_utils import (
@@ -382,28 +380,6 @@ def check_if_image_exists(image: str) -> bool:
return cmd_result.returncode == 0
-def get_ci_image_for_pre_commits() -> str:
- github_repository = os.environ.get("GITHUB_REPOSITORY",
APACHE_AIRFLOW_GITHUB_REPOSITORY)
- python_version = "3.8"
- airflow_image =
f"ghcr.io/{github_repository}/{AIRFLOW_BRANCH}/ci/python{python_version}"
- skip_image_pre_commits = os.environ.get("SKIP_IMAGE_PRE_COMMITS", "false")
- if skip_image_pre_commits[0].lower() == "t":
- get_console().print(
- f"[info]Skipping image check as SKIP_IMAGE_PRE_COMMITS is set to
{skip_image_pre_commits}[/]"
- )
- sys.exit(0)
- if not check_if_image_exists(
- image=airflow_image,
- ):
- get_console().print(f"[red]The image {airflow_image} is not
available.[/]\n")
- get_console().print(
- f"\n[yellow]Please run this to fix it:[/]\n\n"
- f"breeze ci-image build --python {python_version}\n\n"
- )
- sys.exit(1)
- return airflow_image
-
-
def _run_compile_internally(command_to_execute: list[str], dev: bool) ->
RunCommandResult:
from filelock import SoftFileLock, Timeout
diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py
b/scripts/ci/pre_commit/pre_commit_migration_reference.py
index 9dec2e33d9..34d3a94c6a 100755
--- a/scripts/ci/pre_commit/pre_commit_migration_reference.py
+++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py
@@ -17,50 +17,21 @@
# under the License.
from __future__ import annotations
-import os
import sys
from pathlib import Path
-if __name__ not in ("__main__", "__mp_main__"):
- raise SystemExit(
- "This file is intended to be executed as an executable program. You
cannot use it as a module."
- f"To run this script, run the ./{__file__} command"
- )
-
-AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
-GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "apache/airflow")
-os.environ["SKIP_GROUP_OUTPUT"] = "true"
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+from common_precommit_utils import console, initialize_breeze_precommit,
run_command_via_breeze_shell
-if __name__ == "__main__":
- sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
- from airflow_breeze.global_constants import
DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_SELECTED
- from airflow_breeze.params.shell_params import ShellParams
- from airflow_breeze.utils.console import get_console
- from airflow_breeze.utils.docker_command_utils import
get_extra_docker_flags
- from airflow_breeze.utils.run_utils import get_ci_image_for_pre_commits,
run_command
+initialize_breeze_precommit(__name__, __file__)
- shell_params = ShellParams(python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
db_reset=True, backend="none")
- airflow_image = get_ci_image_for_pre_commits()
- cmd_result = run_command(
- [
- "docker",
- "run",
- "-t",
- *get_extra_docker_flags(mount_sources=MOUNT_SELECTED),
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "--pull",
- "never",
- airflow_image,
- "-c",
- "python3
/opt/airflow/scripts/in_container/run_migration_reference.py",
- ],
- check=False,
- env=shell_params.env_variables_for_docker_commands,
+cmd_result = run_command_via_breeze_shell(
+ ["python3",
"/opt/airflow/scripts/in_container/run_migration_reference.py"],
+ backend="sqlite",
+)
+if cmd_result.returncode != 0:
+ console.print(
+ "[warning]\nIf you see strange stacktraces above, "
+ "run `breeze ci-image build --python 3.8` and try again."
)
- if cmd_result.returncode != 0:
- get_console().print(
- "[warning]If you see strange stacktraces above, "
- "run `breeze ci-image build --python 3.8` and try again."
- )
- sys.exit(cmd_result.returncode)
+sys.exit(cmd_result.returncode)
diff --git a/scripts/in_container/run_migration_reference.py
b/scripts/in_container/run_migration_reference.py
index 52c4a5c4d7..f5413aa482 100755
--- a/scripts/in_container/run_migration_reference.py
+++ b/scripts/in_container/run_migration_reference.py
@@ -28,6 +28,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, Iterable
from alembic.script import ScriptDirectory
+from rich.console import Console
from tabulate import tabulate
from airflow import __version__ as airflow_version
@@ -36,6 +37,8 @@ from airflow.utils.db import _get_alembic_config
if TYPE_CHECKING:
from alembic.script import Script
+console = Console(width=400, color_system="standard")
+
airflow_version = re.match(r"(\d+\.\d+\.\d+).*", airflow_version).group(1) #
type: ignore
project_root = Path(__file__).parents[2].resolve()
@@ -180,9 +183,14 @@ def ensure_filenames_are_sorted(revisions):
if __name__ == "__main__":
+ console.print("[bright_blue]Updating migration reference")
revisions = list(reversed(list(get_revisions())))
+ console.print("[bright_blue]Making sure airflow version updated")
ensure_airflow_version(revisions=revisions)
revisions = list(reversed(list(get_revisions())))
+ console.print("[bright_blue]Making sure filenames are sorted")
ensure_filenames_are_sorted(revisions=revisions)
revisions = list(get_revisions())
+ console.print("[bright_blue]Updating documentation")
update_docs(revisions=revisions)
+ console.print("[green]Migrations OK")
