This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new da9c4017d19 Remove pre-commit from devel extra of Airflow (#44059)
da9c4017d19 is described below
commit da9c4017d19bf17a8f3c3015602e54c919be2e2a
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Nov 15 18:46:57 2024 +0100
Remove pre-commit from devel extra of Airflow (#44059)
Having pre-commit as devel dependency is not a good idea because
it might conflict with the one installed globall when you install
a local venv. Instead of installing it as a dependency, we install
it separately in CI and CI image - together with UV and pre-comit-uv
to make it fast and keep consistency and keep it in the same version
in the development environment.
We still have TODO to keep it synchronized and updated automatically
in all places and in github action - that will be done in a separate
PR.
---
.github/actions/install-pre-commit/action.yml | 11 ++-
Dockerfile | 22 +++++-
Dockerfile.ci | 28 ++++++-
dev/breeze/doc/ci/02_images.md | 2 +-
.../commands/release_management_commands.py | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 86 ++++++++++++----------
hatch_build.py | 1 -
scripts/ci/install_breeze.sh | 2 +-
scripts/docker/common.sh | 20 ++++-
scripts/tools/setup_breeze | 2 +-
11 files changed, 119 insertions(+), 59 deletions(-)
diff --git a/.github/actions/install-pre-commit/action.yml
b/.github/actions/install-pre-commit/action.yml
index aa1dee87aa5..af8236fc4bd 100644
--- a/.github/actions/install-pre-commit/action.yml
+++ b/.github/actions/install-pre-commit/action.yml
@@ -24,7 +24,7 @@ inputs:
default: 3.9
uv-version:
description: 'uv version to use'
- default: 0.4.30
+ default: 0.5.2
pre-commit-version:
description: 'pre-commit version to use'
default: 4.0.1
@@ -36,11 +36,10 @@ runs:
steps:
- name: Install pre-commit, uv, and pre-commit-uv
shell: bash
- run: >
- pip install
- pre-commit==${{inputs.pre-commit-version}}
- uv==${{inputs.uv-version}}
- pre-commit-uv==${{inputs.pre-commit-uv-version}}
+ run: |
+ pip install uv==${{inputs.uv-version}} || true
+ uv tool install pre-commit==${{inputs.pre-commit-version}} --with
uv==${{inputs.uv-version}} \
+ --with pre-commit-uv==${{inputs.pre-commit-uv-version}}
- name: Cache pre-commit envs
uses: actions/cache@v4
with:
diff --git a/Dockerfile b/Dockerfile
index d9fb1878f11..b8f2e24b2dc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
ARG AIRFLOW_PIP_VERSION=24.3.1
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main";
-ARG AIRFLOW_UV_VERSION=0.5.1
+ARG AIRFLOW_UV_VERSION=0.5.2
ARG AIRFLOW_USE_UV="false"
ARG UV_HTTP_TIMEOUT="300"
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow";
@@ -606,6 +606,7 @@ function common::show_packaging_tool_version_and_location()
{
}
function common::install_packaging_tools() {
+ : "${AIRFLOW_USE_UV:?Should be set}"
if [[ "${VIRTUAL_ENV=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Checking packaging tools in venv:
${VIRTUAL_ENV}${COLOR_RESET}"
@@ -658,8 +659,23 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"uv==${AIRFLOW_UV_VERSION}"
fi
fi
- # make sure that the venv/user in .local exists
- mkdir -p "${HOME}/.local/bin"
+ if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv
uv${COLOR_RESET}"
+ echo
+ uv tool install pre-commit --with pre-commit-uv --with uv
+ # make sure that the venv/user in .local exists
+ mkdir -p "${HOME}/.local/bin"
+ else
+ echo
+ echo "${COLOR_BLUE}Installing predefined versions of pre-commit with
pre-commit-uv and uv:${COLOR_RESET}"
+ echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION})
uv(${AIRFLOW_UV_VERSION})
pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"
+ echo
+ uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \
+ --with "uv==${AIRFLOW_UV_VERSION}" --with
"pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"
+ # make sure that the venv/user in .local exists
+ mkdir -p "${HOME}/.local/bin"
+ fi
}
function common::import_trusted_gpg() {
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 2328688f0fa..6de02631899 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -547,6 +547,7 @@ function common::show_packaging_tool_version_and_location()
{
}
function common::install_packaging_tools() {
+ : "${AIRFLOW_USE_UV:?Should be set}"
if [[ "${VIRTUAL_ENV=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Checking packaging tools in venv:
${VIRTUAL_ENV}${COLOR_RESET}"
@@ -599,8 +600,23 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"uv==${AIRFLOW_UV_VERSION}"
fi
fi
- # make sure that the venv/user in .local exists
- mkdir -p "${HOME}/.local/bin"
+ if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv
uv${COLOR_RESET}"
+ echo
+ uv tool install pre-commit --with pre-commit-uv --with uv
+ # make sure that the venv/user in .local exists
+ mkdir -p "${HOME}/.local/bin"
+ else
+ echo
+ echo "${COLOR_BLUE}Installing predefined versions of pre-commit with
pre-commit-uv and uv:${COLOR_RESET}"
+ echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION})
uv(${AIRFLOW_UV_VERSION})
pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"
+ echo
+ uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \
+ --with "uv==${AIRFLOW_UV_VERSION}" --with
"pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"
+ # make sure that the venv/user in .local exists
+ mkdir -p "${HOME}/.local/bin"
+ fi
}
function common::import_trusted_gpg() {
@@ -1338,10 +1354,14 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
ARG AIRFLOW_PIP_VERSION=24.3.1
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main";
-ARG AIRFLOW_UV_VERSION=0.5.1
+ARG AIRFLOW_UV_VERSION=0.5.2
+# TODO(potiuk): automate with upgrade check (possibly)
+ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1"
+ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4"
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
- AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION}
+ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
+ AIRFLOW_PRE_COMMIT_VERSION=${AIRFLOW_PRE_COMMIT_VERSION}
# The PATH is needed for PIPX to find the tools installed
ENV PATH="/root/.local/bin:${PATH}"
diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md
index 4e9f63c8952..3ce770dc3b1 100644
--- a/dev/breeze/doc/ci/02_images.md
+++ b/dev/breeze/doc/ci/02_images.md
@@ -448,7 +448,7 @@ can be used for CI images:
| `ADDITIONAL_DEV_APT_DEPS` | | Additional
apt dev dependencies installed in the first part of the image
|
| `ADDITIONAL_DEV_APT_ENV` | | Additional
env variables defined when installing dev deps
|
| `AIRFLOW_PIP_VERSION` | `24.3.1` | PIP version
used.
|
-| `AIRFLOW_UV_VERSION` | `0.5.1` | UV version
used.
|
+| `AIRFLOW_UV_VERSION` | `0.5.2` | UV version
used.
|
| `AIRFLOW_USE_UV` | `true` | Whether to
use UV for installation.
|
| `PIP_PROGRESS_BAR` | `on` | Progress
bar for PIP installation
|
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 1fa57a3d744..7f9228ff45a 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -230,7 +230,7 @@ class VersionedFile(NamedTuple):
AIRFLOW_PIP_VERSION = "24.3.1"
-AIRFLOW_UV_VERSION = "0.5.1"
+AIRFLOW_UV_VERSION = "0.5.2"
AIRFLOW_USE_UV = False
# TODO: automate these as well
WHEEL_VERSION = "0.44.0"
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index c6981bb570e..b6a5476bf09 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -208,7 +208,7 @@ if MYSQL_INNOVATION_RELEASE:
ALLOWED_INSTALL_MYSQL_CLIENT_TYPES = ["mariadb", "mysql"]
PIP_VERSION = "24.3.1"
-UV_VERSION = "0.5.1"
+UV_VERSION = "0.5.2"
DEFAULT_UV_HTTP_TIMEOUT = 300
DEFAULT_WSL2_HTTP_TIMEOUT = 900
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py
b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 25df86203cd..594dc7204f2 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -218,48 +218,58 @@ def assert_pre_commit_installed():
python_executable = sys.executable
get_console().print(f"[info]Checking pre-commit installed for
{python_executable}[/]")
- command_result = run_command(
- ["pre-commit", "--version"],
- capture_output=True,
- text=True,
- check=False,
- )
- if command_result.returncode == 0:
- if command_result.stdout:
- pre_commit_version = command_result.stdout.split(" ")[1].strip()
- if Version(pre_commit_version) >= Version(min_pre_commit_version):
- get_console().print(
- f"\n[success]Package pre_commit is installed. "
- f"Good version {pre_commit_version} (>=
{min_pre_commit_version})[/]\n"
- )
+ need_to_reinstall_precommit = False
+ try:
+ command_result = run_command(
+ ["pre-commit", "--version"],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ if command_result.returncode == 0:
+ if command_result.stdout:
+ pre_commit_version = command_result.stdout.split("
")[1].strip()
+ if Version(pre_commit_version) >=
Version(min_pre_commit_version):
+ get_console().print(
+ f"\n[success]Package pre_commit is installed. "
+ f"Good version {pre_commit_version} (>=
{min_pre_commit_version})[/]\n"
+ )
+ else:
+ get_console().print(
+ f"\n[error]Package name pre_commit version is wrong.
It should be"
+ f"aat least {min_pre_commit_version} and is
{pre_commit_version}.[/]\n\n"
+ )
+ sys.exit(1)
+ if "pre-commit-uv" not in command_result.stdout:
+ get_console().print(
+ "\n[warning]You can significantly improve speed of
installing your pre-commit envs "
+ "by installing `pre-commit-uv` with it.[/]\n"
+ )
+ get_console().print(
+ "\n[warning]With uv you can install it with:[/]\n\n"
+ " uv tool install pre-commit --with
pre-commit-uv --force-reinstall\n"
+ )
+ get_console().print(
+ "\n[warning]With pipx you can install it with:[/]\n\n"
+ " pipx inject pre-commit pre-commit-uv #
optionally if you want to use uv to "
+ "install virtualenvs\n"
+ )
else:
get_console().print(
- f"\n[error]Package name pre_commit version is wrong. It
should be"
- f"aat least {min_pre_commit_version} and is
{pre_commit_version}.[/]\n\n"
- )
- sys.exit(1)
- if "pre-commit-uv" not in command_result.stdout:
- get_console().print(
- "\n[warning]You can significantly improve speed of
installing your pre-commit envs "
- "by installing `pre-commit-uv` with it.[/]\n"
- )
- get_console().print(
- "\n[warning]With uv you can install it with:[/]\n\n"
- " uv tool install pre-commit --with pre-commit-uv
--force-reinstall\n"
- )
- get_console().print(
- "\n[warning]With pipx you can install it with:[/]\n\n"
- " pipx inject pre-commit pre-commit-uv # optionally
if you want to use uv to "
- "install virtualenvs\n"
+ "\n[warning]Could not determine version of pre-commit. You
might need to update it![/]\n"
)
else:
- get_console().print(
- "\n[warning]Could not determine version of pre-commit. You
might need to update it![/]\n"
- )
- else:
- get_console().print("\n[error]Error checking for
pre-commit-installation:[/]\n")
- get_console().print(command_result.stderr)
- get_console().print("\nMake sure to run:\n breeze setup
self-upgrade\n\n")
+ need_to_reinstall_precommit = True
+ get_console().print("\n[error]Error checking for
pre-commit-installation:[/]\n")
+ get_console().print(command_result.stderr)
+ except FileNotFoundError as e:
+ need_to_reinstall_precommit = True
+ get_console().print(f"\n[error]Error checking for
pre-commit-installation: [/]\n{e}\n")
+ if need_to_reinstall_precommit:
+ get_console().print("\n[info]Make sure to install pre-commit. For
example by running\n\n")
+ get_console().print("uv tool install pre-commit\n")
+ get_console().print("Or if you prefer pipx:\n")
+ get_console().print("pipx install pre-commit")
sys.exit(1)
diff --git a/hatch_build.py b/hatch_build.py
index f21d8958b0f..1eddb85b469 100644
--- a/hatch_build.py
+++ b/hatch_build.py
@@ -248,7 +248,6 @@ DEVEL_EXTRAS: dict[str, list[str]] = {
],
"devel-static-checks": [
"black>=23.12.0",
- "pre-commit>=3.5.0",
"ruff==0.7.3",
"yamllint>=1.33.0",
],
diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh
index bdae70b95f5..2aba6229155 100755
--- a/scripts/ci/install_breeze.sh
+++ b/scripts/ci/install_breeze.sh
@@ -22,7 +22,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/../../"
PYTHON_ARG=""
PIP_VERSION="24.3.1"
-UV_VERSION="0.5.1"
+UV_VERSION="0.5.2"
if [[ ${PYTHON_VERSION=} != "" ]]; then
PYTHON_ARG="--python=$(which python"${PYTHON_VERSION}") "
fi
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index 8d820949843..33ab0587321 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -118,6 +118,7 @@ function common::show_packaging_tool_version_and_location()
{
}
function common::install_packaging_tools() {
+ : "${AIRFLOW_USE_UV:?Should be set}"
if [[ "${VIRTUAL_ENV=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Checking packaging tools in venv:
${VIRTUAL_ENV}${COLOR_RESET}"
@@ -170,8 +171,23 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"uv==${AIRFLOW_UV_VERSION}"
fi
fi
- # make sure that the venv/user in .local exists
- mkdir -p "${HOME}/.local/bin"
+ if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then
+ echo
+ echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv
uv${COLOR_RESET}"
+ echo
+ uv tool install pre-commit --with pre-commit-uv --with uv
+ # make sure that the venv/user in .local exists
+ mkdir -p "${HOME}/.local/bin"
+ else
+ echo
+ echo "${COLOR_BLUE}Installing predefined versions of pre-commit with
pre-commit-uv and uv:${COLOR_RESET}"
+ echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION})
uv(${AIRFLOW_UV_VERSION})
pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"
+ echo
+ uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \
+ --with "uv==${AIRFLOW_UV_VERSION}" --with
"pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"
+ # make sure that the venv/user in .local exists
+ mkdir -p "${HOME}/.local/bin"
+ fi
}
function common::import_trusted_gpg() {
diff --git a/scripts/tools/setup_breeze b/scripts/tools/setup_breeze
index 78d4db3a7d9..a0fe8613e46 100755
--- a/scripts/tools/setup_breeze
+++ b/scripts/tools/setup_breeze
@@ -27,7 +27,7 @@ COLOR_YELLOW=$'\e[33m'
COLOR_BLUE=$'\e[34m'
COLOR_RESET=$'\e[0m'
-UV_VERSION="0.5.1"
+UV_VERSION="0.5.2"
function manual_instructions() {
echo
