This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new de6d83a00fd Update uv, pip and pre-commit versions automatically in
more places. (#45398)
de6d83a00fd is described below
commit de6d83a00fd7af30586e58461428c8d3c74d55d3
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Jan 4 22:36:45 2025 +0100
Update uv, pip and pre-commit versions automatically in more places.
(#45398)
---
.github/actions/install-pre-commit/action.yml | 7 +-
.github/workflows/basic-tests.yml | 3 +-
.pre-commit-config.yaml | 8 +-
contributing-docs/08_static_code_checks.rst | 2 +-
dev/breeze/doc/ci/02_images.md | 56 +++---
dev/breeze/doc/images/output_static-checks.svg | 2 +-
dev/breeze/doc/images/output_static-checks.txt | 2 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 2 +-
scripts/ci/pre_commit/update_installers.py | 161 ---------------
.../pre_commit/update_installers_and_pre_commit.py | 218 +++++++++++++++++++++
10 files changed, 260 insertions(+), 201 deletions(-)
diff --git a/.github/actions/install-pre-commit/action.yml
b/.github/actions/install-pre-commit/action.yml
index b4c6a6c9d54..abdd3ea98ff 100644
--- a/.github/actions/install-pre-commit/action.yml
+++ b/.github/actions/install-pre-commit/action.yml
@@ -19,19 +19,18 @@
name: 'Install pre-commit'
description: 'Installs pre-commit and related packages'
inputs:
- # TODO(potiuk): automate update of these versions
python-version:
description: 'Python version to use'
default: "3.9"
uv-version:
description: 'uv version to use'
- default: "0.5.14"
+ default: "0.5.14" # Keep this comment to allow automatic replacement of
uv version
pre-commit-version:
description: 'pre-commit version to use'
- default: "4.0.1"
+ default: "4.0.1" # Keep this comment to allow automatic replacement of
pre-commit version
pre-commit-uv-version:
description: 'pre-commit-uv version to use'
- default: "4.1.4"
+ default: "4.1.4" # Keep this comment to allow automatic replacement of
pre-commit-uv version
runs:
using: "composite"
steps:
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 297a912ea81..353f65d9a6c 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -307,11 +307,12 @@ jobs:
run: >
pre-commit run
--all-files --show-diff-on-failure --color always --verbose
- --hook-stage manual update-installers || true
+ --hook-stage manual update-installers-and-pre-commit || true
if: always()
env:
UPGRADE_UV: "true"
UPGRADE_PIP: "false"
+ UPGRADE_PRE_COMMIT: "true"
- name: "Run automated upgrade for pip"
run: >
pre-commit run
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 09690edc5db..c5d0d154b88 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -197,12 +197,12 @@ repos:
additional_dependencies: ['pyyaml']
pass_filenames: false
require_serial: true
- - id: update-installers
- name: Update installers to latest (manual)
- entry: ./scripts/ci/pre_commit/update_installers.py
+ - id: update-installers-and-pre-commit
+ name: Update installers and pre-commit to latest (manual)
+ entry: ./scripts/ci/pre_commit/update_installers_and_pre_commit.py
stages: ['manual']
language: python
- files:
^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers.py$
+ files:
^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit.py$
pass_filenames: false
require_serial: true
additional_dependencies: ['pyyaml', 'rich>=12.4.4', 'requests']
diff --git a/contributing-docs/08_static_code_checks.rst
b/contributing-docs/08_static_code_checks.rst
index abfb7378904..78462afe305 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -374,7 +374,7 @@ require Breeze Docker image to be built locally.
+-----------------------------------------------------------+--------------------------------------------------------+---------+
| update-installed-providers-to-be-sorted | Sort and
uniquify installed_providers.txt | |
+-----------------------------------------------------------+--------------------------------------------------------+---------+
-| update-installers | Update
installers to latest (manual) | |
+| update-installers-and-pre-commit | Update
installers and pre-commit to latest (manual) | |
+-----------------------------------------------------------+--------------------------------------------------------+---------+
| update-local-yml-file | Update mounts in
the local yml file | |
+-----------------------------------------------------------+--------------------------------------------------------+---------+
diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md
index b613f67c72d..3d1d7d8b53e 100644
--- a/dev/breeze/doc/ci/02_images.md
+++ b/dev/breeze/doc/ci/02_images.md
@@ -419,33 +419,35 @@ DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \
The following build arguments (`--build-arg` in docker build command)
can be used for CI images:
-| Build argument | Default value | Description
|
-|-----------------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python
image
|
-| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor
version of Python (should match base image)
|
-| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing
this number will reinstall all apt dependencies
|
-| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional
`pip` flags passed to the installation commands (except when reinstalling `pip`
itself) |
-| `HOME` | `/root` | Home
directory of the root user (CI image has root user as default)
|
-| `AIRFLOW_HOME` | `/root/airflow` | Airflow's
HOME (that's where logs and sqlite databases are stored)
|
-| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted
sources of Airflow
|
-| `AIRFLOW_REPO` | `apache/airflow` | the
repository from which PIP dependencies are pre-installed
|
-| `AIRFLOW_BRANCH` | `main` | the branch
from which PIP dependencies are pre-installed
|
-| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing
this value will reinstall PIP dependencies from the repository from scratch
|
-| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not
empty, it will override the source of the constraints with the specified URL or
file. |
-| `AIRFLOW_CONSTRAINTS_REFERENCE` | | reference
(branch or tag) from GitHub repository from which constraints are used. By
default it is set to `constraints-main` but can be `constraints-2-X`. |
-| `AIRFLOW_EXTRAS` | `all` | extras to
install
|
-| `UPGRADE_INVALIDATION_STRING` | | If set to
any random value the dependencies are upgraded to newer versions. In CI it is
set to build id. |
-| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional
extras to install
|
-| `ADDITIONAL_PYTHON_DEPS` | | additional
Python dependencies to install
|
-| `DEV_APT_COMMAND` | | Dev apt
command executed before dev deps are installed in the first part of image
|
-| `ADDITIONAL_DEV_APT_COMMAND` | | Additional
Dev apt command executed before dev dep are installed in the first part of the
image |
-| `DEV_APT_DEPS` | | Dev APT
dependencies installed in the first part of the image (default empty means
default dependencies are used) |
-| `ADDITIONAL_DEV_APT_DEPS` | | Additional
apt dev dependencies installed in the first part of the image
|
-| `ADDITIONAL_DEV_APT_ENV` | | Additional
env variables defined when installing dev deps
|
-| `AIRFLOW_PIP_VERSION` | `24.3.1` | PIP version
used.
|
-| `AIRFLOW_UV_VERSION` | `0.5.14` | UV version
used.
|
-| `AIRFLOW_USE_UV` | `true` | Whether to
use UV for installation.
|
-| `PIP_PROGRESS_BAR` | `on` | Progress
bar for PIP installation
|
+| Build argument | Default value | Description
|
+|---------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------|
+| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python
image
|
+| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor
version of Python (should match base image)
|
+| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing
this number will reinstall all apt dependencies
|
+| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional
`pip` flags passed to the installation commands (except when reinstalling `pip`
itself) |
+| `HOME` | `/root` | Home
directory of the root user (CI image has root user as default)
|
+| `AIRFLOW_HOME` | `/root/airflow` | Airflow's
HOME (that's where logs and sqlite databases are stored)
|
+| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted
sources of Airflow
|
+| `AIRFLOW_REPO` | `apache/airflow` | the
repository from which PIP dependencies are pre-installed
|
+| `AIRFLOW_BRANCH` | `main` | the branch
from which PIP dependencies are pre-installed
|
+| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing
this value will reinstall PIP dependencies from the repository from scratch
|
+| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty,
it will override the source of the constraints with the specified URL or file.
|
+| `AIRFLOW_CONSTRAINTS_REFERENCE` | `constraints-main` | reference
(branch or tag) from GitHub repository from which constraints are used.
|
+| `AIRFLOW_EXTRAS` | `all` | extras to
install
|
+| `UPGRADE_INVALIDATION_STRING` | | If set to any
random value the dependencies are upgraded to newer versions. In CI it is set
to build id. |
+| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional
extras to install
|
+| `ADDITIONAL_PYTHON_DEPS` | | additional
Python dependencies to install
|
+| `DEV_APT_COMMAND` | | Dev apt
command executed before dev deps are installed in the first part of image
|
+| `ADDITIONAL_DEV_APT_COMMAND` | | Additional
Dev apt command executed before dev dep are installed in the first part of the
image |
+| `DEV_APT_DEPS` | | Dev APT
dependencies installed in the first part of the image (default empty means
default dependencies are used) |
+| `ADDITIONAL_DEV_APT_DEPS` | | Additional
apt dev dependencies installed in the first part of the image
|
+| `ADDITIONAL_DEV_APT_ENV` | | Additional
env variables defined when installing dev deps
|
+| `AIRFLOW_PIP_VERSION` | `24.3.1` | `pip` version
used.
|
+| `AIRFLOW_UV_VERSION` | `0.5.14` | `uv` version
used.
|
+| `AIRFLOW_PRE_COMMIT_VERSION` | `4.0.1` | `pre-commit`
version used.
|
+| `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` |
`pre-commit-uv` version used.
|
+| `AIRFLOW_USE_UV` | `true` | Whether to
use UV for installation.
|
+| `PIP_PROGRESS_BAR` | `on` | Progress bar
for PIP installation
|
Here are some examples of how CI images can built manually. CI is always
diff --git a/dev/breeze/doc/images/output_static-checks.svg
b/dev/breeze/doc/images/output_static-checks.svg
index a2422824dd9..bfea55c495f 100644
--- a/dev/breeze/doc/images/output_static-checks.svg
+++ b/dev/breeze/doc/images/output_static-checks.svg
@@ -371,7 +371,7 @@
</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2"
clip-path="url(#breeze-static-checks-line-50)">update-breeze-readme-config-hash | update-chart-dependencies |                   </text><text
class="breeze-static-checks-r5" [...]
</text><text class="breeze-static-checks-r5" x="0" y="1264.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-51)">update-common-sql-api-stubs | update-er-diagram | update-extras |                </text><text
class="breeze-static-checks-r5" x [...]
</text><text class="breeze-static-checks-r5" x="0" y="1288.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1288.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-52)">update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            </text><text
class="breeze-static-checks-r5" x="1451.8" y="1288.8" textLengt [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1313.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1313.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-53)">update-installed-providers-to-be-sorted | update-installers |                    </text><text
class="breeze-static-ch [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1313.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1313.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-53)">update-installed-providers-to-be-sorted | update-installers-and-pre-commit |     </text><text
class="breeze-static-checks-r5" x="1451.8" y="1313.2" textLength="12.2"
clip-path="url(#breeze-sta [...]
</text><text class="breeze-static-checks-r5" x="0" y="1337.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1337.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-54)">update-local-yml-file | update-migration-references |                           &#
[...]
</text><text class="breeze-static-checks-r5" x="0" y="1362" textLength="12.2"
clip-path="url(#breeze-static-checks-line-55)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1362" textLength="988.2"
clip-path="url(#breeze-static-checks-line-55)">update-openapi-spec-tags-to-be-sorted | update-providers-dependencies |          </text><text
class="breeze-static-checks-r5" x="1451.8" y="1362" textLength="12.2" clip-pa
[...]
</text><text class="breeze-static-checks-r5" x="0" y="1386.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-56)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1386.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-56)">update-providers-init-py | update-reproducible-source-date-epoch |               </text><text
class="breeze-static-checks-r5" x="1451.8" y="13 [...]
diff --git a/dev/breeze/doc/images/output_static-checks.txt
b/dev/breeze/doc/images/output_static-checks.txt
index a685635825f..38529eb9753 100644
--- a/dev/breeze/doc/images/output_static-checks.txt
+++ b/dev/breeze/doc/images/output_static-checks.txt
@@ -1 +1 @@
-1e545fdd89efbdd78a883519b6d93ce2
+6239e6a528459f731b6908ce668a8950
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index a4495df607e..8667a2cc4b7 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -141,7 +141,7 @@ PRE_COMMIT_LIST = [
"update-in-the-wild-to-be-sorted",
"update-inlined-dockerfile-scripts",
"update-installed-providers-to-be-sorted",
- "update-installers",
+ "update-installers-and-pre-commit",
"update-local-yml-file",
"update-migration-references",
"update-openapi-spec-tags-to-be-sorted",
diff --git a/scripts/ci/pre_commit/update_installers.py
b/scripts/ci/pre_commit/update_installers.py
deleted file mode 100755
index 28de1988240..00000000000
--- a/scripts/ci/pre_commit/update_installers.py
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import os
-import re
-import sys
-from pathlib import Path
-
-import requests
-
-sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure
common_precommit_utils is imported
-from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH, console
-
-FILES_TO_UPDATE = [
- AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile",
- AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci",
- AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh",
- AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh",
- AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze",
- AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml",
- AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" /
"global_constants.py",
- AIRFLOW_SOURCES_ROOT_PATH
- / "dev"
- / "breeze"
- / "src"
- / "airflow_breeze"
- / "commands"
- / "release_management_commands.py",
-]
-
-
-DOC_FILES_TO_UPDATE: list[Path] = [
- AIRFLOW_SOURCES_ROOT_PATH / "dev/" / "breeze" / "doc" / "ci" /
"02_images.md"
-]
-
-
-def get_latest_pypi_version(package_name: str) -> str:
- response = requests.get(f"https://pypi.org/pypi/{package_name}/json";)
- response.raise_for_status() # Ensure we got a successful response
- data = response.json()
- latest_version = data["info"]["version"] # The version info is under the
'info' key
- return latest_version
-
-
-AIRFLOW_PIP_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)")
-AIRFLOW_PIP_QUOTED_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION =
)(\"[0-9.]+\")")
-PIP_QUOTED_PATTERN = re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")")
-PIP_QUOTED_PATTERN_NO_SPACES = re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")")
-AIRFLOW_PIP_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\|
*)(`[0-9.]+`)( *\|)")
-AIRFLOW_PIP_UPGRADE_PATTERN = re.compile(r"(python -m pip install --upgrade
pip==)([0-9.]+)")
-
-AIRFLOW_UV_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)")
-AIRFLOW_UV_QUOTED_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")")
-UV_QUOTED_PATTERN = re.compile(r"(UV_VERSION = )(\"[0-9.]+\")")
-UV_QUOTED_PATTERN_NO_SPACES = re.compile(r"(UV_VERSION=)(\"[0-9.]+\")")
-AIRFLOW_UV_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\|
*)(`[0-9.]+`)( *\|)")
-UV_GREATER_PATTERN = re.compile(r'"(uv>=)([0-9]+)"')
-
-UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true"
-UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true"
-
-
-def replace_group_2_while_keeping_total_length(pattern: re.Pattern[str],
replacement: str, text: str) -> str:
- def replacer(match):
- original_length = len(match.group(2))
- padding = ""
- if len(match.groups()) > 2:
- padding = match.group(3)
- new_length = len(replacement)
- diff = new_length - original_length
- if diff <= 0:
- padding = " " * -diff + padding
- else:
- padding = padding[diff:]
- padded_replacement = match.group(1) + replacement + padding
- return padded_replacement.strip()
-
- return re.sub(pattern, replacer, text)
-
-
-if __name__ == "__main__":
- pip_version = get_latest_pypi_version("pip")
- console.print(f"[bright_blue]Latest pip version: {pip_version}")
- uv_version = get_latest_pypi_version("uv")
- console.print(f"[bright_blue]Latest uv version: {uv_version}")
-
- changed = False
- for file in FILES_TO_UPDATE:
- console.print(f"[bright_blue]Updating {file}")
- file_content = file.read_text()
- new_content = file_content
- if UPGRADE_PIP:
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_PIP_PATTERN, pip_version, new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_PIP_UPGRADE_PATTERN, pip_version, new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- PIP_QUOTED_PATTERN_NO_SPACES, f'"{pip_version}"', new_content
- )
- if UPGRADE_UV:
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_UV_PATTERN, uv_version, new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- UV_GREATER_PATTERN, uv_version, new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_UV_QUOTED_PATTERN, f'"{uv_version}"', new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- UV_QUOTED_PATTERN, f'"{uv_version}"', new_content
- )
- new_content = replace_group_2_while_keeping_total_length(
- UV_QUOTED_PATTERN_NO_SPACES, f'"{uv_version}"', new_content
- )
- if new_content != file_content:
- file.write_text(new_content)
- console.print(f"[bright_blue]Updated {file}")
- changed = True
- for file in DOC_FILES_TO_UPDATE:
- console.print(f"[bright_blue]Updating {file}")
- file_content = file.read_text()
- new_content = file_content
- if UPGRADE_PIP:
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_PIP_DOC_PATTERN, f"`{pip_version}`", new_content
- )
- if UPGRADE_UV:
- new_content = replace_group_2_while_keeping_total_length(
- AIRFLOW_UV_DOC_PATTERN, f"`{uv_version}`", new_content
- )
- if new_content != file_content:
- file.write_text(new_content)
- console.print(f"[bright_blue]Updated {file}")
- changed = True
- if changed:
- sys.exit(1)
diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py
b/scripts/ci/pre_commit/update_installers_and_pre_commit.py
new file mode 100755
index 00000000000..9e5b2c68cf5
--- /dev/null
+++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py
@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+import re
+import sys
+from enum import Enum
+from pathlib import Path
+
+import requests
+
+sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure
common_precommit_utils is imported
+from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH, console
+
+# List of files to update and whether to keep total length of the original
value when replacing.
+FILES_TO_UPDATE: list[tuple[Path, bool]] = [
+ (AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile", False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci", False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh",
False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh", False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze", False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml", False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" /
"global_constants.py", False),
+ (
+ AIRFLOW_SOURCES_ROOT_PATH
+ / "dev"
+ / "breeze"
+ / "src"
+ / "airflow_breeze"
+ / "commands"
+ / "release_management_commands.py",
+ False,
+ ),
+ (AIRFLOW_SOURCES_ROOT_PATH / ".github" / "actions" / "install-pre-commit"
/ "action.yml", False),
+ (AIRFLOW_SOURCES_ROOT_PATH / "dev/" / "breeze" / "doc" / "ci" /
"02_images.md", True),
+]
+
+
+def get_latest_pypi_version(package_name: str) -> str:
+ response = requests.get(f"https://pypi.org/pypi/{package_name}/json";)
+ response.raise_for_status() # Ensure we got a successful response
+ data = response.json()
+ latest_version = data["info"]["version"] # The version info is under the
'info' key
+ return latest_version
+
+
+class Quoting(Enum):
+ UNQUOTED = 0
+ SINGLE_QUOTED = 1
+ DOUBLE_QUOTED = 2
+ REVERSE_SINGLE_QUOTED = 3
+
+
+PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+ (re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)"),
Quoting.UNQUOTED),
+ (re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"),
Quoting.REVERSE_SINGLE_QUOTED),
+]
+
+UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+ (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(uv>=)([0-9]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(UV_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)"),
Quoting.REVERSE_SINGLE_QUOTED),
+ (
+ re.compile(
+ r"(default: \")([0-9.]+)(\" # Keep this comment to "
+ r"allow automatic replacement of uv version)"
+ ),
+ Quoting.UNQUOTED,
+ ),
+]
+
+PRE_COMMIT_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+ (re.compile(r"(AIRFLOW_PRE_COMMIT_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(AIRFLOW_PRE_COMMIT_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(pre-commit>=)([0-9]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(PRE_COMMIT_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(PRE_COMMIT_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED),
+ (
+ re.compile(r"(\| *`AIRFLOW_PRE_COMMIT_VERSION` *\| *)(`[0-9.]+`)(
*\|)"),
+ Quoting.REVERSE_SINGLE_QUOTED,
+ ),
+ (
+ re.compile(
+ r"(default: \")([0-9.]+)(\" # Keep this comment to allow
automatic "
+ r"replacement of pre-commit version)"
+ ),
+ Quoting.UNQUOTED,
+ ),
+]
+
+PRE_COMMIT_UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+ (re.compile(r"(AIRFLOW_PRE_COMMIT_UV_VERSION=)([0-9.]+)"),
Quoting.UNQUOTED),
+ (re.compile(r"(AIRFLOW_PRE_COMMIT_UV_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(pre-commit-uv>=)([0-9]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(PRE_COMMIT_UV_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(PRE_COMMIT_UV_VERSION=)(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (
+ re.compile(r"(\| *`AIRFLOW_PRE_COMMIT_UV_VERSION` *\| *)(`[0-9.]+`)(
*\|)"),
+ Quoting.REVERSE_SINGLE_QUOTED,
+ ),
+ (
+ re.compile(
+ r"(default: \")([0-9.]+)(\" # Keep this comment to allow
automatic "
+ r"replacement of pre-commit-uv version)"
+ ),
+ Quoting.UNQUOTED,
+ ),
+]
+
+
+def get_replacement(value: str, quoting: Quoting) -> str:
+ if quoting == Quoting.DOUBLE_QUOTED:
+ return f'"{value}"'
+ elif quoting == Quoting.SINGLE_QUOTED:
+ return f"'{value}'"
+ elif quoting == Quoting.REVERSE_SINGLE_QUOTED:
+ return f"`{value}`"
+ return value
+
+
+UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true"
+UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true"
+UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT",
"true").lower() == "true"
+
+
+def replace_version(pattern: re.Pattern[str], version: str, text: str,
keep_total_length: bool = True) -> str:
+ # Assume that the pattern has up to 3 replacement groups:
+ # 1. Prefix
+ # 2. Original version
+ # 3. Suffix
+ #
+ # (prefix)(version)(suffix)
+ # In case "keep_total_length" is set to True, the replacement will be
padded with spaces to match
+ # the original length
+ def replacer(match):
+ prefix = match.group(1)
+ postfix = match.group(3) if len(match.groups()) > 2 else ""
+ if not keep_total_length:
+ return prefix + version + postfix
+ original_length = len(match.group(2))
+ new_length = len(version)
+ diff = new_length - original_length
+ if diff <= 0:
+ postfix = " " * -diff + postfix
+ else:
+ postfix = postfix[diff:]
+ padded_replacement = prefix + version + postfix
+ return padded_replacement.strip()
+
+ return re.sub(pattern, replacer, text)
+
+
+if __name__ == "__main__":
+ changed = False
+ for file, keep_length in FILES_TO_UPDATE:
+ console.print(f"[bright_blue]Updating {file}")
+ file_content = file.read_text()
+ new_content = file_content
+ if UPGRADE_PIP:
+ pip_version = get_latest_pypi_version("pip")
+ console.print(f"[bright_blue]Latest pip version: {pip_version}")
+ for line_pattern, quoting in PIP_PATTERNS:
+ new_content = replace_version(
+ line_pattern, get_replacement(pip_version, quoting),
new_content, keep_length
+ )
+ if UPGRADE_UV:
+ uv_version = get_latest_pypi_version("uv")
+ console.print(f"[bright_blue]Latest uv version: {uv_version}")
+ for line_pattern, quoting in UV_PATTERNS:
+ new_content = replace_version(
+ line_pattern, get_replacement(uv_version, quoting),
new_content, keep_length
+ )
+ if UPGRADE_PRE_COMMIT:
+ pre_commit_version = get_latest_pypi_version("pre-commit")
+ console.print(f"[bright_blue]Latest pre-commit version:
{pre_commit_version}")
+ for line_pattern, quoting in PRE_COMMIT_PATTERNS:
+ new_content = replace_version(
+ line_pattern, get_replacement(pre_commit_version,
quoting), new_content, keep_length
+ )
+ if UPGRADE_UV:
+ pre_commit_uv_version =
get_latest_pypi_version("pre-commit-uv")
+ console.print(f"[bright_blue]Latest pre-commit-uv version:
{pre_commit_uv_version}")
+ for line_pattern, quoting in PRE_COMMIT_UV_PATTERNS:
+ new_content = replace_version(
+ line_pattern,
+ get_replacement(pre_commit_uv_version, quoting),
+ new_content,
+ keep_length,
+ )
+ if new_content != file_content:
+ file.write_text(new_content)
+ console.print(f"[bright_blue]Updated {file}")
+ changed = True
+ if changed:
+ sys.exit(1)
