This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ddc5670a8c6 mAKe Breeze installation and reinstallation support both
uv and pipx (#43607)
ddc5670a8c6 is described below
commit ddc5670a8c6f2facb490d3f8de297fb7705d3887
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Nov 2 22:06:41 2024 +0100
mAKe Breeze installation and reinstallation support both uv and pipx
(#43607)
So far `breeze` fully supported only `pipx` installation. For
example it would reinstall itself automatically with pipx if you
attempted to run it from another workspace/checked out repository
of Airflow, and it only provided instructions for pipx.
With this PR:
* the `uv tool` is preferred way to install breeze
* the `pipx` is PSF-governance managed alternative
* breeze will reinstall itself using uv if it has been installed with uv
before when it is run from a different workspace or different
airflow repo checked out in another folder
Also documentation is updated to make `uv` the recommended tool and
describing how to install it - with `pipx` provided as an alternative.
Warning is printed in case pre-commit-uv is not installed with the
pre-commit (pre-commit-uv significantly speeds up installation of
the venvs by pre-commit). This warning also provides instructions
how to install it.
---
.github/actions/install-pre-commit/action.yml | 50 ++++++++++++++++++++++
.github/workflows/basic-tests.yml | 13 ++----
.github/workflows/static-checks-mypy-docs.yml | 20 ++++-----
contributing-docs/03_contributors_quick_start.rst | 2 +-
dev/breeze/doc/01_installation.rst | 25 ++++++++---
.../airflow_breeze/commands/developer_commands.py | 4 +-
.../commands/release_management_commands.py | 15 ++++---
dev/breeze/src/airflow_breeze/global_constants.py | 28 +++++++-----
.../src/airflow_breeze/utils/kubernetes_utils.py | 5 +--
dev/breeze/src/airflow_breeze/utils/path_utils.py | 13 +++---
.../src/airflow_breeze/utils/python_versions.py | 7 +--
dev/breeze/src/airflow_breeze/utils/reinstall.py | 13 +++++-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 26 +++++++----
scripts/ci/pre_commit/common_precommit_utils.py | 7 ++-
14 files changed, 159 insertions(+), 69 deletions(-)
diff --git a/.github/actions/install-pre-commit/action.yml
b/.github/actions/install-pre-commit/action.yml
new file mode 100644
index 00000000000..02eea2c7229
--- /dev/null
+++ b/.github/actions/install-pre-commit/action.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Install pre-commit'
+description: 'Installs pre-commit and related packages'
+inputs:
+ python-version:
+ description: 'Python version to use'
+ default: 3.9
+ uv-version:
+ description: 'uv version to use'
+ default: 0.4.29
+ pre-commit-version:
+ description: 'pre-commit version to use'
+ default: 4.0.1
+ pre-commit-uv-version:
+ description: 'pre-commit-uv version to use'
+ default: 4.1.4
+runs:
+ using: "composite"
+ steps:
+ - name: Install pre-commit, uv, and pre-commit-uv
+ shell: bash
+ run: >
+ pip install
+ pre-commit==${{inputs.pre-commit-version}}
+ uv==${{inputs.uv-version}}
+ pre-commit-uv==${{inputs.pre-commit-uv-version}}
+ - name: Cache pre-commit envs
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/pre-commit
+ key: "pre-commit-${{inputs.python-version}}-${{
hashFiles('.pre-commit-config.yaml') }}"
+ restore-keys: |
+ pre-commit-${{inputs.python-version}}-
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 5b6600e560c..bf7e8ab7bf7 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -285,16 +285,11 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
id: breeze
- - name: Cache pre-commit envs
- uses: actions/cache@v4
+ - name: "Install pre-commit"
+ uses: ./.github/actions/install-pre-commit
+ id: pre-commit
with:
- path: ~/.cache/pre-commit
- # yamllint disable-line rule:line-length
- key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{
hashFiles('.pre-commit-config.yaml') }}"
- restore-keys: "\
- pre-commit-${{steps.breeze.outputs.host-python-version}}-\
- ${{ hashFiles('.pre-commit-config.yaml') }}\n
- pre-commit-${{steps.breeze.outputs.host-python-version}}-"
+ python-version: ${{steps.breeze.outputs.host-python-version}}
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/static-checks-mypy-docs.yml
b/.github/workflows/static-checks-mypy-docs.yml
index b34ad2c36f4..be2c4f8e286 100644
--- a/.github/workflows/static-checks-mypy-docs.yml
+++ b/.github/workflows/static-checks-mypy-docs.yml
@@ -126,14 +126,11 @@ jobs:
- name: "Prepare breeze & CI image: ${{
inputs.default-python-version}}:${{ inputs.image-tag }}"
uses: ./.github/actions/prepare_breeze_and_image
id: breeze
- - name: Cache pre-commit envs
- uses: actions/cache@v4
+ - name: "Install pre-commit"
+ uses: ./.github/actions/install-pre-commit
+ id: pre-commit
with:
- path: ~/.cache/pre-commit
- # yamllint disable-line rule:line-length
- key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{
hashFiles('.pre-commit-config.yaml') }}"
- restore-keys: |
- pre-commit-${{steps.breeze.outputs.host-python-version}}-
+ python-version: ${{steps.breeze.outputs.host-python-version}}
- name: "Static checks"
run: breeze static-checks --all-files --show-diff-on-failure --color
always --initialize-environment
env:
@@ -170,10 +167,13 @@ jobs:
- name: "Prepare breeze & CI image: ${{ inputs.default-python-version
}}:${{ inputs.image-tag }}"
uses: ./.github/actions/prepare_breeze_and_image
id: breeze
+ - name: "Install pre-commit"
+ uses: ./.github/actions/install-pre-commit
+ id: pre-commit
+ with:
+ python-version: ${{steps.breeze.outputs.host-python-version}}
- name: "MyPy checks for ${{ matrix.mypy-check }}"
- run: |
- pip install pre-commit
- pre-commit run --color always --verbose --hook-stage manual
${{matrix.mypy-check}} --all-files
+ run: pre-commit run --color always --verbose --hook-stage manual
${{matrix.mypy-check}} --all-files
env:
VERBOSE: "false"
COLUMNS: "250"
diff --git a/contributing-docs/03_contributors_quick_start.rst
b/contributing-docs/03_contributors_quick_start.rst
index a088e3cb0d4..96f80220770 100644
--- a/contributing-docs/03_contributors_quick_start.rst
+++ b/contributing-docs/03_contributors_quick_start.rst
@@ -476,7 +476,7 @@ You can still add uv support for pre-commit if you use pipx
using the commands:
pipx install pre-commit
pipx inject
- pipx inject pre-commit pre-commit-uv
+ pipx inject prepare_breeze_and_image
Also, if you already use ``uvx`` instead of ``pipx``, use this command:
diff --git a/dev/breeze/doc/01_installation.rst
b/dev/breeze/doc/01_installation.rst
index 1c7ad0ee628..052dc3faca9 100644
--- a/dev/breeze/doc/01_installation.rst
+++ b/dev/breeze/doc/01_installation.rst
@@ -151,13 +151,28 @@ Docker in WSL 2
If VS Code is installed on the Windows host system then in the WSL Linux
Distro
you can run ``code .`` in the root directory of you Airflow repo to launch
VS Code.
-The pipx tool
---------------
+The uv tool
+-----------
+
+We are recommending to use the ``uv`` tool to manage your virtual environments
and generally as a swiss-knife
+of your Python environment (it supports installing various versions of Python,
creating virtual environments,
+installing packages, managing workspaces and running development tools.).
+
+Installing ``uv`` is described in the `uv documentation
<https://docs.astral.sh/uv/getting-started/installation/>`_.
+We highly recommend using ``uv`` to manage your Python environments, as it is
very comprehensive,
+easy to use, it is faster than any of the other tools availables (way faster!)
and has a lot of features
+that make it easier to work with Python.
+
+Alternative: pipx tool
+----------------------
-We are using ``pipx`` tool to install and manage Breeze. The ``pipx`` tool is
created by the creators
+However, we do not want to be entirely dependent on ``uv`` as it is a software
governed by a VC-backed vendor,
+so we always want to provide open-source governed alternatives for our tools.
If you can't or do not want to
+use ``uv``, we got you covered. Another too you can use to manage development
tools (and ``breeze`` development
+environment is Python-Software-Foundation managed ``pipx``. The ``pipx`` tool
is created by the creators
of ``pip`` from `Python Packaging Authority <https://www.pypa.io/en/latest/>`_
-Note that ``pipx`` >= 1.4.1 is used.
+Note that ``pipx`` >= 1.4.1 should be used.
Install pipx
@@ -172,7 +187,7 @@ environments. This can be done automatically by the
following command (follow in
pipx ensurepath
-In Mac
+In case ``pipx`` is not in your PATH, you can run it with Python module:
.. code-block:: bash
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 0f9cec28710..b26230c2e16 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -861,7 +861,7 @@ def static_checks(
for attempt in range(1, 1 + max_initialization_attempts):
get_console().print(f"[info]Attempt number {attempt} to install
pre-commit environments")
initialization_result = run_command(
- [sys.executable, "-m", "pre_commit", "install",
"--install-hooks"],
+ ["pre-commit", "install", "--install-hooks"],
check=False,
no_output_dump_on_exception=True,
text=True,
@@ -874,7 +874,7 @@ def static_checks(
get_console().print("[error]Could not install pre-commit
environments[/]")
sys.exit(return_code)
- command_to_execute = [sys.executable, "-m", "pre_commit", "run"]
+ command_to_execute = ["pre-commit", "run"]
if not one_or_none_set([last_commit, commit_ref, only_my_changes,
all_files]):
get_console().print(
"\n[error]You can only specify "
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index f906ffaa442..61a26e9993f 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -232,13 +232,14 @@ class VersionedFile(NamedTuple):
AIRFLOW_PIP_VERSION = "24.3.1"
AIRFLOW_UV_VERSION = "0.4.29"
AIRFLOW_USE_UV = False
-WHEEL_VERSION = "0.36.2"
-GITPYTHON_VERSION = "3.1.40"
-RICH_VERSION = "13.7.0"
-NODE_VERSION = "21.2.0"
-PRE_COMMIT_VERSION = "3.5.0"
-HATCH_VERSION = "1.9.1"
-PYYAML_VERSION = "6.0.1"
+# TODO: automate thsese as well
+WHEEL_VERSION = "0.44.0"
+GITPYTHON_VERSION = "3.1.43"
+RICH_VERSION = "13.9.4"
+NODE_VERSION = "22.2.0"
+PRE_COMMIT_VERSION = "4.0.1"
+HATCH_VERSION = "1.13.0"
+PYYAML_VERSION = "6.0.2"
AIRFLOW_BUILD_DOCKERFILE = f"""
FROM
python:{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}-slim-{ALLOWED_DEBIAN_VERSIONS[0]}
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index c80a5a9355e..a674b142b3c 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -32,10 +32,15 @@ try:
except ImportError:
get_console().print(
"\n[error]Breeze doesn't support Python version <=3.8\n\n"
- "[warning]Use Python 3.9 and force reinstall breeze with pipx\n\n"
- " pipx install --force -e ./dev/breeze\n"
+ "[warning]Use Python 3.9 and force reinstall breeze:"
+ ""
+ " either with uv: \n\n"
+ " uv tool install --force --reinstall --editable ./dev/breeze\n\n"
+ ""
+ " or with pipx\n\n"
+ " pipx install --force -e ./dev/breeze --python 3.9\n"
"\nTo find out more, visit [info]https://github.com/apache/airflow/";
- "blob/main/dev/breeze/doc/01_installation.rst#the-pipx-tool[/]\n"
+ "blob/main/dev/breeze/doc/01_installation.rst[/]\n"
)
sys.exit(1)
from pathlib import Path
@@ -253,13 +258,16 @@ ALLOWED_HELM_TEST_PACKAGES = [
@cache
def all_task_sdk_test_packages() -> list[str]:
- return sorted(
- [
- candidate.name
- for candidate in (AIRFLOW_SOURCES_ROOT / "task_sdk" /
"tests").iterdir()
- if candidate.is_dir() and candidate.name != "__pycache__"
- ]
- )
+ try:
+ return sorted(
+ [
+ candidate.name
+ for candidate in (AIRFLOW_SOURCES_ROOT / "task_sdk" /
"tests").iterdir()
+ if candidate.is_dir() and candidate.name != "__pycache__"
+ ]
+ )
+ except FileNotFoundError:
+ return []
ALLOWED_TASK_SDK_TEST_PACKAGES = [
diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
index 3aca9d51c13..b9bdc5302bd 100644
--- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
@@ -391,10 +391,7 @@ def create_virtualenv(force_venv_setup: bool) ->
RunCommandResult:
"[info]You can uninstall breeze and install it again with earlier
Python "
"version. For example:[/]\n"
)
- get_console().print("pipx reinstall --python PYTHON_PATH
apache-airflow-breeze\n")
- get_console().print(
- f"[info]PYTHON_PATH - path to your Python binary(<
{higher_python_version_tuple})[/]\n"
- )
+
get_console().print("[info]Then recreate your k8s virtualenv
with:[/]\n")
get_console().print("breeze k8s setup-env --force-venv-setup\n")
sys.exit(1)
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py
b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index 0feba56356b..cf04ecc2787 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -167,8 +167,9 @@ def reinstall_if_setup_changed() -> bool:
return False
if "apache-airflow-breeze" in e.msg:
print(
- """Missing Package `apache-airflow-breeze`.
- Use `pipx install -e ./dev/breeze` to install the
package."""
+ """Missing Package `apache-airflow-breeze`. Please install
it.\n
+ Use `uv tool install -e ./dev/breeze or `pipx install -e
./dev/breeze`
+ to install the package."""
)
return False
sources_hash = get_installation_sources_config_metadata_hash()
@@ -224,10 +225,10 @@ def get_used_airflow_sources() -> Path:
@cache
def find_airflow_sources_root_to_operate_on() -> Path:
"""
- Find the root of airflow sources we operate on. Handle the case when
Breeze is installed via `pipx` from
- a different source tree, so it searches upwards of the current directory
to find the right root of
- airflow directory we are actually in. This **might** be different than the
sources of Airflow Breeze
- was installed from.
+ Find the root of airflow sources we operate on. Handle the case when
Breeze is installed via
+ `pipx` or `uv tool` from a different source tree, so it searches upwards
of the current directory
+ to find the right root of airflow directory we are actually in. This
**might** be different
+ than the sources of Airflow Breeze was installed from.
If not found, we operate on Airflow sources that we were installed it.
This handles the case when
we run Breeze from a "random" directory.
diff --git a/dev/breeze/src/airflow_breeze/utils/python_versions.py
b/dev/breeze/src/airflow_breeze/utils/python_versions.py
index d144139b068..4f5a7a00bb5 100644
--- a/dev/breeze/src/airflow_breeze/utils/python_versions.py
+++ b/dev/breeze/src/airflow_breeze/utils/python_versions.py
@@ -51,8 +51,9 @@ def check_python_version(release_provider_packages: bool =
False):
get_console().print(
"[warning]Please reinstall Breeze using Python 3.9 - 3.11
environment because not all "
"provider packages support Python 3.12 yet.[/]\n\n"
- "For example:\n\n"
- "pipx uninstall apache-airflow-breeze\n"
- "pipx install --python $(which python3.9) -e ./dev/breeze
--force\n"
+ "If you are using uv:\n\n"
+ " uv tool install --force --reinstall --python 3.9 -e
./dev/breeze\n\n"
+ "If you are using pipx:\n\n"
+ " pipx install --python $(which python3.9) --force -e
./dev/breeze\n"
)
sys.exit(1)
diff --git a/dev/breeze/src/airflow_breeze/utils/reinstall.py
b/dev/breeze/src/airflow_breeze/utils/reinstall.py
index de3da928554..6165c8a3072 100644
--- a/dev/breeze/src/airflow_breeze/utils/reinstall.py
+++ b/dev/breeze/src/airflow_breeze/utils/reinstall.py
@@ -27,15 +27,24 @@ from airflow_breeze.utils.console import get_console
def reinstall_breeze(breeze_sources: Path, re_run: bool = True):
"""
- Reinstalls Breeze from specified sources.
+ Re-installs Breeze from specified sources.
:param breeze_sources: Sources where to install Breeze from.
:param re_run: whether to re-run the original command that breeze was run
with.
"""
+ # First check if `breeze` is installed with uv and if it is, reinstall it
using uv
+ # If not - we assume pipx is used and we reinstall it using pipx
# Note that we cannot use `pipx upgrade` here because we sometimes install
# Breeze from different sources than originally installed (i.e. when we
reinstall airflow
# From the current directory.
get_console().print(f"\n[info]Reinstalling Breeze from {breeze_sources}\n")
- subprocess.check_call(["pipx", "install", "-e", str(breeze_sources),
"--force"])
+ result = subprocess.run(["uv", "tool", "list"], text=True,
capture_output=True, check=False)
+ if result.returncode == 0:
+ if "apache-airflow-breeze" in result.stdout:
+ subprocess.check_call(
+ ["uv", "tool", "install", "--force", "--reinstall", "-e",
breeze_sources.as_posix()]
+ )
+ else:
+ subprocess.check_call(["pipx", "install", "-e",
breeze_sources.as_posix(), "--force"])
if re_run:
# Make sure we don't loop forever if the metadata hash hasn't been
updated yet (else it is tricky to
# run pre-commit checks via breeze!)
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py
b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 2e828936aa6..8396c0016de 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -219,14 +219,14 @@ def assert_pre_commit_installed():
python_executable = sys.executable
get_console().print(f"[info]Checking pre-commit installed for
{python_executable}[/]")
command_result = run_command(
- [python_executable, "-m", "pre_commit", "--version"],
+ ["pre-commit", "--version"],
capture_output=True,
text=True,
check=False,
)
if command_result.returncode == 0:
if command_result.stdout:
- pre_commit_version = command_result.stdout.split(" ")[-1].strip()
+ pre_commit_version = command_result.stdout.split(" ")[1].strip()
if Version(pre_commit_version) >= Version(min_pre_commit_version):
get_console().print(
f"\n[success]Package pre_commit is installed. "
@@ -238,6 +238,20 @@ def assert_pre_commit_installed():
f"aat least {min_pre_commit_version} and is
{pre_commit_version}.[/]\n\n"
)
sys.exit(1)
+ if "pre-commit-uv" not in command_result.stdout:
+ get_console().print(
+ "\n[warning]You can significantly improve speed of
installing your pre-commit envs "
+ "by installing `pre-commit-uv` with it.[/]\n"
+ )
+ get_console().print(
+ "\n[warning]With uv you can install it with:[/]\n\n"
+ " uv tool install pre-commit --with pre-commit-uv
--force-reinstall\n"
+ )
+ get_console().print(
+ "\n[warning]With pipx you can install it with:[/]\n\n"
+ " pipx inject\n"
+ " pipx inject pre-commit pre-commit-uv\n"
+ )
else:
get_console().print(
"\n[warning]Could not determine version of pre-commit. You
might need to update it![/]\n"
@@ -459,9 +473,7 @@ def run_compile_www_assets(
"[info]However, it requires you to have local yarn installation.\n"
)
command_to_execute = [
- sys.executable,
- "-m",
- "pre_commit",
+ "pre-commit",
"run",
"--hook-stage",
"manual",
@@ -512,9 +524,7 @@ def run_compile_ui_assets(
"[info]However, it requires you to have local pnpm installation.\n"
)
command_to_execute = [
- sys.executable,
- "-m",
- "pre_commit",
+ "pre-commit",
"run",
"--hook-stage",
"manual",
diff --git a/scripts/ci/pre_commit/common_precommit_utils.py
b/scripts/ci/pre_commit/common_precommit_utils.py
index c3cb40ffeb5..b8d29410db4 100644
--- a/scripts/ci/pre_commit/common_precommit_utils.py
+++ b/scripts/ci/pre_commit/common_precommit_utils.py
@@ -118,8 +118,11 @@ def initialize_breeze_precommit(name: str, file: str):
if shutil.which("breeze") is None:
console.print(
"[red]The `breeze` command is not on path.[/]\n\n"
- "[yellow]Please install breeze with `pipx install -e ./dev/breeze`
from Airflow sources "
- "and make sure you run `pipx ensurepath`[/]\n\n"
+ "[yellow]Please install breeze.\n"
+ "You can use uv with `uv tool install -e ./dev/breeze or "
+ "`pipx install -e ./dev/breeze`.\n"
+ "It will install breeze from Airflow sources "
+ "(make sure you run `pipx ensurepath` if you use pipx)[/]\n\n"
"[bright_blue]You can also set SKIP_BREEZE_PRE_COMMITS env
variable to non-empty "
"value to skip all breeze tests."
)
