This is an automated email from the ASF dual-hosted git repository.
vatsrahul1001 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 91508249689d955af02f7aeadc6b316821e23baa
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 29 16:59:59 2026 +0800
[v3-4-test] Run the slow root prek checks in parallel (#75437) (#75933)
Co-authored-by: PoAn Yang <[email protected]>
Signed-off-by: PoAn Yang <[email protected]>
---
.pre-commit-config.yaml | 50 +++++++++--
contributing-docs/08_static_code_checks.rst | 6 ++
scripts/ci/prek/AGENTS.md | 30 ++++++-
scripts/ci/prek/check_prek_parallel_hooks.py | 97 ++++++++++++++++++++++
.../ci/prek/test_check_prek_parallel_hooks.py | 75 +++++++++++++++++
5 files changed, 249 insertions(+), 9 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 92e716ce9b3..8a470cbdf5e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,6 +27,8 @@ default_env:
# the committed ones. Hooks in nested configs do not inherit this and must
pass --frozen to uv themselves.
UV_FROZEN: "1"
exclude: ^.*/.*_vendor/
+# Hooks with `priority: 1000` run in parallel, after every other hook in this
file. Only the hooks
+# listed in scripts/ci/prek/check_prek_parallel_hooks.py may set a priority.
repos:
- repo: meta
hooks:
@@ -251,6 +253,19 @@ repos:
^scripts/ci/license-templates/
- repo: local
hooks:
+ # Hooks with `priority: 1000` start in the order they appear in this
file. This hook takes the
+ # longest of them, so it comes first. Because it needs the CI image, it
would normally go in the
+ # CI image section at the end of this file. Its priority still makes it
run after every hook that
+ # has no priority.
+ - id: check-template-fields-valid
+ name: Check templated fields mapped in operators/sensors
+ env:
+ PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
+ language: python
+ entry: ./scripts/ci/prek/check_template_fields.py
+ files:
^(providers/[^/]+|airflow-core)/src/airflow/(.*/)?(sensors|operators)/.*\.py$
+ require_serial: true
- id: check-min-python-version
name: Check minimum Python version
env:
@@ -739,10 +754,12 @@ repos:
name: Detect accidentally committed debug statements
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
- id: check-builtin-literals
name: Require literal syntax when initializing builtins
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
- id: detect-private-key
name: Detect if private key is added to the repository
env:
@@ -795,6 +812,7 @@ repos:
name: Check if there are no deprecate log warn
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
- repo: https://github.com/adrienverge/yamllint
rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # frozen: v1.38.0
hooks:
@@ -802,6 +820,7 @@ repos:
name: Check YAML files with yamllint
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: yamllint -c yamllint-config.yml --strict
types: [yaml]
exclude: >
@@ -841,6 +860,7 @@ repos:
name: Run codespell
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
description: Run codespell to check for common misspellings in files
entry: bash -c 'echo "If you think that this failure is an error,
consider adding the word(s)
to the codespell dictionary at docs/spelling_wordlist.txt.
@@ -871,6 +891,7 @@ repos:
name: Run zizmor to check for github workflow syntax errors
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
types: [yaml]
files: ^\.github/workflows/.*$|^\.github/actions/.*$
require_serial: true
@@ -888,6 +909,7 @@ repos:
name: Check internal Markdown links with lychee (offline mode)
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
description: >-
Validate intra-repo links in Markdown files (relative paths, anchors,
and fragments) without making any HTTP requests. Auto-generated
client
@@ -956,6 +978,7 @@ repos:
name: Check for test-only imports in production source
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/check_test_only_imports_in_src.py
language: python
pass_filenames: true
@@ -1028,6 +1051,7 @@ repos:
name: Lint Dockerfile
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
language: python
entry: ./scripts/ci/prek/lint_dockerfile.py
files: Dockerfile.*$
@@ -1074,6 +1098,7 @@ repos:
name: Prevent deprecated sqlalchemy usage
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py
language: python
files: >
@@ -1202,6 +1227,7 @@ repos:
- id: check-for-inclusive-language
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
language: pygrep
name: Check for language that we do not accept as community
description: Please use more appropriate words for community
documentation.
@@ -1415,6 +1441,7 @@ repos:
name: Check if licenses are OK for Apache
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/check_license.py
language: python
files: ^LICENSE$
@@ -1423,6 +1450,7 @@ repos:
name: Check that metrics in the codebase are in sync with the metrics
registry YAML file.
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/check_metrics_synced_with_the_registry.py
language: python
files:
\.py$|^shared/observability/src/airflow_shared/observability/metrics/metrics_template\.yaml$
@@ -1469,6 +1497,7 @@ repos:
name: Check Shell scripts syntax correctness
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
language: docker_image
entry: koalaman/shellcheck:v0.8.0 -x -a
files: \.(bash|sh)$|^hooks/build$|^hooks/push$
@@ -1692,6 +1721,7 @@ repos:
name: Run markdownlint
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
description: Checks the style of Markdown files.
entry: markdownlint
language: node
@@ -1856,6 +1886,7 @@ repos:
name: Check that all logging methods are lazy
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/check_lazy_logging.py
language: python
pass_filenames: true
@@ -1864,6 +1895,7 @@ repos:
name: Check XComModel columns are read through xcom_entity, not
directly in SQLAlchemy queries
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/check_xcom_model_columns.py
language: python
pass_filenames: true
@@ -1910,6 +1942,7 @@ repos:
name: bandit
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
description: "Bandit is a tool for finding common security issues in
Python code"
entry: bandit
language: python
@@ -1979,6 +2012,14 @@ repos:
entry: ./scripts/ci/prek/check_prek_hooks_pycache_prefix.py
pass_filenames: false
files:
(^|/)\.pre-commit-config\.yaml$|^scripts/ci/prek/check_prek_hooks_pycache_prefix\.py$
+ - id: check-prek-parallel-hooks
+ name: Only audited hooks run in parallel in the root prek config
+ env:
+ PYTHONPYCACHEPREFIX: .build/pycache
+ language: python
+ entry: ./scripts/ci/prek/check_prek_parallel_hooks.py
+ pass_filenames: false
+ files:
^\.pre-commit-config\.yaml$|^scripts/ci/prek/check_prek_parallel_hooks\.py$
- id: check-migration-patterns
name: Check migration files for anti-patterns and style (MIG001-MIG004)
env:
@@ -1992,14 +2033,6 @@ repos:
## ADD MOST PREK HOOK ABOVE THAT LINE
# The below prek hooks are those requiring CI image to be built
## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
- - id: check-template-fields-valid
- name: Check templated fields mapped in operators/sensors
- env:
- PYTHONPYCACHEPREFIX: .build/pycache
- language: python
- entry: ./scripts/ci/prek/check_template_fields.py
- files:
^(providers/[^/]+|airflow-core)/src/airflow/(.*/)?(sensors|operators)/.*\.py$
- require_serial: true
- id: check-execution-api-versions
name: Check execution API datamodel changes have corresponding version
updates
env:
@@ -2073,6 +2106,7 @@ repos:
name: Check for problematic context manager class decorators
env:
PYTHONPYCACHEPREFIX: .build/pycache
+ priority: 1000
entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py
language: python
files: .*test.*\.py$
diff --git a/contributing-docs/08_static_code_checks.rst
b/contributing-docs/08_static_code_checks.rst
index 11eee3ebb53..91bd8f22f7b 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -161,6 +161,12 @@ After installation, prek hooks are run automatically when
you commit the
code or push it to the repository (depending on stages configured for the
hooks). Some of the
hooks are configured to run on "manual" stage only and are not run
automatically.
+The hooks of the root ``.pre-commit-config.yaml`` run one at a time, except
for some slow checks
+that set ``priority: 1000``. Those checks run in parallel at the end. Before
you set that priority
+on another hook, read the rules in ``scripts/ci/prek/AGENTS.md``.
``check-prek-parallel-hooks``
+fails when a hook sets a priority other than 1000, or sets 1000 but is not
listed in
+``PARALLEL_HOOKS`` in ``scripts/ci/prek/check_prek_parallel_hooks.py``.
+
By default when you run ``prek``, the ``pre-commit`` stage hooks are run.
But you can run prek hooks manually as needed.
diff --git a/scripts/ci/prek/AGENTS.md b/scripts/ci/prek/AGENTS.md
index d8b605f6684..64427787af2 100644
--- a/scripts/ci/prek/AGENTS.md
+++ b/scripts/ci/prek/AGENTS.md
@@ -22,7 +22,9 @@ When adding a new breeze-dependent hook:
1. Import and use `run_command_via_breeze_run` from `common_prek_utils` — do
not shell out
to `breeze` directly.
2. Register the hook at the **end** of the relevant `.pre-commit-config.yaml`
file (breeze
- hooks are slow and should run after fast, local checks).
+ hooks are slow and should run after fast, local checks). A hook in the
`priority: 1000` group
+ described below runs after the other hooks no matter where it sits in the
file, so this rule
+ does not apply to that hook.
## Adding new hooks
@@ -36,3 +38,29 @@ When adding a new breeze-dependent hook:
breeze deletes `__pycache__` directories on start-up, so a hook writing
`.pyc` next to the
sources can make a concurrent breeze hook fail with `Directory not empty`.
`check-prek-hooks-pycache-prefix` enforces this.
+- In `/.pre-commit-config.yaml`, the hooks with `priority: 1000` form one
group. prek runs them in
+ parallel, after every other hook in the file. Leave `priority:` unset on a
hook unless it joins
+ this group. A hook may join the group only if it follows all of these rules:
+ - It never changes a tracked file, not even when it finds drift. prek runs
`git diff` before and
+ after the group. When tracked files change, prek fails the whole group and
does not say which
+ hook changed them.
+ - It writes untracked files only where no other hook in the group reads
them, for example under
+ `.build/`.
+ - It does not change the git index, refs or worktrees, and it does not use
fixed Docker
+ container, network or volume names.
+ - It shares a virtualenv or cache with another hook only if the tool that
writes the virtualenv
+ or cache locks it or writes each file atomically. For example, uv locks a
virtualenv while
+ installing into it, and Python writes each `.pyc` file atomically.
+ - Unless the hook is `check-template-fields-valid`, it does not run Breeze
and it does not walk a
+ directory tree with `Path.rglob()` on Python 3.11.
`check-template-fields-valid` is the only
+ hook in the group that runs Breeze, and Breeze deletes `__pycache__`
directories at startup.
+ On Python 3.11, `rglob()` fails when a directory disappears during the
walk. So while
+ `check-template-fields-valid` runs, a hook that walks a tree with
`rglob()` on Python 3.11 can
+ fail. `check-template-fields-valid` itself walks `providers/` with
`rglob()` on Python 3.11
+ inside the Breeze container, so a second hook that runs Breeze can make
that walk fail.
+
+ To add a hook to the group, set `priority: 1000` on it and list it in
`PARALLEL_HOOKS` in
+ `check_prek_parallel_hooks.py`. The `check-prek-parallel-hooks` hook fails
if a hook in
+ `/.pre-commit-config.yaml` sets a priority other than 1000, or sets 1000 but
is not listed in
+ `PARALLEL_HOOKS`. If you change a listed hook so that it breaks one of the
rules above, take it
+ out of the group.
diff --git a/scripts/ci/prek/check_prek_parallel_hooks.py
b/scripts/ci/prek/check_prek_parallel_hooks.py
new file mode 100755
index 00000000000..1042766cdd4
--- /dev/null
+++ b/scripts/ci/prek/check_prek_parallel_hooks.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+# "pyyaml>=6.0.3",
+# ]
+# ///
+"""Fail when a hook of the root prek config sets a priority that is not
allowed.
+
+prek runs the hooks that share a ``priority`` value in parallel. In the root
+``.pre-commit-config.yaml``, only the hooks in ``PARALLEL_HOOKS`` may set a
priority, and they
+must all use ``PARALLEL_PRIORITY``. ``scripts/ci/prek/AGENTS.md`` lists the
rules that a hook must
+follow before it can be added to ``PARALLEL_HOOKS``.
+
+The check rejects every value other than ``PARALLEL_PRIORITY`` because hooks
that set no priority
+get their position in the file, counted from 0, as their priority. For
example, a hook that sets
+``50`` would run together with the 51st hook.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import yaml
+
+ROOT_CONFIG = Path(".pre-commit-config.yaml")
+PARALLEL_PRIORITY = 1000
+PARALLEL_HOOKS = frozenset(
+ {
+ "bandit",
+ "check-apache-license-rat",
+ "check-builtin-literals",
+ "check-contextmanager-class-decorators",
+ "check-for-inclusive-language",
+ "check-lazy-logging",
+ "check-metrics-synced-with-registry",
+ "check-template-fields-valid",
+ "check-test-only-imports-in-src",
+ "check-xcom-model-columns",
+ "codespell",
+ "debug-statements",
+ "lint-dockerfile",
+ "lint-markdown",
+ "lychee-docker",
+ "prevent-deprecated-sqlalchemy-usage",
+ "python-no-log-warn",
+ "shellcheck",
+ "yamllint",
+ "zizmor",
+ }
+)
+
+
+def find_unaudited_priority_hooks(config_path: Path) -> list[str]:
+ config = yaml.safe_load(config_path.read_text()) or {}
+ return [
+ f"{hook['id']} (priority {hook['priority']!r})"
+ for repo in config.get("repos", [])
+ for hook in repo.get("hooks", [])
+ if "priority" in hook and (hook["priority"] != PARALLEL_PRIORITY or
hook["id"] not in PARALLEL_HOOKS)
+ ]
+
+
+def main() -> int:
+ errors = find_unaudited_priority_hooks(ROOT_CONFIG)
+ if errors:
+ print(f"ERROR: These hooks in {ROOT_CONFIG} set a priority that is not
allowed:\n")
+ print("\n".join(f" {error}" for error in errors))
+ print(
+ "\nOnly the hooks listed in PARALLEL_HOOKS in
scripts/ci/prek/check_prek_parallel_hooks.py"
+ f" may set a priority, and the value must be {PARALLEL_PRIORITY}.
Those hooks run in parallel."
+ " Before you add a hook to PARALLEL_HOOKS, check that it follows
the rules in"
+ " scripts/ci/prek/AGENTS.md."
+ )
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/ci/prek/test_check_prek_parallel_hooks.py
b/scripts/tests/ci/prek/test_check_prek_parallel_hooks.py
new file mode 100644
index 00000000000..d56cf844a3d
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_prek_parallel_hooks.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+from check_prek_parallel_hooks import PARALLEL_HOOKS,
find_unaudited_priority_hooks, main
+
+AUDITED_HOOK = sorted(PARALLEL_HOOKS)[0]
+
+CONFIG = f"""\
+repos:
+ - repo: meta
+ hooks:
+ - id: no-priority
+ - repo: local
+ hooks:
+ - id: {AUDITED_HOOK}
+ priority: 1000
+ - id: not-audited
+ priority: 1000
+ - id: {AUDITED_HOOK}
+ priority: 50
+ - id: alias
+ priority: checks
+"""
+
+
+def test_find_unaudited_priority_hooks(tmp_path):
+ config = tmp_path / ".pre-commit-config.yaml"
+ config.write_text(CONFIG)
+
+ assert find_unaudited_priority_hooks(config) == [
+ "not-audited (priority 1000)",
+ f"{AUDITED_HOOK} (priority 50)",
+ "alias (priority 'checks')",
+ ]
+
+
+def test_find_unaudited_priority_hooks_empty_config(tmp_path):
+ config = tmp_path / ".pre-commit-config.yaml"
+ config.write_text("")
+
+ assert find_unaudited_priority_hooks(config) == []
+
+
[email protected](
+ ("config", "expected_returncode"),
+ [
+ pytest.param(CONFIG, 1, id="unaudited-priority"),
+ pytest.param(
+ f"repos:\n - repo: local\n hooks:\n - id:
{AUDITED_HOOK}\n priority: 1000\n",
+ 0,
+ id="audited-priority",
+ ),
+ ],
+)
+def test_main(tmp_path, monkeypatch, config, expected_returncode):
+ (tmp_path / ".pre-commit-config.yaml").write_text(config)
+ monkeypatch.chdir(tmp_path)
+
+ assert main() == expected_returncode
