This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch backport-ci-sync-v3-4-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f2d690d980d00244421babcdbf8a18be59795865 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Oct 25 19:39:32 2026 +0100 Stop parallel prek hooks racing on __pycache__ cleanup (#73823) * Stop parallel prek hooks racing on __pycache__ cleanup prek runs hooks in parallel and every breeze invocation removes all __pycache__ directories in the checkout on start-up. Any other hook process importing from the source tree (breeze itself, prek hook scripts, local mypy runs) can write a fresh .pyc between rmtree emptying a directory and removing it, which fails the breeze hook with "OSError: [Errno 39] Directory not empty" - a flaky Static checks failure in CI. Redirecting bytecode of every hook into a hidden, gitignored .build/ directory keeps the source tree free of new .pyc files while hooks run, and a new check keeps future hooks from reintroducing the race. The cleanup additionally tolerates ENOTEMPTY for writers prek does not control, such as an IDE. Generated-by: Claude Opus 5 * Set PYTHONPYCACHEPREFIX on the hooks added since the branch was cut Eight hooks that landed on main after this branch was created did not redirect their bytecode, so the new check-prek-hooks-pycache-prefix hook failed after the rebase. Generated-by: Claude Opus 5 * Update scripts/ci/prek/check_prek_hooks_pycache_prefix.py Co-authored-by: Wei Lee <[email protected]> --------- Co-authored-by: Wei Lee <[email protected]> (cherry picked from commit 0cde487d422e86032128d459d3d1a317e3547d47) --- .pre-commit-config.yaml | 324 ++++++++++++++++++++- airflow-core/.pre-commit-config.yaml | 62 ++++ airflow-ctl-tests/.pre-commit-config.yaml | 4 + airflow-ctl/.pre-commit-config.yaml | 8 + airflow-e2e-tests/.pre-commit-config.yaml | 2 + chart/.pre-commit-config.yaml | 28 ++ dev/.pre-commit-config.yaml | 2 + dev/breeze/src/airflow_breeze/utils/path_utils.py | 6 + dev/breeze/tests/test_path_utils.py | 51 ++++ dev/mypy/.pre-commit-config.yaml | 4 + docker-tests/.pre-commit-config.yaml | 2 + go-sdk/.pre-commit-config.yaml | 14 + kubernetes-tests/.pre-commit-config.yaml | 2 + providers/.pre-commit-config.yaml | 60 ++++ providers/amazon/.pre-commit-config.yaml | 2 + providers/common/ai/.pre-commit-config.yaml | 6 + providers/common/compat/.pre-commit-config.yaml | 2 + providers/edge3/.pre-commit-config.yaml | 12 + providers/fab/.pre-commit-config.yaml | 12 + providers/keycloak/.pre-commit-config.yaml | 4 + scripts/.pre-commit-config.yaml | 2 + scripts/ci/prek/AGENTS.md | 4 + scripts/ci/prek/check_prek_hooks_pycache_prefix.py | 76 +++++ scripts/ci/prek/check_shared_mypy_hooks.py | 2 + .../prek/test_check_prek_hooks_pycache_prefix.py | 51 ++++ shared/configuration/.pre-commit-config.yaml | 2 + shared/dagnode/.pre-commit-config.yaml | 2 + shared/listeners/.pre-commit-config.yaml | 2 + shared/logging/.pre-commit-config.yaml | 2 + shared/module_loading/.pre-commit-config.yaml | 2 + shared/observability/.pre-commit-config.yaml | 2 + shared/plugins_manager/.pre-commit-config.yaml | 2 + shared/providers_discovery/.pre-commit-config.yaml | 2 + shared/secrets_backend/.pre-commit-config.yaml | 2 + shared/secrets_masker/.pre-commit-config.yaml | 2 + shared/serialization/.pre-commit-config.yaml | 2 + shared/state/.pre-commit-config.yaml | 2 + shared/template_rendering/.pre-commit-config.yaml | 2 + shared/timezones/.pre-commit-config.yaml | 2 + task-sdk-integration-tests/.pre-commit-config.yaml | 2 + task-sdk/.pre-commit-config.yaml | 6 + ts-sdk/.pre-commit-config.yaml | 14 + 42 files changed, 789 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e4e2302cb..036fbd26746 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,14 +32,20 @@ repos: hooks: - id: identity name: Print checked files + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Print input to the static check hooks for troubleshooting - id: check-hooks-apply name: Check if all hooks apply to the repository + env: + PYTHONPYCACHEPREFIX: .build/pycache - repo: https://github.com/thlorenz/doctoc.git rev: 59dcae910a6ad4bbe8711275b854e3cd5c30f08b # frozen: v2.5.0 hooks: - id: doctoc name: Add TOC for Markdown and RST files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: (?x) ^README\.md$| @@ -63,6 +69,8 @@ repos: hooks: - id: insert-license name: Add license for all SQL files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: \.sql$ exclude: | (?x) @@ -79,6 +87,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all RST files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --comment-style - "||" @@ -93,6 +103,8 @@ repos: ^scripts/ci/license-templates/ - id: insert-license name: Add license for CSS/JS/JSX/PUML/TS/TSX + env: + PYTHONPYCACHEPREFIX: .build/pycache files: \.(css|jsx?|puml|tsx?)$ exclude: ^\.github/.*$|ui/openapi-gen/|www/openapi-gen/|.*/dist/.* args: @@ -103,6 +115,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Shell files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$ files: \.bash$|\.sh$ args: @@ -113,6 +127,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all toml files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$|^dev/breeze/autocomplete/.*$ files: \.toml$ args: @@ -123,6 +139,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Python files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$|^.*/_vendor/.*$|^airflow-ctl/.*/.*generated\.py$ files: \.py$|\.pyi$ args: @@ -133,6 +151,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all XML files + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^\.github/.*$ files: \.xml$ args: @@ -143,6 +163,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all YAML files except Helm templates + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^\.github/.*$| @@ -162,6 +184,8 @@ repos: - --fuzzy-match-generates-todo - id: insert-license name: Add license for all Markdown files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: \.md$ args: - --comment-style @@ -180,6 +204,8 @@ repos: ^scripts/ci/license-templates/ - id: insert-license name: Add short license for agentic Markdown files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --comment-style - "||" @@ -201,6 +227,8 @@ repos: ^scripts/ci/license-templates/ - id: insert-license name: Add license for all other files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --comment-style - "|#|" @@ -225,11 +253,15 @@ repos: hooks: - id: check-min-python-version name: Check minimum Python version + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_min_python_version.py language: python require_serial: true - id: update-notice-year name: Update NOTICE files to current year (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_notice_year.py stages: ['manual'] language: python @@ -237,11 +269,15 @@ repos: require_serial: true - id: check-notice-files name: Check NOTICE files for current year and ASF references + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_notice_files.py language: python files: ^.*NOTICE$ - id: check-version-consistency name: Check version consistency + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_version_consistency.py language: python files: > @@ -254,6 +290,8 @@ repos: require_serial: true - id: check-applies-to-roots-in-sync name: Check plugin applies_to destination tables stay in sync + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "The UI mirrors plugins_manager's destination -> record map; drift silently widens plugin scope" entry: ./scripts/ci/prek/check_applies_to_roots_in_sync.py language: python @@ -265,6 +303,8 @@ repos: require_serial: true - id: check-ci-workflows-in-sync name: Check ci-arm.yml and ci-amd.yml stay in sync + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_ci_workflows_in_sync.py language: python files: ^\.github/workflows/ci-(arm|amd)\.yml$ @@ -272,6 +312,8 @@ repos: require_serial: true - id: check-java-sdk-version-in-sync name: Check Java SDK JDK version is consistent across build files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_java_sdk_version_in_sync.py language: python files: > @@ -300,6 +342,8 @@ repos: pass_filenames: false - id: sync-java-sdk-supervisor-schema name: Sync Java SDK supervisor schema with configured version + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Download the Supervisor Schema when Java SDK's bundled schema.json drifts" entry: ./scripts/ci/prek/sync_java_sdk_supervisor_schema.py language: python @@ -310,6 +354,8 @@ repos: ^java-sdk/sdk/schema/schema\.json$ - id: regenerate-java-sdk-verification-metadata name: Regenerate the Java SDK dependency verification metadata + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Drop trusted checksums that no Java SDK build task resolves any more" entry: ./scripts/ci/prek/regenerate_java_sdk_verification_metadata.py language: python @@ -329,6 +375,8 @@ repos: ^java-sdk/(?!example/|scala_spark_example/)[^/]+/(?:build\.gradle\.kts|gradle\.properties)$ - id: update-java-sdk-readme-matrix name: Update the Java SDK compatibility matrix in java-sdk/README.md and Dokka module doc + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_java_sdk_readme_matrix.py language: python files: > @@ -344,6 +392,8 @@ repos: require_serial: true - id: update-ts-sdk-readme-matrix name: Update the TypeScript SDK compatibility matrix in ts-sdk/README.md + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_ts_sdk_readme_matrix.py language: python files: > @@ -358,6 +408,8 @@ repos: require_serial: true - id: sync-ts-sdk-schemas name: Sync TypeScript SDK vendored schemas with the distributions that own them + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Copy airflow-core's Dag schema and the Task SDK's supervisor schema when TS SDK's vendored copies drift" entry: ./scripts/ci/prek/sync_ts_sdk_schemas.py language: python @@ -374,6 +426,8 @@ repos: ^scripts/ci/prek/sync_ts_sdk_schemas\.py$ - id: sync-java-sdk-dag-schema name: Sync Java SDK Dag serialization schema with airflow-core + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Copy airflow-core's serialization schema when Java SDK's vendored dag-schema.json drifts" entry: ./java-sdk/gradlew -p ./java-sdk :sdk:syncDagSchema -PfailOnDagSchemaDrift language: system @@ -391,6 +445,8 @@ repos: ^java-sdk/sdk/schema/dag-schema\.json$ - id: sync-go-sdk-schemas name: Sync Go SDK vendored schemas with the distributions that own them + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Copy airflow-core's Dag schema and task-sdk's supervisor schema when go-sdk's vendored copies drift" entry: ./scripts/ci/prek/sync_go_sdk_schemas.py language: python @@ -410,6 +466,8 @@ repos: ^scripts/ci/prek/sync_go_sdk_schemas\.py$ - id: check-go-sdk-generated-drift name: Check Go SDK generated files against the schemas they generate from + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Regenerate the Dag specs and the coordinator-protocol models, and fail when a committed file differs" entry: ./scripts/ci/prek/check_go_sdk_generated_drift.py # golang so prek provisions the toolchain the generators need, the way the @@ -438,6 +496,8 @@ repos: # check when the shared harness or Airflow's serializer changes, which that project cannot see. - id: check-ts-sdk-serialization-conformance-shared name: Check the TS SDK serializes Dags the way Airflow does, after a shared change + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the TS SDK and with Airflow, and compare the two" entry: ./ts-sdk/scripts/ci/prek/check_serialization_conformance.py language: node @@ -451,6 +511,8 @@ repos: ^scripts/ci/lang_sdk_serialization/.*$ - id: check-go-sdk-serialization-conformance name: Check the Go SDK serializes Dags the way Airflow does + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the Go SDK and with Airflow, and compare the two" entry: ./scripts/ci/prek/check_go_sdk_serialization_conformance.py # golang so prek provisions the toolchain that builds the Go serializer. @@ -467,6 +529,8 @@ repos: ^scripts/ci/prek/check_go_sdk_serialization_conformance\.py$ - id: check-java-sdk-serialization-conformance name: Check the Java SDK serializes Dags the way Airflow does + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the Java SDK and with Airflow, and compare the two" entry: ./java-sdk/scripts/ci/prek/check_serialization_conformance.py language: system @@ -486,6 +550,8 @@ repos: ^scripts/ci/lang_sdk_serialization/.*$ - id: check-go-version-in-sync name: Check Go toolchain version is consistent across build files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_go_version_in_sync.py language: python files: > @@ -501,6 +567,8 @@ repos: require_serial: true - id: update-go-sdk-readme-matrix name: Update the Go SDK compatibility matrix in go-sdk/README.md + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_go_sdk_readme_matrix.py language: python files: > @@ -530,6 +598,8 @@ repos: require_serial: true - id: check-go-example-mod-tidy name: Check lang-SDK Go example module is tidy against the Go SDK + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_go_example_mod_tidy.py # golang so prek provisions the toolchain the check needs: static checks run on a # runner whose preinstalled toolchains are wiped to free disk space, and the SDK's @@ -547,6 +617,8 @@ repos: require_serial: true - id: check-partition-mapper-defaults-in-sync name: Check partition-mapper core/SDK sync (FanOutMapper table + SegmentWindow/FixedKeyMapper) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py language: python files: > @@ -561,6 +633,8 @@ repos: require_serial: true - id: check-window-in-sync name: Check Window definitions stay in sync (core/SDK) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_window_in_sync.py language: python files: > @@ -571,6 +645,8 @@ repos: require_serial: true - id: sync-uv-min-version-markers name: Sync `# sync-uv-min-version` markers with [tool.uv] required-version + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sync_uv_min_version_markers.py language: python # Hook reads pyproject.toml itself; the file list selects places that may carry @@ -584,6 +660,8 @@ repos: require_serial: true - id: check-distribution-gitignore name: Check distribution .gitignore files have *.iml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_distribution_gitignore.py language: python files: > @@ -596,9 +674,11 @@ repos: require_serial: true - id: upgrade-important-versions name: Upgrade important versions (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache + UV_FROZEN: "0" entry: ./scripts/ci/prek/upgrade_important_versions.py stages: ['manual'] - env: {UV_FROZEN: "0"} language: python files: > (?x) @@ -609,9 +689,11 @@ repos: require_serial: true - id: upgrade-dependency-floors name: Upgrade curated dependency floors (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache + UV_FROZEN: "0" entry: ./scripts/ci/prek/upgrade_dependency_floors.py stages: ['manual'] - env: {UV_FROZEN: "0"} language: python files: > (?x) @@ -625,6 +707,8 @@ repos: hooks: - id: blacken-docs name: Run black on docs + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --line-length=110 - --target-version=py311 @@ -641,15 +725,25 @@ repos: hooks: - id: check-merge-conflict name: Check that merge conflicts are not being committed + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: debug-statements name: Detect accidentally committed debug statements + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: check-builtin-literals name: Require literal syntax when initializing builtins + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: detect-private-key name: Detect if private key is added to the repository + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: ^providers/ssh/docs/connections/ssh\.rst$ - id: end-of-file-fixer name: Make sure that there is an empty line at the end + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^airflow-core/docs/img/.*\.dot| @@ -659,15 +753,23 @@ repos: .*/openapi-gen/.* - id: mixed-line-ending name: Detect if mixed line ending is used (\r vs. \r\n) + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: check-executables-have-shebangs name: Check that executables have shebang + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: check-xml name: Check XML files with xmllint + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^scripts/ci/docker-compose/gremlin/. - id: trailing-whitespace name: Remove trailing whitespace at end of line + env: + PYTHONPYCACHEPREFIX: .build/pycache exclude: > (?x) ^airflow-core/docs/img/.*\.dot$| @@ -679,13 +781,19 @@ repos: hooks: - id: rst-backticks name: Check if RST files use double backticks for code + env: + PYTHONPYCACHEPREFIX: .build/pycache - id: python-no-log-warn name: Check if there are no deprecate log warn + env: + PYTHONPYCACHEPREFIX: .build/pycache - repo: https://github.com/adrienverge/yamllint rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # frozen: v1.38.0 hooks: - id: yamllint name: Check YAML files with yamllint + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: yamllint -c yamllint-config.yml --strict types: [yaml] exclude: > @@ -706,6 +814,8 @@ repos: hooks: - id: flynt name: Run flynt string format converter for Python + env: + PYTHONPYCACHEPREFIX: .build/pycache args: # If flynt detects too long text it ignores it. So we set a very large limit to make it easy # to split the text by hand. Too long lines are detected by flake8 (below), @@ -717,6 +827,8 @@ repos: hooks: - id: codespell name: Run codespell + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Run codespell to check for common misspellings in files entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s) to the codespell dictionary at docs/spelling_wordlist.txt. @@ -745,6 +857,8 @@ repos: hooks: - id: zizmor name: Run zizmor to check for github workflow syntax errors + env: + PYTHONPYCACHEPREFIX: .build/pycache types: [yaml] files: ^\.github/workflows/.*$|^\.github/actions/.*$ require_serial: true @@ -760,6 +874,8 @@ repos: # bundles its own libc and is portable across runners. - id: lychee-docker name: Check internal Markdown links with lychee (offline mode) + env: + PYTHONPYCACHEPREFIX: .build/pycache description: >- Validate intra-repo links in Markdown files (relative paths, anchors, and fragments) without making any HTTP requests. Auto-generated client @@ -790,6 +906,8 @@ repos: hooks: - id: check-shared-distributions-structure name: Check shared distributions structure + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_shared_distributions_structure.py language: python pass_filenames: false @@ -802,12 +920,16 @@ repos: ^task-sdk/.*\.py$ - id: check-shared-distributions-usage name: Check shared distributions usage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_shared_distributions_usage.py language: python pass_filenames: false files: ^shared/.*$|^.*/pyproject.toml$|^.*/_shared/.*$ - id: check-airflow-imports-in-shared name: Check for core/sdk imports in shared libraries + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_airflow_imports_in_shared.py language: python pass_filenames: true @@ -820,6 +942,8 @@ repos: ^shared/secrets_backend/src/airflow_shared/secrets_backend/base\.py$ - id: check-test-only-imports-in-src name: Check for test-only imports in production source + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_test_only_imports_in_src.py language: python pass_filenames: true @@ -832,18 +956,24 @@ repos: ^shared/.*/src/.*\.py$ - id: check-secrets-search-path-sync name: Check sync between sdk and core + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_secrets_search_path_sync.py language: python pass_filenames: false files: ^airflow-core/src/airflow/secrets/base_secrets\.py$|^task-sdk/src/airflow/sdk/execution_time/secrets/__init__\.py$ - id: check-registry-types-json-sync name: Check registry types.json in sync with types.py + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_registry_types_json_sync.py language: python pass_filenames: false files: ^dev/registry/registry_tools/types\.py$|^registry/src/_data/types\.json$ - id: ruff name: Run 'ruff' for extremely fast Python linting + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Run 'ruff' for extremely fast Python linting" entry: ruff check --force-exclude language: python @@ -854,6 +984,8 @@ repos: exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$ - id: ruff-format name: Run 'ruff format' + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Run 'ruff format' for extremely fast Python formatting" entry: ./scripts/ci/prek/ruff_format.py language: python @@ -863,6 +995,8 @@ repos: exclude: ^airflow-core/tests/unit/dags/test_imports\.py$ - id: replace-bad-characters name: Replace bad characters + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/replace_bad_characters.py language: python types: [file, text] @@ -880,6 +1014,8 @@ repos: /go\.(mod|sum)$ - id: lint-dockerfile name: Lint Dockerfile + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/lint_dockerfile.py files: Dockerfile.*$ @@ -887,24 +1023,32 @@ repos: require_serial: true - id: check-airflow-bug-report-template name: Sort airflow-bug-report provider list + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^\.github/ISSUE_TEMPLATE/1-airflow_bug_report\.yml$ require_serial: true entry: ./scripts/ci/prek/check_airflow_bug_report_template.py - id: update-local-yml-file name: Update mounts in the local yml file + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/local_yml_mounts.py language: python files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$ pass_filenames: false - id: check-extras-order name: Check order of extras in Dockerfile + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_order_dockerfile_extras.py language: python files: ^Dockerfile$ pass_filenames: false - id: generate-airflow-diagrams name: Generate airflow diagrams + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_airflow_diagrams.py language: python files: > @@ -916,6 +1060,8 @@ repos: pass_filenames: true - id: prevent-deprecated-sqlalchemy-usage name: Prevent deprecated sqlalchemy usage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py language: python files: > @@ -931,12 +1077,16 @@ repos: pass_filenames: true - id: update-supported-versions name: Updates supported versions in documentation + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/supported_versions.py language: python files: ^airflow-core/docs/installation/supported-versions\.rst$|^scripts/ci/prek/supported_versions\.py$|^README\.md$ pass_filenames: false - id: generate-api-permissions-doc name: Generate REST API permission reference documentation + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/extract_permissions.py language: python files: > @@ -948,6 +1098,8 @@ repos: pass_filenames: false - id: generate-fab-permissions-doc name: Generate FAB auth manager REST API permission table + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/fab_permissions_doc.py language: python files: > @@ -963,12 +1115,16 @@ repos: pass_filenames: false - id: update-tested-versions name: Update tested Python/DB/Kubernetes versions in docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_tested_versions.py language: python files: ^airflow-core/docs/installation/prerequisites\.rst$|^README\.md$|^dev/breeze/src/airflow_breeze/global_constants\.py$|^scripts/ci/prek/update_tested_versions\.py$|^scripts/ci/prek/common_prek_utils\.py$ pass_filenames: false - id: check-revision-heads-map name: Check that the REVISION_HEADS_MAP is up-to-date + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_revision_heads_map.py pass_filenames: false @@ -980,17 +1136,23 @@ repos: ^airflow-core/src/airflow/utils/db\.py$ - id: update-version name: Update versions in docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_versions.py language: python files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$ pass_filenames: false - id: check-pydevd-left-in-code + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check for pydevd debug statements accidentally left entry: "pydevd.*settrace\\(" pass_filenames: true files: \.py$ - id: check-safe-filter-usage-in-html + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Don't use safe in templates description: the Safe filter is error-prone, use Markup() in code instead @@ -998,6 +1160,8 @@ repos: files: \.html$ pass_filenames: true - id: check-urlparse-usage-in-code + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Don't use urlparse in code description: urlparse is not recommended, use urlsplit() in code instead @@ -1015,6 +1179,8 @@ repos: files: ^\.github/workflows/ci-amd\.yml$|^scripts/ci/prek/check_ci_gate_needs\.py$ - id: check-auth-manager-mock-targets name: Patch SimpleAuthManager, not BaseAuthManager, for methods it overrides + env: + PYTHONPYCACHEPREFIX: .build/pycache description: The API test client uses SimpleAuthManager, so patching an overridden BaseAuthManager method has no effect entry: ./scripts/ci/prek/check_auth_manager_mock_targets.py language: python @@ -1022,6 +1188,8 @@ repos: files: ^airflow-core/tests/unit/api_fastapi/.*\.py$ exclude: ^airflow-core/tests/unit/api_fastapi/auth/managers/simple/ - id: check-for-inclusive-language + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check for language that we do not accept as community description: Please use more appropriate words for community documentation. @@ -1133,6 +1301,8 @@ repos: ^scripts/ci/prek/vendor_k8s_json_schema\.py$ - id: check-template-context-variable-in-sync name: Sync template context variable refs + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_template_context_variable_in_sync.py files: @@ -1141,6 +1311,8 @@ repos: ^task-sdk/src/airflow/sdk/definitions/context\.py$| ^airflow-core/docs/templates-ref\.rst$ - id: check-base-operator-usage + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check BaseOperator core imports description: Make sure BaseOperator is imported from airflow.models.baseoperator in core @@ -1154,6 +1326,8 @@ repos: ^airflow-core/src/airflow/operators/.*$| ^providers/.*$ - id: check-base-operator-usage + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check BaseOperatorLink core imports description: Make sure BaseOperatorLink is not imported from airflow.models in core @@ -1168,6 +1342,8 @@ repos: ^providers/.*/src/airflow/providers/.*$| ^providers/.*/src/airflow/providers/standard/sensors/.*$ - id: check-core-deprecation-classes + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Verify usage of Airflow deprecation classes in core entry: category=DeprecationWarning|category=PendingDeprecationWarning @@ -1183,6 +1359,8 @@ repos: ^devel-common/ pass_filenames: true - id: check-provide-create-sessions-imports + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check session util imports description: NEW_SESSION, provide_session, and create_session should be imported from airflow.utils.session to avoid import cycles. @@ -1190,12 +1368,16 @@ repos: files: \.py$ pass_filenames: true - id: check-incorrect-use-of-LoggingMixin + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Make sure LoggingMixin is not used alone entry: "LoggingMixin\\(\\)" files: \.py$ pass_filenames: true - id: check-start-date-not-used-in-defaults + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: start_date not in default_args entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):" @@ -1203,24 +1385,32 @@ repos: pass_filenames: true - id: check-example-dag-tags name: Check example DAGs carry the "example" tag + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_example_dag_tags.py language: python files: ^.*/example_dags/.*\.py$ pass_filenames: true - id: check-example-dag-shell-commands name: Check example Dags do not build shell commands from runtime values + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_example_dag_shell_commands.py language: python files: ^.*/example_dags/.*\.py$ pass_filenames: true - id: check-apache-license-rat name: Check if licenses are OK for Apache + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_license.py language: python files: ^LICENSE$ pass_filenames: false - id: check-metrics-synced-with-registry name: Check that metrics in the codebase are in sync with the metrics registry YAML file. + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_metrics_synced_with_the_registry.py language: python files: \.py$|^shared/observability/src/airflow_shared/observability/metrics/metrics_template\.yaml$ @@ -1230,12 +1420,16 @@ repos: additional_dependencies: ["PyYAML>=6.0", "rich>=13.6.0"] - id: check-boring-cyborg-configuration name: Checks for Boring Cyborg configuration consistency + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/boring_cyborg.py pass_filenames: false require_serial: true - id: update-in-the-wild-to-be-sorted name: Sort INTHEWILD.md alphabetically + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sort_in_the_wild.py language: python files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$ @@ -1243,6 +1437,8 @@ repos: require_serial: true - id: update-installed-providers-to-be-sorted name: Sort and uniquify installed_providers.txt + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sort_installed_providers.py language: python files: ^\.pre-commit-config\.yaml$|^.*_installed_providers\.txt$ @@ -1250,6 +1446,8 @@ repos: require_serial: true - id: update-spelling-wordlist-to-be-sorted name: Sort spelling_wordlist.txt + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sort_spelling_wordlist.py language: python files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$ @@ -1257,12 +1455,16 @@ repos: pass_filenames: false - id: shellcheck name: Check Shell scripts syntax correctness + env: + PYTHONPYCACHEPREFIX: .build/pycache language: docker_image entry: koalaman/shellcheck:v0.8.0 -x -a files: \.(bash|sh)$|^hooks/build$|^hooks/push$ exclude: ^dev/breeze/autocomplete/.*$ - id: check-integrations-list-consistent name: Sync integrations list with docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_integrations_list.py language: python files: ^scripts/ci/docker-compose/integration-.*\.yml$|^contributing-docs/testing/integration_tests\.rst$ @@ -1270,6 +1472,8 @@ repos: pass_filenames: false - id: sync-translation-namespaces name: Sync translation namespace file list + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/sync_translation_namespaces.py language: python files: > @@ -1279,6 +1483,8 @@ repos: pass_filenames: false - id: generate-agent-skills name: Sync AGENTS.md commands from contributing docs + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_agent_skills.py language: python files: > @@ -1290,6 +1496,8 @@ repos: additional_dependencies: ['pyyaml', 'rich>=13.6.0'] - id: check-documented-commands name: Check documented breeze and prek commands exist + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_documented_commands.py language: python files: > @@ -1307,6 +1515,8 @@ repos: # its installation there, so update the action alongside any id change. - id: run-skill-eval name: Run skill-eval against AGENTS.md and cases + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./dev/skill-evals/eval.py language: node language_version: '22.22.0' @@ -1321,6 +1531,8 @@ repos: verbose: true - id: run-skill-eval-codex name: Run skill-eval against AGENTS.md and cases (Codex runtime) + env: + PYTHONPYCACHEPREFIX: .build/pycache # Kept separate from run-skill-eval so only contributors who actually run # the Codex arm build an env carrying the Codex CLI binary. promptfoo # bundles its own @openai/codex-sdk, but that build ships a binary whose @@ -1342,6 +1554,8 @@ repos: verbose: true - id: view-skill-eval name: View skill-eval results in browser (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: env PROMPTFOO_CONFIG_DIR=.build/promptfoo promptfoo view language: node language_version: '22.22.0' @@ -1355,6 +1569,8 @@ repos: require_serial: true - id: check-eval-hash name: Verify skill-eval ran on current AGENTS.md and cases + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_eval_hash.py language: python files: > @@ -1365,6 +1581,8 @@ repos: pass_filenames: false - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/update_airflow_pyproject_toml.py files: > @@ -1377,6 +1595,8 @@ repos: require_serial: true - id: check-excluded-provider-markers name: Check excluded-provider python_version and platform markers in pyproject.toml + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_excluded_provider_markers.py files: > @@ -1388,6 +1608,8 @@ repos: additional_dependencies: ['packaging>=25', 'pyyaml', 'rich>=13.6.0'] - id: check-provider-config-fallback-defaults name: Check provider_config_fallback_defaults.cfg is in sync with provider.yaml files + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_provider_config_fallback_defaults.py files: > @@ -1400,6 +1622,8 @@ repos: additional_dependencies: ['pyyaml', 'rich>=13.6.0'] - id: check-dependency-lower-bounds name: Check that dependencies in pyproject.toml have lower bounds + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_dependency_lower_bounds.py files: (^|/)pyproject\.toml$ @@ -1407,12 +1631,16 @@ repos: additional_dependencies: ['packaging>=25', 'rich>=13.6.0'] - id: update-reproducible-source-date-epoch name: Update Source Date Epoch for reproducible builds + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/update_source_date_epoch.py files: ^RELEASE_NOTES\.rst$|^chart/RELEASE_NOTES\.rst$ require_serial: true - id: check-breeze-top-dependencies-limited name: Check top-level breeze deps + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Breeze should have small number of top-level dependencies language: python entry: ./scripts/tools/check_if_limited_dependencies.py @@ -1421,18 +1649,24 @@ repos: require_serial: true - id: check-system-tests-present name: Check if system tests have required segments of code + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_system_tests.py language: python files: ^.*/tests/system/.*/example_[^/]*\.py$ pass_filenames: true - id: generate-pypi-readme name: Generate PyPI README + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_pypi_readme.py language: python files: ^README\.md$ pass_filenames: false - id: sync-provider-readme name: Sync provider README.rst Requirements table with pyproject.toml + env: + PYTHONPYCACHEPREFIX: .build/pycache description: | Provider README.rst files are regenerated from pyproject.toml at distribution-build time. Between releases, dependency bumps in pyproject.toml do not propagate to the @@ -1444,6 +1678,8 @@ repos: pass_filenames: true - id: lint-markdown name: Run markdownlint + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Checks the style of Markdown files. entry: markdownlint language: node @@ -1455,6 +1691,8 @@ repos: additional_dependencies: ['[email protected]'] - id: lint-json-schema name: Lint JSON Schema files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-file @@ -1465,6 +1703,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint NodePort Service + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url @@ -1475,6 +1715,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint Docker compose files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/lint_json_schema.py args: - --spec-url @@ -1490,6 +1732,8 @@ repos: require_serial: true - id: check-persist-credentials-disabled-in-github-workflows name: Check persistent creds in workflow files + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check that workflow files have persist-credentials disabled entry: ./scripts/ci/prek/checkout_no_credentials.py language: python @@ -1497,12 +1741,16 @@ repos: files: ^\.github/workflows/.*\.yml$ - id: check-docstring-param-types name: Check that docstrings do not specify param types + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/docstring_param_type.py language: python pass_filenames: true files: \.py$ - id: check-zip-file-is-not-committed name: Check no zip files are committed + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Zip files are not allowed in the repository language: fail entry: | @@ -1511,6 +1759,8 @@ repos: files: \.zip$ - id: update-inlined-dockerfile-scripts name: Inline Dockerfile and Dockerfile.ci scripts + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/inline_scripts_in_docker.py language: python pass_filenames: false @@ -1518,6 +1768,8 @@ repos: require_serial: true - id: update-docker-gpg-keys name: Update Docker build GPG keys (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_docker_gpg_keys.py stages: ['manual'] language: python @@ -1526,12 +1778,16 @@ repos: require_serial: true - id: check-changelog-has-no-duplicates name: Check changelogs for duplicate entries + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: changelog\.(rst|txt)$ entry: ./scripts/ci/prek/changelog_duplicates.py pass_filenames: true - id: check-no-conventional-commit-message name: Check commit message is not Conventional Commits style + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Reject feat:/fix:/chore: style commit subjects - Airflow uses plain prose" language: python entry: ./scripts/ci/prek/check_no_conventional_commit_message.py @@ -1540,12 +1796,16 @@ repos: pass_filenames: true - id: check-changelog-format name: Check changelog format + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: changelog\.(rst|txt)$ entry: ./scripts/ci/prek/check_changelog_format.py pass_filenames: true - id: check-newsfragments-are-valid name: Check newsfragments are valid + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: newsfragments/.*\.rst$ entry: ./scripts/ci/prek/newsfragments.py @@ -1555,6 +1815,8 @@ repos: always_run: true - id: update-breeze-cmd-output name: Update breeze docs + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Update output of breeze commands in Breeze documentation entry: ./scripts/ci/prek/breeze_cmd_line.py language: python @@ -1567,6 +1829,8 @@ repos: pass_filenames: false - id: check-example-dags-urls name: Check that example dags url include provider versions + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/update_example_dags_paths.py language: python pass_filenames: true @@ -1578,12 +1842,16 @@ repos: always_run: true - id: check-lazy-logging name: Check that all logging methods are lazy + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_lazy_logging.py language: python pass_filenames: true files: \.py$ - id: check-xcom-model-columns name: Check XComModel columns are read through xcom_entity, not directly in SQLAlchemy queries + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_xcom_model_columns.py language: python pass_filenames: true @@ -1591,24 +1859,32 @@ repos: exclude: ^airflow-core/src/airflow/(models/xcom\.py|api_fastapi/common/parameters/xcom\.py)$ - id: check-no-new-airflow-exceptions name: Check that no new raise AirflowException usages are added + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_new_airflow_exception_usage.py language: python pass_filenames: true files: ^(airflow-core|airflow-ctl|task-sdk|providers|shared)/.*\.py$ - id: check-no-new-provide-session-positional name: Check that no new @provide_session functions declare `session` positionally + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_provide_session_kwargs.py language: python pass_filenames: true files: ^(airflow-core|providers)/.*\.py$|^scripts/ci/prek/known_provide_session_positional\.txt$|^scripts/ci/prek/check_provide_session_kwargs\.py$ - id: check-no-new-airflow-core-utils-modules name: Check that no new modules are added under airflow-core/src/airflow/utils + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_no_new_airflow_core_utils_modules.py language: python pass_filenames: false files: ^airflow-core/src/airflow/utils/[^/]+$|^scripts/ci/prek/known_airflow_core_utils_modules\.txt$|^scripts/ci/prek/check_no_new_airflow_core_utils_modules\.py$ - id: check-mapped-column-nullable-annotations name: Check that nullable mapped_column attributes are annotated Mapped[X | None] + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_mapped_column_nullable_annotations.py language: python pass_filenames: true @@ -1620,6 +1896,8 @@ repos: ^scripts/ci/prek/check_mapped_column_nullable_annotations\.py$ - id: bandit name: bandit + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Bandit is a tool for finding common security issues in Python code" entry: bandit language: python @@ -1637,6 +1915,8 @@ repos: - "high" # TODO: remove this line when we fix all the issues - id: check-k8s-schemas-published name: Check K8s schemas are published on airflow.apache.org + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_k8s_schemas_published.py language: python pass_filenames: false @@ -1650,8 +1930,10 @@ repos: # because under it `--check` only validates the lock, it does not detect staleness. - id: update-uv-lock name: Update uv.lock + env: + PYTHONPYCACHEPREFIX: .build/pycache + UV_FROZEN: "0" entry: uv lock --check - env: {UV_FROZEN: "0"} language: system files: > (?x) @@ -1661,6 +1943,8 @@ repos: require_serial: true - id: mypy-devel-common name: Run mypy for devel-common + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py devel-common pass_filenames: false @@ -1668,13 +1952,25 @@ repos: require_serial: true - id: check-shared-mypy-hooks name: Every shared/<dist> has a mypy-shared-<dist> prek hook + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_shared_mypy_hooks.py pass_filenames: false files: ^shared/.*|^scripts/ci/prek/check_shared_mypy_hooks\.py$ require_serial: true + - id: check-prek-hooks-pycache-prefix + name: Every prek hook writes Python bytecode outside the source tree + env: + PYTHONPYCACHEPREFIX: .build/pycache + language: python + entry: ./scripts/ci/prek/check_prek_hooks_pycache_prefix.py + pass_filenames: false + files: (^|/)\.pre-commit-config\.yaml$|^scripts/ci/prek/check_prek_hooks_pycache_prefix\.py$ - id: check-migration-patterns name: Check migration files for anti-patterns and style (MIG001-MIG004) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_migration_patterns.py language: python pass_filenames: true @@ -1686,12 +1982,16 @@ repos: ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./scripts/ci/prek/check_template_fields.py files: ^(providers/[^/]+|airflow-core)/src/airflow/(.*/)?(sensors|operators)/.*\.py$ require_serial: true - id: check-execution-api-versions name: Check execution API datamodel changes have corresponding version updates + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_execution_api_versions.py language: python pass_filenames: true @@ -1699,6 +1999,8 @@ repos: require_serial: true - id: generate-supervisor-schemas-snapshot name: Regenerate supervisor schema snapshot + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/generate_supervisor_schemas_snapshot.py language: python pass_filenames: false @@ -1706,6 +2008,8 @@ repos: require_serial: true - id: check-supervisor-schemas-versions name: Check supervisor schema changes have corresponding version updates + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_supervisor_schemas_versions.py language: python pass_filenames: true @@ -1713,6 +2017,8 @@ repos: require_serial: true - id: generate-yaml-importer-schema-snapshot name: Regenerate YAML DAG importer schema snapshot + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: uv run -p 3.12 --frozen --no-progress --project task-sdk -s scripts/ci/prek/generate_yaml_importer_schema.py language: python pass_filenames: false @@ -1720,6 +2026,8 @@ repos: require_serial: true - id: check-yaml-importer-schema-versions name: Check YAML DAG importer schema changes have corresponding version updates + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_yaml_importer_schema_versions.py language: python pass_filenames: true @@ -1727,6 +2035,8 @@ repos: require_serial: true - id: generate-tasksdk-datamodels name: Generate Datamodels for TaskSDK client + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: uv run -p 3.12 --no-progress --active --group codegen --package apache-airflow-task-sdk --directory task-sdk -s dev/generate_task_sdk_models.py pass_filenames: false @@ -1734,6 +2044,8 @@ repos: require_serial: true - id: generate-airflowctl-datamodels name: Generate Datamodels for AirflowCTL + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: > bash -c ' @@ -1747,12 +2059,16 @@ repos: require_serial: true - id: check-contextmanager-class-decorators name: Check for problematic context manager class decorators + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_contextmanager_class_decorators.py language: python files: .*test.*\.py$ pass_filenames: true - id: ktlint name: Run ktlint format + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Use ktlint (via Gradle) to format Kotlin and Java files" entry: ./java-sdk/gradlew -p ./java-sdk ktlintFormat language: system @@ -1760,6 +2076,8 @@ repos: files: ^java-sdk/.*$ - id: spotless name: Run spotless apply + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Use spotless (via Gradle) to format Java files" entry: ./java-sdk/gradlew -p ./java-sdk spotlessApply language: system diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml index a16401bf481..293a892159b 100644 --- a/airflow-core/.pre-commit-config.yaml +++ b/airflow-core/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: check-first-limit name: Check single-row lookups have an explicit limit + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_first_limit.py language: python files: ^src/airflow/.*\.py$ pass_filenames: true - id: check-airflow-k8s-not-used name: Check airflow.kubernetes imports are not used + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/.*\.py$ require_serial: true @@ -41,6 +45,8 @@ repos: --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`." - id: check-common-compat-used-for-openlineage name: Check common.compat is used for OL deprecated classes + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/.*\.py$ require_serial: true @@ -50,6 +56,8 @@ repos: --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead." - id: check-cncf-k8s-only-for-executors name: Check cncf.kubernetes imports used for executors only + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/.*\.py$ require_serial: true @@ -66,12 +74,16 @@ repos: "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`." - id: check-extra-packages-references name: Checks setup extra packages + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Checks if extras defined in hatch_build.py are listed in extra-packages-ref.rst file language: python files: ^docs/extra-packages-ref\.rst$|^hatch_build\.py$ pass_filenames: false entry: ../scripts/ci/prek/check_extra_packages_ref.py - id: check-no-providers-in-core-examples + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: No providers imports in core example DAGs description: The core example DAGs have no dependencies other than standard provider or core Airflow @@ -80,6 +92,8 @@ repos: files: ^src/airflow/example_dags/.*\.py$ - id: check-only-new-session-with-provide-session name: Check NEW_SESSION is only used with @provide_session + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/new_session_in_provide_session.py pass_filenames: true @@ -87,18 +101,24 @@ repos: exclude: ^src/airflow/serialization/pydantic/.* - id: check-base-operator-partial-arguments name: Check BaseOperator and partial() arguments + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_base_operator_partial_arguments.py pass_filenames: false files: ^src/airflow/models/(?:base|mapped)operator\.py$ - id: check-decorated-operator-implements-custom-name name: Check @task decorator implements custom_operator_name + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/decorator_operator_implements_custom_name.py pass_filenames: true files: ^src/airflow/.*\.py$ - id: compile-ui-assets name: Compile ui assets (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['manual'] types_or: [javascript, ts, tsx] @@ -111,6 +131,8 @@ repos: additional_dependencies: ['[email protected]'] - id: compile-ui-assets-dev name: Compile ui assets in dev mode (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['manual'] types_or: [javascript, ts, tsx] @@ -123,6 +145,8 @@ repos: additional_dependencies: ['[email protected]'] - id: check-tests-in-the-right-folders name: Check if tests are in the right folders + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_tests_in_right_folders.py language: python files: ^tests/.*\.py$ @@ -130,6 +154,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint config_templates/config.yml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_json_schema.py args: - --spec-file @@ -140,12 +166,16 @@ repos: require_serial: true - id: check-code-deprecations name: Check deprecations categories in decorators + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_deprecations.py language: python pass_filenames: true files: ^src/airflow/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true @@ -157,24 +187,32 @@ repos: ^tests/unit/utils/test_serve_logs\.py$ - id: check-openapi-exception-doc-in-sync name: Check API routes declare the HTTP statuses they raise + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_openapi_exception_doc_in_sync.py language: python pass_filenames: true files: ^src/airflow/api_fastapi/.*/routes/.*\.py$ - id: create-missing-init-py-files-tests name: Create missing init.py files in tests + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_init_in_tests.py language: python pass_filenames: false files: ^tests/.*\.py$ - id: check-tests-unittest-testcase name: Unit tests do not inherit from unittest.TestCase + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check that unit tests do not inherit from unittest.TestCase entry: ../scripts/ci/prek/unittest_testcase.py language: python pass_filenames: true files: ^tests/.*\.py$ - id: check-no-fab-migrations + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check no migration is done on FAB related table description: > @@ -202,6 +240,8 @@ repos: ^src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key\.py$ - id: ts-compile-lint-ui name: Compile / format / lint UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files language: node files: | @@ -218,6 +258,8 @@ repos: require_serial: true - id: ts-compile-lint-simple-auth-manager-ui name: Compile / format / lint simple auth manager UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files language: node files: | @@ -235,6 +277,8 @@ repos: require_serial: true - id: mypy-airflow-core name: Run mypy for airflow-core + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-core pass_filenames: false @@ -244,6 +288,8 @@ repos: # The below prek hooks are those requiring CI image to be built - id: generate-openapi-spec name: Generate the FastAPI API spec + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/generate_openapi_spec.py pass_filenames: false @@ -254,6 +300,8 @@ repos: exclude: ^src/airflow/api_fastapi/execution_api/.* - id: check-i18n-json name: Check i18n files validity + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check i18n files are valid json, have no TODOs, and auto-format them language: python files: ^src/airflow/ui/public/i18n/locales/.*\.json$ @@ -261,6 +309,8 @@ repos: pass_filenames: false - id: update-migration-references name: Update migration ref doc + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/migration_reference.py args: ["--app", "airflow"] @@ -271,6 +321,8 @@ repos: ^docs/migrations-ref\.rst$ - id: migration-round-trip name: Migration round-trip SQLite FK enforcement + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/migration_round_trip.py pass_filenames: false @@ -279,6 +331,8 @@ repos: files: ^src/airflow/migrations/versions/.*\.py$ - id: check-default-configuration name: Check the default configuration + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_default_configuration.py language: python require_serial: true @@ -286,6 +340,8 @@ repos: files: ^src/airflow/config_templates/config\.yml$ - id: check-security-doc-constants name: Check security docs match config.yml constants + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_security_doc_constants.py language: python pass_filenames: false @@ -295,6 +351,8 @@ repos: ^docs/security/jwt_token_authentication\.rst$| ^docs/security/security_model\.rst$ - id: check-airflow-version-checks-in-core + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: No AIRFLOW_V_* imports in airflow-core entry: "import AIRFLOW_V_" @@ -312,6 +370,8 @@ repos: ^tests/unit/models/test_variable\.py$ - id: check-sdk-imports-in-core name: Check for SDK imports in core files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_sdk_imports_in_core.py language: python types: [python] @@ -319,6 +379,8 @@ repos: ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-schema-defaults name: Check schema defaults match server-side defaults + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_schema_defaults.py language: python files: diff --git a/airflow-ctl-tests/.pre-commit-config.yaml b/airflow-ctl-tests/.pre-commit-config.yaml index 1016fa939eb..791ddc4d8eb 100644 --- a/airflow-ctl-tests/.pre-commit-config.yaml +++ b/airflow-ctl-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: check-airflowctl-command-coverage name: Check airflowctl CLI command test coverage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_airflowctl_command_coverage.py language: python pass_filenames: false @@ -32,6 +34,8 @@ repos: ^tests/airflowctl_tests/.*\.py$ - id: mypy-airflow-ctl-tests name: Run mypy for airflow-ctl-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl-tests pass_filenames: false diff --git a/airflow-ctl/.pre-commit-config.yaml b/airflow-ctl/.pre-commit-config.yaml index 4561533c574..24353137cbb 100644 --- a/airflow-ctl/.pre-commit-config.yaml +++ b/airflow-ctl/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: mypy-airflow-ctl name: Run mypy for airflow-ctl + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl pass_filenames: false @@ -33,6 +35,8 @@ repos: require_serial: true - id: generate-airflowctl-help-images name: Generate SVG from Airflow CTL Commands + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/capture_airflowctl_help.py language: python pass_filenames: false @@ -43,6 +47,8 @@ repos: ^src/airflowctl/ctl/commands/.*\.py$ - id: check-airflowctl-command-coverage name: Check airflowctl CLI command test coverage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_airflowctl_command_coverage.py language: python pass_filenames: false @@ -52,6 +58,8 @@ repos: ^docs/images/command_hashes.txt$ - id: check-airflowctl-help-texts name: Check airflowctl CLI commands have help texts + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_airflowctl_help_texts.py language: python pass_filenames: false diff --git a/airflow-e2e-tests/.pre-commit-config.yaml b/airflow-e2e-tests/.pre-commit-config.yaml index e3d6413706d..0819cbdda0d 100644 --- a/airflow-e2e-tests/.pre-commit-config.yaml +++ b/airflow-e2e-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-airflow-e2e-tests name: Run mypy for airflow-e2e-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-e2e-tests pass_filenames: false diff --git a/chart/.pre-commit-config.yaml b/chart/.pre-commit-config.yaml index 6267bd1bfc3..fd60314f6ce 100644 --- a/chart/.pre-commit-config.yaml +++ b/chart/.pre-commit-config.yaml @@ -27,6 +27,8 @@ repos: hooks: - id: insert-license name: Add license for all Helm template files + env: + PYTHONPYCACHEPREFIX: .build/pycache files: ^templates/.* args: - --comment-style @@ -38,6 +40,8 @@ repos: hooks: - id: update-chart-dependencies name: Update chart dependencies to latest (manual) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_chart_dependencies.py stages: ['manual'] language: python @@ -46,6 +50,8 @@ repos: require_serial: true - id: mypy-helm-tests name: Run mypy for helm-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py chart/tests pass_filenames: false @@ -56,6 +62,8 @@ repos: hooks: - id: pretty-format-json name: Format JSON files + env: + PYTHONPYCACHEPREFIX: .build/pycache args: - --autofix - --no-sort-keys @@ -70,6 +78,8 @@ repos: hooks: - id: lint-helm-chart name: Lint Helm Chart + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_helm.py language: python pass_filenames: false @@ -77,12 +87,16 @@ repos: require_serial: true - id: validate-chart-annotations name: Validate chart annotations + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/validate_chart_annotations.py language: python pass_filenames: false files: ^Chart\.yaml$ - id: kubeconform name: Kubeconform check on our helm chart + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_kubeconform.py language: python pass_filenames: false @@ -90,6 +104,8 @@ repos: require_serial: true - id: build-kustomize-overlays name: Build and smoke-test Kustomize overlays + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/build_kustomize_overlays.py language: python pass_filenames: false @@ -97,6 +113,8 @@ repos: require_serial: true - id: sort-helm-values-schema-file-type-field name: Sort Helm Values Schema File (type field) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/sort_helm_values_schema_file_type_field.py language: python pass_filenames: false @@ -104,6 +122,8 @@ repos: require_serial: true - id: update-helm-values-schema-schema-file-type-field-match name: Update Helm Values Schema Schema File (type field match) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_helm_schema_schema_type_fields_match.py language: python pass_filenames: false @@ -111,6 +131,8 @@ repos: require_serial: true - id: lint-json-schema name: Lint chart/values.schema.json + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_json_schema.py args: - --spec-file @@ -122,11 +144,15 @@ repos: require_serial: true - id: update-vendored-in-k8s-json-schema name: Vendor k8s definitions into values.schema.json + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/vendor_k8s_json_schema.py language: python files: ^values\.schema\.json$ - id: lint-json-schema name: Lint chart/values.yaml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/lint_json_schema.py args: - --enforce-defaults @@ -139,6 +165,8 @@ repos: require_serial: true - id: lint-chart-schema name: Lint chart/values.schema.json file + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/chart_schema.py language: python pass_filenames: false diff --git a/dev/.pre-commit-config.yaml b/dev/.pre-commit-config.yaml index f401bd97545..98e0d6f1d3d 100644 --- a/dev/.pre-commit-config.yaml +++ b/dev/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-dev name: Run mypy for dev + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py dev pass_filenames: false diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index c85b9a34e2f..34b2b571a6a 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -21,6 +21,7 @@ Useful tools for various Paths used inside Airflow Sources. from __future__ import annotations import contextlib +import errno import hashlib import os import platform @@ -625,6 +626,11 @@ def cleanup_python_generated_files(): pass except PermissionError: permission_errors.append(Path(dirpath)) + except OSError as e: + # A concurrent breeze process (e.g. parallel prek hooks) can write a fresh .pyc + # between rmtree emptying the directory and removing it. + if e.errno != errno.ENOTEMPTY: + raise dirnames.clear() if permission_errors: if platform.uname().system.lower() == "linux": diff --git a/dev/breeze/tests/test_path_utils.py b/dev/breeze/tests/test_path_utils.py new file mode 100644 index 00000000000..2c161a01d4e --- /dev/null +++ b/dev/breeze/tests/test_path_utils.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import errno +from unittest import mock + +import pytest + +from airflow_breeze.utils import path_utils +from airflow_breeze.utils.path_utils import cleanup_python_generated_files + + [email protected] +def root_with_pycache(tmp_path, monkeypatch): + pycache = tmp_path / "pkg" / "__pycache__" + pycache.mkdir(parents=True) + (pycache / "mod.cpython-310.pyc").write_bytes(b"") + monkeypatch.setattr(path_utils, "AIRFLOW_ROOT_PATH", tmp_path) + return pycache + + [email protected](path_utils.shutil, "rmtree", autospec=True) +def test_cleanup_ignores_pycache_repopulated_concurrently(mock_rmtree, root_with_pycache): + mock_rmtree.side_effect = OSError(errno.ENOTEMPTY, "Directory not empty") + + cleanup_python_generated_files() + + mock_rmtree.assert_called_once_with(str(root_with_pycache)) + + [email protected](path_utils.shutil, "rmtree", autospec=True) +def test_cleanup_reraises_other_os_errors(mock_rmtree, root_with_pycache): + mock_rmtree.side_effect = OSError(errno.EIO, "I/O error") + + with pytest.raises(OSError, match="I/O error"): + cleanup_python_generated_files() diff --git a/dev/mypy/.pre-commit-config.yaml b/dev/mypy/.pre-commit-config.yaml index a75260a9276..2e7f12f2a9f 100644 --- a/dev/mypy/.pre-commit-config.yaml +++ b/dev/mypy/.pre-commit-config.yaml @@ -25,6 +25,8 @@ repos: - repo: local hooks: - id: mypy-airflow-mypy + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['pre-push'] name: Run mypy for airflow-mypy plugins language: python @@ -35,6 +37,8 @@ repos: ^tests/.*\.py$ require_serial: true - id: mypy-airflow-mypy-manual + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['manual'] name: Run mypy for airflow-mypy (manual) language: python diff --git a/docker-tests/.pre-commit-config.yaml b/docker-tests/.pre-commit-config.yaml index 27e73ecc067..c3cc7803445 100644 --- a/docker-tests/.pre-commit-config.yaml +++ b/docker-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-docker-tests name: Run mypy for docker-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py docker-tests pass_filenames: false diff --git a/go-sdk/.pre-commit-config.yaml b/go-sdk/.pre-commit-config.yaml index 523b64d8dd0..ee0f25b2eba 100644 --- a/go-sdk/.pre-commit-config.yaml +++ b/go-sdk/.pre-commit-config.yaml @@ -26,14 +26,20 @@ repos: hooks: - id: identity name: Print checked files + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Print input to the static check hooks for troubleshooting - id: check-hooks-apply name: Check if all hooks apply to the repository + env: + PYTHONPYCACHEPREFIX: .build/pycache - repo: https://github.com/Lucas-C/pre-commit-hooks rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 hooks: - id: insert-license name: Add license for all Go files + env: + PYTHONPYCACHEPREFIX: .build/pycache types: [go] exclude: mocks/.*\.go$|\.pb\.go$ args: @@ -48,6 +54,8 @@ repos: hooks: - id: go-mod-tidy name: Run go mod tidy + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: go mod tidy types: [go] exclude: mocks/.*\.go$ @@ -55,6 +63,8 @@ repos: language: golang - id: imports name: Consistent import ordering for Go files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: goimports-reviser -rm-unused -set-exit-status additional_dependencies: [github.com/incu6us/goimports-reviser/[email protected]] types: [go] @@ -62,12 +72,16 @@ repos: require_serial: true - id: gofmt name: Format go code + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: golines --base-formatter=gofumpt --write-output --max-len=100 --chain-split-dots additional_dependencies: [github.com/segmentio/golines@latest, mvdan.cc/[email protected]] types: [go] language: golang - id: staticcheck name: Check naming in Go code + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: staticcheck -checks ST1003 ./... additional_dependencies: [honnef.co/go/tools/cmd/[email protected]] types: [go] diff --git a/kubernetes-tests/.pre-commit-config.yaml b/kubernetes-tests/.pre-commit-config.yaml index 4bfb8077b9c..0c5ae79e0d5 100644 --- a/kubernetes-tests/.pre-commit-config.yaml +++ b/kubernetes-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-kubernetes-tests name: Run mypy for kubernetes-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py kubernetes-tests pass_filenames: false diff --git a/providers/.pre-commit-config.yaml b/providers/.pre-commit-config.yaml index 6761b1ab24b..7307f362c43 100644 --- a/providers/.pre-commit-config.yaml +++ b/providers/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: check-deferrable-default name: Check and fix default value of default_deferrable + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_deferrable_default.py pass_filenames: false files: ^(.*/)?airflow/.*/(sensors|operators)/.*\.py$ - id: check-trigger-serialize-init name: Check trigger __init__ and serialize() are in sync + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Every trigger __init__ parameter must appear in its serialize() return dict language: python entry: ../scripts/ci/prek/check_trigger_serialize_init.py @@ -39,6 +43,8 @@ repos: files: ^.*/src/airflow/providers/.*/triggers/[^/]+\.py$ - id: check-deferred-hook-configuration name: Check deferred AWS tasks pass their hook configuration + env: + PYTHONPYCACHEPREFIX: .build/pycache description: An operator that defers must hand region_name, verify and botocore_config to its trigger language: python entry: ../scripts/ci/prek/check_deferred_hook_configuration.py @@ -46,6 +52,8 @@ repos: files: ^amazon/src/airflow/providers/amazon/aws/.*\.py$ - id: check-migration-patterns name: Check migration files for anti-patterns and style (MIG001-MIG004) + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_migration_patterns.py language: python pass_filenames: true @@ -54,12 +62,16 @@ repos: require_serial: true - id: update-providers-dependencies name: Update dependencies for providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_providers_dependencies.py language: python always_run: true pass_filenames: false - id: validate-operators-init name: Validate templated fields in operator __init__ + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Require plain assignment of templated fields and forbid reading their un-rendered values language: python entry: ../scripts/ci/prek/validate_operators_init.py @@ -67,6 +79,8 @@ repos: files: ^.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$ - id: check-conn-id-templated name: Check connection ids accepted by operators are in template_fields + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Every *conn_id argument an operator, sensor or notifier accepts must be templated language: python entry: ../scripts/ci/prek/check_conn_id_templated.py @@ -76,6 +90,8 @@ repos: require_serial: true - id: update-providers-build-files name: Update providers build files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/update_providers_build_files.py language: python pass_filenames: true @@ -91,24 +107,32 @@ repos: require_serial: true - id: check-airflow-v-imports-in-tests name: Check AIRFLOW_V imports in tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_airflow_v_imports_in_tests.py pass_filenames: true files: ^.*/tests/.+\.py$ - id: check-sql-dependency-common-data-structure name: Check dependency of SQL providers + env: + PYTHONPYCACHEPREFIX: .build/pycache description: Check dependency of SQL Providers with common data structure entry: ../scripts/ci/prek/check_common_sql_dependency.py language: python files: ^.*/src/airflow/providers/.*/hooks/.*\.py$ - id: generate-volumes-for-sources name: Generate volumes for docker compose + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/generate_volumes_for_sources.py language: python files: ^.*/provider\.yaml$ pass_filenames: false require_serial: true - id: check-pytest-mark-db-test-in-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check pytest.mark.db_test use in providers entry: pytest\.mark\.db_test @@ -169,6 +193,8 @@ repos: ^zendesk/.*\.py$ - id: check-links-to-example-dags-do-not-use-hardcoded-versions name: Verify no hard-coded version in example dags + env: + PYTHONPYCACHEPREFIX: .build/pycache description: The links to example dags should use |version| as version specification language: pygrep entry: > @@ -178,6 +204,8 @@ repos: pass_filenames: true files: ^.*/docs/.*\.rst - id: check-no-airflow-deprecation-in-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Do not use DeprecationWarning in providers description: Use AirflowProviderDeprecationWarning in providers @@ -185,6 +213,8 @@ repos: pass_filenames: true files: ^.*/src/airflow/providers/.*\.py$ - id: check-base-operator-usage + env: + PYTHONPYCACHEPREFIX: .build/pycache language: pygrep name: Check BaseOperator other imports description: Make sure BaseOperator is imported from airflow.models outside of core @@ -193,6 +223,8 @@ repos: files: ^.*/src/airflow/providers/.*\.py$ exclude: ^standard/.*/.*\.py$ - id: check-get-lineage-collector-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python name: Check providers import hook lineage code from compat description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of @@ -205,6 +237,8 @@ repos: exclude: ^common/compat/src/airflow/providers/common/compat/.*\.py$ - id: check-aiobotocore-optional name: Check if aiobotocore is an optional dependency only + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_aiobotocore_optional.py language: python files: ^.*/provider\.yaml$ @@ -212,6 +246,8 @@ repos: require_serial: true - id: check-providers-subpackages-init-file-exist name: Provider subpackage init files are there + env: + PYTHONPYCACHEPREFIX: .build/pycache pass_filenames: false always_run: true entry: ../scripts/ci/prek/check_providers_subpackages_all_have_init.py @@ -219,12 +255,16 @@ repos: require_serial: true - id: check-system-tests-tocs name: Check that system tests is properly added + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_system_tests_hidden_in_index.py language: python pass_filenames: true files: ^.*/docs/index\.rst$ - id: check-provider-docs-valid name: Validate provider doc files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_provider_docs.py language: python files: ^.*/provider\.yaml$|^.*/docs/.* @@ -232,6 +272,8 @@ repos: pass_filenames: false - id: check-connection-doc-labels name: Validate connection doc labels match provider.yaml + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_connection_doc_labels.py language: python files: ^.*/provider\.yaml$|^.*/docs/.*/connections.*\.rst$|^.*/docs/connections\.rst$ @@ -239,12 +281,16 @@ repos: pass_filenames: false - id: check-common-ai-operators-index name: Check common.ai operators are listed in the docs index + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_common_ai_operators_index.py language: python files: ^common/ai/(src/airflow/providers/common/ai/operators/.*\.py|docs/operators/index\.rst)$ require_serial: true pass_filenames: false - id: mypy-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['pre-push'] name: Run mypy for providers language: python @@ -252,6 +298,8 @@ repos: files: ^.*\.py$ require_serial: true - id: mypy-providers + env: + PYTHONPYCACHEPREFIX: .build/pycache stages: ['manual'] name: Run mypy for providers (manual) language: python @@ -261,6 +309,8 @@ repos: require_serial: true - id: check-provider-yaml-valid name: Validate provider.yaml files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_provider_yaml_files.py language: python files: > @@ -271,18 +321,24 @@ repos: require_serial: true - id: check-cli-definition-imports name: Check CLI definition files only import allowed modules + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_cli_definition_imports.py language: python files: ^.*/cli/definition\.py$ require_serial: false - id: check-imports-in-providers name: Check imports in providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_imports_in_providers.py language: python files: ^.*/src/airflow/providers/.*version_compat.*\.py$ require_serial: true - id: check-conf-import-in-providers name: Check conf is imported from compat SDK in providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_conf_import_in_providers.py language: python types: [python] @@ -291,6 +347,8 @@ repos: require_serial: false - id: provider-version-compat name: Check for correct version_compat imports in providers + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_provider_version_compat.py language: python types: [python] @@ -301,6 +359,8 @@ repos: hooks: - id: ast-grep name: Check providers import httpx2 rather than httpx + env: + PYTHONPYCACHEPREFIX: .build/pycache types: [python] # Pinned to the providers config rather than relying on ast-grep walking up # from the working directory, so a future root sgconfig.yml cannot shadow it. diff --git a/providers/amazon/.pre-commit-config.yaml b/providers/amazon/.pre-commit-config.yaml index 3e634c3ef2d..eda8ca78672 100644 --- a/providers/amazon/.pre-commit-config.yaml +++ b/providers/amazon/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true diff --git a/providers/common/ai/.pre-commit-config.yaml b/providers/common/ai/.pre-commit-config.yaml index 396081858a8..10929d328da 100644 --- a/providers/common/ai/.pre-commit-config.yaml +++ b/providers/common/ai/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: ts-compile-lint-hitl-review-ui name: Compile / format / lint HITL Review UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files in HITL Review language: node files: | @@ -41,6 +43,8 @@ repos: require_serial: true - id: compile-common-ai-provider-assets name: Compile Common AI provider assets + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['pre-commit', 'manual'] files: ^src/airflow/providers/common/ai/plugins/www/ @@ -49,6 +53,8 @@ repos: additional_dependencies: ['[email protected]'] - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true diff --git a/providers/common/compat/.pre-commit-config.yaml b/providers/common/compat/.pre-commit-config.yaml index 71cfbd6dad2..537eeaf88b2 100644 --- a/providers/common/compat/.pre-commit-config.yaml +++ b/providers/common/compat/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: check-common-compat-sdk-imports-in-sync name: Check common.compat sdk TYPE_CHECKING matches runtime maps + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python files: ^src/airflow/providers/common/compat/sdk\.py$ pass_filenames: false diff --git a/providers/edge3/.pre-commit-config.yaml b/providers/edge3/.pre-commit-config.yaml index 000f8b6dab6..d1b7ab19cc7 100644 --- a/providers/edge3/.pre-commit-config.yaml +++ b/providers/edge3/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: generate-openapi-spec-edge name: Generate the FastAPI API spec for Edge + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/generate_openapi_spec_providers.py edge pass_filenames: false files: ^src/airflow/providers/edge3/worker_api/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true @@ -41,6 +45,8 @@ repos: ^tests/unit/edge3/(worker_api|plugins)/.*\.py$ - id: ts-compile-lint-edge-ui name: Compile / format / lint edge UI + env: + PYTHONPYCACHEPREFIX: .build/pycache description: TS types generation / ESLint / Prettier new UI files in Edge Provider language: node files: | @@ -57,6 +63,8 @@ repos: require_serial: true - id: update-migration-references-edge3 name: Update migration ref doc for Edge3 + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/migration_reference.py args: ["--app", "edge3"] @@ -67,6 +75,8 @@ repos: ^(providers/edge3/)?docs/migrations-ref\.rst$ - id: check-revision-heads-map-edge3 name: Check that the REVISION_HEADS_MAP is up-to-date + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/check_revision_heads_map.py pass_filenames: false @@ -76,6 +86,8 @@ repos: ^(providers/edge3/)?src/airflow/providers/edge3/models/db\.py$ - id: compile-edge-assets name: Compile Edge provider assets + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['pre-commit', 'manual'] files: ^src/airflow/providers/edge3/plugins/www diff --git a/providers/fab/.pre-commit-config.yaml b/providers/fab/.pre-commit-config.yaml index 2b5032ea365..560e3c9fe9c 100644 --- a/providers/fab/.pre-commit-config.yaml +++ b/providers/fab/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: compile-fab-assets name: Compile FAB provider assets + env: + PYTHONPYCACHEPREFIX: .build/pycache language: node stages: ['pre-commit', 'manual'] files: ^src/airflow/providers/fab/www/ @@ -38,12 +40,16 @@ repos: additional_dependencies: ['[email protected]'] - id: generate-openapi-spec-fab name: Generate the FastAPI API spec for FAB + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/generate_openapi_spec_providers.py fab pass_filenames: false files: ^src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true @@ -53,6 +59,8 @@ repos: ^tests/unit/fab/auth_manager/api_fastapi/.*\.py$ - id: update-migration-references-fab name: Update migration ref doc for FAB + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/migration_reference.py args: ["--app", "fab"] @@ -63,6 +71,8 @@ repos: ^(providers/fab/)?docs/migrations-ref\.rst$ - id: check-revision-heads-map-fab name: Check that the REVISION_HEADS_MAP is up-to-date + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/check_revision_heads_map.py pass_filenames: false @@ -72,6 +82,8 @@ repos: ^(providers/fab/)?src/airflow/providers/fab/auth_manager/models/db\.py$ - id: prevent-deprecated-sqlalchemy-usage-fab name: Prevent deprecated sqlalchemy usage + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py language: python files: ^.*\.py$ diff --git a/providers/keycloak/.pre-commit-config.yaml b/providers/keycloak/.pre-commit-config.yaml index 2215a6cd5a0..d7c5319f98c 100644 --- a/providers/keycloak/.pre-commit-config.yaml +++ b/providers/keycloak/.pre-commit-config.yaml @@ -26,12 +26,16 @@ repos: hooks: - id: generate-openapi-spec-keycloak name: Generate the FastAPI API spec for Keycloak + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/generate_openapi_spec_providers.py keycloak pass_filenames: false files: ^src/airflow/providers/keycloak/auth_manager/.*\.py$ - id: check-http-exception-import-from-fastapi name: Check HTTPException is imported from fastapi + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py language: python pass_filenames: true diff --git a/scripts/.pre-commit-config.yaml b/scripts/.pre-commit-config.yaml index 2b55cb7cfc9..dc44bf615ef 100644 --- a/scripts/.pre-commit-config.yaml +++ b/scripts/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-scripts name: Run mypy for scripts + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ./ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py scripts pass_filenames: false diff --git a/scripts/ci/prek/AGENTS.md b/scripts/ci/prek/AGENTS.md index bcaf0b8a4d5..d8b605f6684 100644 --- a/scripts/ci/prek/AGENTS.md +++ b/scripts/ci/prek/AGENTS.md @@ -32,3 +32,7 @@ When adding a new breeze-dependent hook: - Register the script in the appropriate `.pre-commit-config.yaml` (`/.pre-commit-config.yaml` for repo-wide hooks, `/airflow-core/.pre-commit-config.yaml` for core-specific hooks, or a provider-level config). +- Give every hook `env: {PYTHONPYCACHEPREFIX: .build/pycache}`. Hooks run in parallel and + breeze deletes `__pycache__` directories on start-up, so a hook writing `.pyc` next to the + sources can make a concurrent breeze hook fail with `Directory not empty`. + `check-prek-hooks-pycache-prefix` enforces this. diff --git a/scripts/ci/prek/check_prek_hooks_pycache_prefix.py b/scripts/ci/prek/check_prek_hooks_pycache_prefix.py new file mode 100755 index 00000000000..c22e3cc2fc7 --- /dev/null +++ b/scripts/ci/prek/check_prek_hooks_pycache_prefix.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0.3", +# ] +# /// +"""Fail when a prek hook does not redirect Python bytecode out of the source tree. + +prek runs hooks in parallel, and breeze removes every ``__pycache__`` under the +repository on start-up. A hook that writes ``.pyc`` files next to the sources can +refill a directory while a concurrent breeze is removing it, failing that hook +with ``Directory not empty``. Every hook therefore sets ``PYTHONPYCACHEPREFIX`` +to a hidden ``.build/`` directory, which breeze skips and git ignores. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import yaml + +PYCACHE_PREFIX = ".build/pycache" + + +def find_hooks_without_pycache_prefix(config_path: Path) -> list[str]: + config = yaml.safe_load(config_path.read_text()) or {} + return [ + hook["id"] + for repo in config.get("repos", []) + for hook in repo.get("hooks", []) + if (hook.get("env") or {}).get("PYTHONPYCACHEPREFIX") != PYCACHE_PREFIX + ] + + +def main() -> int: + config_paths = subprocess.check_output( + ["git", "ls-files", "*.pre-commit-config.yaml", ".pre-commit-config.yaml"], text=True + ).split() + errors = [ + f"{config_path}: {hook_id}" + for config_path in config_paths + for hook_id in find_hooks_without_pycache_prefix(Path(config_path)) + ] + if errors: + print("ERROR: These prek hooks do not set PYTHONPYCACHEPREFIX:\n") + print("\n".join(f" {error}" for error in errors)) + print( + "\nAdd the following to each of them, right after `name:`:\n\n" + " env:\n" + f" PYTHONPYCACHEPREFIX: {PYCACHE_PREFIX}\n" + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/prek/check_shared_mypy_hooks.py b/scripts/ci/prek/check_shared_mypy_hooks.py index fe30f36a3aa..6aea7fd4a7e 100755 --- a/scripts/ci/prek/check_shared_mypy_hooks.py +++ b/scripts/ci/prek/check_shared_mypy_hooks.py @@ -47,6 +47,8 @@ repos: hooks: - id: mypy-shared-{dist} name: Run mypy for shared-{dist} + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: >- ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py diff --git a/scripts/tests/ci/prek/test_check_prek_hooks_pycache_prefix.py b/scripts/tests/ci/prek/test_check_prek_hooks_pycache_prefix.py new file mode 100644 index 00000000000..c3ed9ad9d0f --- /dev/null +++ b/scripts/tests/ci/prek/test_check_prek_hooks_pycache_prefix.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from check_prek_hooks_pycache_prefix import find_hooks_without_pycache_prefix + +CONFIG = """\ +repos: + - repo: meta + hooks: + - id: with-prefix + env: + PYTHONPYCACHEPREFIX: .build/pycache + - repo: local + hooks: + - id: no-env + - id: other-env + env: + FOO: bar + - id: wrong-prefix + env: + PYTHONPYCACHEPREFIX: /tmp/pycache +""" + + +def test_find_hooks_without_pycache_prefix(tmp_path): + config = tmp_path / ".pre-commit-config.yaml" + config.write_text(CONFIG) + + assert find_hooks_without_pycache_prefix(config) == ["no-env", "other-env", "wrong-prefix"] + + +def test_find_hooks_without_pycache_prefix_empty_config(tmp_path): + config = tmp_path / ".pre-commit-config.yaml" + config.write_text("") + + assert find_hooks_without_pycache_prefix(config) == [] diff --git a/shared/configuration/.pre-commit-config.yaml b/shared/configuration/.pre-commit-config.yaml index 9706d8602d6..52dbc4dbcdd 100644 --- a/shared/configuration/.pre-commit-config.yaml +++ b/shared/configuration/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-configuration name: Run mypy for shared-configuration + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/configuration pass_filenames: false diff --git a/shared/dagnode/.pre-commit-config.yaml b/shared/dagnode/.pre-commit-config.yaml index 34f7fe42212..523a4709c6f 100644 --- a/shared/dagnode/.pre-commit-config.yaml +++ b/shared/dagnode/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-dagnode name: Run mypy for shared-dagnode + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/dagnode pass_filenames: false diff --git a/shared/listeners/.pre-commit-config.yaml b/shared/listeners/.pre-commit-config.yaml index 62f226fab74..abf48f879ab 100644 --- a/shared/listeners/.pre-commit-config.yaml +++ b/shared/listeners/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-listeners name: Run mypy for shared-listeners + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/listeners pass_filenames: false diff --git a/shared/logging/.pre-commit-config.yaml b/shared/logging/.pre-commit-config.yaml index 7440bc4add9..07bb0476759 100644 --- a/shared/logging/.pre-commit-config.yaml +++ b/shared/logging/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-logging name: Run mypy for shared-logging + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/logging pass_filenames: false diff --git a/shared/module_loading/.pre-commit-config.yaml b/shared/module_loading/.pre-commit-config.yaml index fd00e315b0e..a4d31f74c4d 100644 --- a/shared/module_loading/.pre-commit-config.yaml +++ b/shared/module_loading/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-module_loading name: Run mypy for shared-module_loading + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/module_loading pass_filenames: false diff --git a/shared/observability/.pre-commit-config.yaml b/shared/observability/.pre-commit-config.yaml index eab5206f133..0184d22126b 100644 --- a/shared/observability/.pre-commit-config.yaml +++ b/shared/observability/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-observability name: Run mypy for shared-observability + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/observability pass_filenames: false diff --git a/shared/plugins_manager/.pre-commit-config.yaml b/shared/plugins_manager/.pre-commit-config.yaml index 9180e2b1035..b3fa3ccc116 100644 --- a/shared/plugins_manager/.pre-commit-config.yaml +++ b/shared/plugins_manager/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-plugins_manager name: Run mypy for shared-plugins_manager + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/plugins_manager pass_filenames: false diff --git a/shared/providers_discovery/.pre-commit-config.yaml b/shared/providers_discovery/.pre-commit-config.yaml index 2c26b25a954..dd3093def9d 100644 --- a/shared/providers_discovery/.pre-commit-config.yaml +++ b/shared/providers_discovery/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-providers_discovery name: Run mypy for shared-providers_discovery + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: >- ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py diff --git a/shared/secrets_backend/.pre-commit-config.yaml b/shared/secrets_backend/.pre-commit-config.yaml index 93175830f9f..7db57194545 100644 --- a/shared/secrets_backend/.pre-commit-config.yaml +++ b/shared/secrets_backend/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-secrets_backend name: Run mypy for shared-secrets_backend + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/secrets_backend pass_filenames: false diff --git a/shared/secrets_masker/.pre-commit-config.yaml b/shared/secrets_masker/.pre-commit-config.yaml index 8025b4429c2..296a9d4c90f 100644 --- a/shared/secrets_masker/.pre-commit-config.yaml +++ b/shared/secrets_masker/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-secrets_masker name: Run mypy for shared-secrets_masker + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/secrets_masker pass_filenames: false diff --git a/shared/serialization/.pre-commit-config.yaml b/shared/serialization/.pre-commit-config.yaml index e0619c11dfd..a5ec870dc90 100644 --- a/shared/serialization/.pre-commit-config.yaml +++ b/shared/serialization/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-serialization name: Run mypy for shared-serialization + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/serialization pass_filenames: false diff --git a/shared/state/.pre-commit-config.yaml b/shared/state/.pre-commit-config.yaml index 9c22fb9a0ec..7317ae12afd 100644 --- a/shared/state/.pre-commit-config.yaml +++ b/shared/state/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-state name: Run mypy for shared-state + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/state pass_filenames: false diff --git a/shared/template_rendering/.pre-commit-config.yaml b/shared/template_rendering/.pre-commit-config.yaml index cbe27c38eba..cfb2023474b 100644 --- a/shared/template_rendering/.pre-commit-config.yaml +++ b/shared/template_rendering/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-template_rendering name: Run mypy for shared-template_rendering + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: >- ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py diff --git a/shared/timezones/.pre-commit-config.yaml b/shared/timezones/.pre-commit-config.yaml index 25c6cf6d8a5..01da7365f6f 100644 --- a/shared/timezones/.pre-commit-config.yaml +++ b/shared/timezones/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-shared-timezones name: Run mypy for shared-timezones + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/timezones pass_filenames: false diff --git a/task-sdk-integration-tests/.pre-commit-config.yaml b/task-sdk-integration-tests/.pre-commit-config.yaml index f09e0e1f6fe..75cee84f0d3 100644 --- a/task-sdk-integration-tests/.pre-commit-config.yaml +++ b/task-sdk-integration-tests/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: hooks: - id: mypy-task-sdk-integration-tests name: Run mypy for task-sdk-integration-tests + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk-integration-tests pass_filenames: false diff --git a/task-sdk/.pre-commit-config.yaml b/task-sdk/.pre-commit-config.yaml index a44abd29c15..219e9783209 100644 --- a/task-sdk/.pre-commit-config.yaml +++ b/task-sdk/.pre-commit-config.yaml @@ -26,6 +26,8 @@ repos: hooks: - id: check-core-imports name: Check for core imports in task-sdk files + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_core_imports_in_sdk.py language: python types: [python] @@ -54,12 +56,16 @@ repos: ^src/airflow/sdk/types.py$ - id: check-init-decorator-arguments name: Sync model __init__ and decorator arguments + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/check_init_decorator_arguments.py pass_filenames: false files: ^src/airflow/sdk/definitions/dag\.py$|^src/airflow/sdk/definitions/decorators/task_group\.py$ - id: mypy-task-sdk name: Run mypy for task-sdk + env: + PYTHONPYCACHEPREFIX: .build/pycache language: python entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py task-sdk pass_filenames: false diff --git a/ts-sdk/.pre-commit-config.yaml b/ts-sdk/.pre-commit-config.yaml index ea1ad619478..0a927e4dd01 100644 --- a/ts-sdk/.pre-commit-config.yaml +++ b/ts-sdk/.pre-commit-config.yaml @@ -25,6 +25,8 @@ repos: hooks: - id: ts-sdk-lint name: Lint TypeScript SDK + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/ts_sdk_lint.py language: node files: | @@ -40,6 +42,8 @@ repos: require_serial: true - id: check-ts-sdk-supervisor-schema name: Check TypeScript SDK supervisor schema is up to date + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_supervisor_schema.py language: node files: > @@ -52,6 +56,8 @@ repos: require_serial: true - id: check-ts-sdk-dag-schema name: Check TypeScript SDK Dag schema fields are up to date + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/check_dag_schema.py language: node files: > @@ -64,6 +70,8 @@ repos: require_serial: true - id: check-ts-sdk-serialization-conformance name: Check the TS SDK serializes Dags the way Airflow does + env: + PYTHONPYCACHEPREFIX: .build/pycache description: "Serialize the shared test Dags with the TS SDK and with Airflow, and compare the two" entry: ./scripts/ci/prek/check_serialization_conformance.py language: node @@ -78,6 +86,8 @@ repos: require_serial: true - id: check-ts-sdk-docs-package-version-in-sync name: Check ts-sdk/docs package.json versions match ts-sdk/package.json + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ../scripts/ci/prek/check_ts_sdk_docs_package_version_in_sync.py language: python files: > @@ -88,6 +98,8 @@ repos: require_serial: true - id: compile-ts-sdk name: Compile TypeScript SDK + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/compile_ts_sdk.py language: node stages: [manual] @@ -103,6 +115,8 @@ repos: require_serial: true - id: verify-ts-sdk-package name: Verify TypeScript SDK package artifact + env: + PYTHONPYCACHEPREFIX: .build/pycache entry: ./scripts/ci/prek/verify_ts_sdk_package.py language: node stages: [manual]
