This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new b14dbf2c8c Automatically upgrade various components in canary builds 
(#37113)
b14dbf2c8c is described below

commit b14dbf2c8c83f32e3347522c8559d99889f8e0ed
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Jan 31 10:50:27 2024 +0100

    Automatically upgrade various components in canary builds (#37113)
    
    This change attempts to automatically upgrade our dependencies in
    the canary builds. This means that canary builds will fail
    occasionally when there is a need to update some dependencies that
    otherwise are not updated automatically. This gives an opportunity
    to fix such failing main build as soon as it happns.
    
    It does not impact generation of constraints or building the
    cache, the only sign that things are wrong is failing canary build,
    so impact to it should be minimal - we will either see the main
    build failing after merge, or we will fix it alongside with fixing
    some other reason for main failing (such us broken dependencies)
    
    Currently, the following upgrades are done in canary build:
    
    * Upgrading pre-commits
    * Upgrading black version
    * Upgrading build pinned dependencies
---
 .github/workflows/ci.yml                           | 39 +++++++++++
 .pre-commit-config.yaml                            | 12 +++-
 contributing-docs/08_static_code_checks.rst        |  4 +-
 dev/breeze/doc/ci/05_workflows.md                  | 75 +++++++++++-----------
 dev/breeze/doc/ci/06_diagrams.md                   |  2 +
 dev/breeze/doc/images/output_static-checks.svg     | 16 ++---
 dev/breeze/doc/images/output_static-checks.txt     |  2 +-
 dev/breeze/src/airflow_breeze/pre_commit_ids.py    |  1 +
 pyproject.toml                                     | 15 ++---
 .../pre_commit_update_build_dependencies.py        | 56 ++++++++++++++++
 10 files changed, 165 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3e7ab144c8..e0e4ca56e9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -583,6 +583,45 @@ jobs:
           DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
           RUFF_FORMAT: "github"
 
+  upgrade-check:
+    timeout-minutes: 45
+    name: "Upgrade checks"
+    runs-on: ["ubuntu-22.04"]
+    needs: [build-info]
+    env:
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      PYTHON_MAJOR_MINOR_VERSION: 
"${{needs.build-info.outputs.default-python-version}}"
+    if: needs.build-info.outputs.canary-run == 'true'
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      # Install python from scratch. No cache used. We always want to have 
fresh version of everything
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 
"${{needs.build-info.outputs.default-python-version}}"
+      - name: "Install latest pre-commit"
+        run: pip install pre-commit
+      - name: "Autoupdate all pre-commits"
+        run: pre-commit autoupdate
+      - name: "Run automated upgrade for black"
+        run: >
+          pre-commit run
+          --all-files --show-diff-on-failure --color always --verbose
+          --hook-stage manual
+          update-black-version
+        if: always()
+      - name: "Run automated upgrade for build dependencies"
+        run: >
+          pre-commit run
+          --all-files --show-diff-on-failure --color always --verbose
+          --hook-stage manual
+          update-build-dependencies
+        if: always()
+
   # Runs static checks for groups of files in the repository in a single 
process without passing .
   # List of files
   mypy:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 100c8da645..cd7565f0b4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -176,13 +176,23 @@ repos:
         pass_filenames: false
         require_serial: true
       - id: update-black-version
-        name: Update black versions everywhere
+        name: Update black versions everywhere (manual)
         entry: ./scripts/ci/pre_commit/pre_commit_update_black_version.py
+        stages: ['manual']
         language: python
         files: ^.pre-commit-config.yaml$
         additional_dependencies: ['pyyaml']
         pass_filenames: false
         require_serial: true
+      - id: update-build-dependencies
+        name: Update build-dependencies to latest (manual)
+        entry: ./scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
+        stages: ['manual']
+        language: python
+        language_version: python38
+        files: 
^.pre-commit-config.yaml$|^scripts/ci/pre_commit/pre_commit_update_build_dependencies.py$
+        pass_filenames: false
+        require_serial: true
   - repo: https://github.com/asottile/blacken-docs
     rev: 1.16.0
     hooks:
diff --git a/contributing-docs/08_static_code_checks.rst 
b/contributing-docs/08_static_code_checks.rst
index c49eea4430..8b5acb0048 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -323,12 +323,14 @@ require Breeze Docker image to be built locally.
 
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | ts-compile-format-lint-www                                | TS types 
generation / ESLint / Prettier against UI files     |         |
 
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| update-black-version                                      | Update black 
versions everywhere                             |         |
+| update-black-version                                      | Update black 
versions everywhere (manual)                    |         |
 
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | update-breeze-cmd-output                                  | Update output of 
breeze commands in Breeze documentation     |         |
 
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | update-breeze-readme-config-hash                          | Update Breeze 
README.md with config files hash               |         |
 
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
+| update-build-dependencies                                 | Update 
build-dependencies to latest (manual)                 |         |
++-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | update-common-sql-api-stubs                               | Check and update 
common.sql API stubs                        |         |
 
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | update-er-diagram                                         | Update ER 
diagram                                            | *       |
diff --git a/dev/breeze/doc/ci/05_workflows.md 
b/dev/breeze/doc/ci/05_workflows.md
index 8310b9ce30..eaa2279262 100644
--- a/dev/breeze/doc/ci/05_workflows.md
+++ b/dev/breeze/doc/ci/05_workflows.md
@@ -219,43 +219,44 @@ builds use "Public" runners. For committers, this can be 
overridden by setting t
 This workflow is a regular workflow that performs all checks of Airflow
 code.
 
-| Job                              | Description                               
               | PR        | Canary   | Scheduled  | Release branches  |
-|----------------------------------|----------------------------------------------------------|-----------|----------|------------|-------------------|
-| Build info                       | Prints detailed information about the 
build              | Yes       | Yes      | Yes        | Yes               |
-| Push early cache & images        | Pushes early cache/images to GitHub 
Registry             |           | Yes      |            |                   |
-| Check that image builds quickly  | Checks that image builds quickly          
               |           | Yes      |            | Yes               |
-| Build CI images                  | Builds images in-workflow (not in the 
build images)      |           | Yes      | Yes (1)    | Yes (4)           |
-| Generate constraints/CI verify   | Generate constraints for the build and 
verify CI image   | Yes (2)   | Yes (2)  | Yes (2)    | Yes (2)           |
-| Build PROD images                | Builds images in-workflow (not in the 
build images)      |           | Yes      | Yes (1)    | Yes (4)           |
-| Build Bullseye PROD images       | Builds images based on Bullseye debian    
               |           | Yes      | Yes        | Yes               |
-| Run breeze tests                 | Run unit tests for Breeze                 
               | Yes       | Yes      | Yes        | Yes               |
-| Test OpenAPI client gen          | Tests if OpenAPIClient continues to 
generate             | Yes       | Yes      | Yes        | Yes               |
-| React WWW tests                  | React UI tests for new Airflow UI         
               | Yes       | Yes      | Yes        | Yes               |
-| Test examples image building     | Tests if PROD image build examples work   
               | Yes       | Yes      | Yes        | Yes               |
-| Test git clone on Windows        | Tests if Git clone for for Windows        
               | Yes (5)   | Yes (5)  | Yes (5)    | Yes (5)           |
-| Waits for CI Images              | Waits for and verify CI Images            
               | Yes (2)   | Yes (2)  | Yes (2)    | Yes (2)           |
-| Static checks                    | Performs full static checks               
               | Yes (6)   | Yes      | Yes        | Yes (7)           |
-| Basic static checks              | Performs basic static checks (no image)   
               | Yes (6)   |          |            |                   |
-| Build docs                       | Builds and tests publishing of the 
documentation         | Yes       | Yes      | Yes        | Yes               |
-| Spellcheck docs                  | Spellcheck docs                           
               | Yes       | Yes      | Yes        | Yes               |
-| Tests wheel provider packages    | Tests if provider packages can be built 
and released     | Yes       | Yes      | Yes        |                   |
-| Tests Airflow compatibility      | Compatibility of provider packages with 
older Airflow    | Yes       | Yes      | Yes        |                   |
-| Tests dist provider packages     | Tests if dist provider packages can be 
built             |           | Yes      | Yes        |                   |
-| Tests airflow release commands   | Tests if airflow release command works    
               |           | Yes      | Yes        |                   |
-| Tests (Backend/Python matrix)    | Run the Pytest unit DB tests 
(Backend/Python matrix)     | Yes       | Yes      | Yes        | Yes (8)       
    |
-| No DB tests                      | Run the Pytest unit Non-DB tests (with 
pytest-xdist)     | Yes       | Yes      | Yes        | Yes (8)           |
-| Integration tests                | Runs integration tests (Postgres/Mysql)   
               | Yes       | Yes      | Yes        | Yes (9)           |
-| Quarantined tests                | Runs quarantined tests (with flakiness 
and side-effects) | Yes       | Yes      | Yes        | Yes (8)           |
-| Test airflow packages            | Tests that Airflow package can be built 
and released     | Yes       | Yes      | Yes        | Yes               |
-| Helm tests                       | Run the Helm integration tests            
               | Yes       | Yes      | Yes        |                   |
-| Helm release tests               | Run the tests for Helm releasing          
               | Yes       | Yes      | Yes        |                   |
-| Summarize warnings               | Summarizes warnings from all other tests  
               | Yes       | Yes      | Yes        | Yes               |
-| Wait for PROD Images             | Waits for and verify PROD Images          
               | Yes (2)   | Yes (2)  | Yes (2)    | Yes (2)           |
-| Docker Compose test/PROD verify  | Tests quick-start Docker Compose and 
verify PROD image   | Yes       | Yes      | Yes        | Yes               |
-| Tests Kubernetes                 | Run Kubernetes test                       
               | Yes       | Yes      | Yes        |                   |
-| Update constraints               | Upgrade constraints to latest ones        
               | Yes (3)   | Yes (3)  | Yes (3)    | Yes (3)           |
-| Push cache & images              | Pushes cache/images to GitHub Registry 
(3)               |           | Yes (3)  |            | Yes               |
-| Build CI ARM images              | Builds CI images for ARM                  
               | Yes (10)  |          | Yes        |                   |
+| Job                             | Description                                
              | PR       | Canary   | Scheduled  | Release branches |
+|---------------------------------|----------------------------------------------------------|----------|----------|------------|------------------|
+| Build info                      | Prints detailed information about the 
build              | Yes      | Yes      | Yes        | Yes              |
+| Push early cache & images       | Pushes early cache/images to GitHub 
Registry             |          | Yes      |            |                  |
+| Check that image builds quickly | Checks that image builds quickly           
              |          | Yes      |            | Yes              |
+| Build CI images                 | Builds images in-workflow (not in the 
build images)      |          | Yes      | Yes (1)    | Yes (4)          |
+| Generate constraints/CI verify  | Generate constraints for the build and 
verify CI image   | Yes (2)  | Yes (2)  | Yes (2)    | Yes (2)          |
+| Build PROD images               | Builds images in-workflow (not in the 
build images)      |          | Yes      | Yes (1)    | Yes (4)          |
+| Build Bullseye PROD images      | Builds images based on Bullseye debian     
              |          | Yes      | Yes        | Yes              |
+| Run breeze tests                | Run unit tests for Breeze                  
              | Yes      | Yes      | Yes        | Yes              |
+| Test OpenAPI client gen         | Tests if OpenAPIClient continues to 
generate             | Yes      | Yes      | Yes        | Yes              |
+| React WWW tests                 | React UI tests for new Airflow UI          
              | Yes      | Yes      | Yes        | Yes              |
+| Test examples image building    | Tests if PROD image build examples work    
              | Yes      | Yes      | Yes        | Yes              |
+| Test git clone on Windows       | Tests if Git clone for for Windows         
              | Yes (5)  | Yes (5)  | Yes (5)    | Yes (5)          |
+| Waits for CI Images             | Waits for and verify CI Images             
              | Yes (2)  | Yes (2)  | Yes (2)    | Yes (2)          |
+| Upgrade checks                  | Performs checks if there are some pending 
upgrades       |          | Yes      | Yes        | Yes              |
+| Static checks                   | Performs full static checks                
              | Yes (6)  | Yes      | Yes        | Yes (7)          |
+| Basic static checks             | Performs basic static checks (no image)    
              | Yes (6)  |          |            |                  |
+| Build docs                      | Builds and tests publishing of the 
documentation         | Yes      | Yes      | Yes        | Yes              |
+| Spellcheck docs                 | Spellcheck docs                            
              | Yes      | Yes      | Yes        | Yes              |
+| Tests wheel provider packages   | Tests if provider packages can be built 
and released     | Yes      | Yes      | Yes        |                  |
+| Tests Airflow compatibility     | Compatibility of provider packages with 
older Airflow    | Yes      | Yes      | Yes        |                  |
+| Tests dist provider packages    | Tests if dist provider packages can be 
built             |          | Yes      | Yes        |                  |
+| Tests airflow release commands  | Tests if airflow release command works     
              |          | Yes      | Yes        |                  |
+| Tests (Backend/Python matrix)   | Run the Pytest unit DB tests 
(Backend/Python matrix)     | Yes      | Yes      | Yes        | Yes (8)        
  |
+| No DB tests                     | Run the Pytest unit Non-DB tests (with 
pytest-xdist)     | Yes      | Yes      | Yes        | Yes (8)          |
+| Integration tests               | Runs integration tests (Postgres/Mysql)    
              | Yes      | Yes      | Yes        | Yes (9)          |
+| Quarantined tests               | Runs quarantined tests (with flakiness and 
side-effects) | Yes      | Yes      | Yes        | Yes (8)          |
+| Test airflow packages           | Tests that Airflow package can be built 
and released     | Yes      | Yes      | Yes        | Yes              |
+| Helm tests                      | Run the Helm integration tests             
              | Yes      | Yes      | Yes        |                  |
+| Helm release tests              | Run the tests for Helm releasing           
              | Yes      | Yes      | Yes        |                  |
+| Summarize warnings              | Summarizes warnings from all other tests   
              | Yes      | Yes      | Yes        | Yes              |
+| Wait for PROD Images            | Waits for and verify PROD Images           
              | Yes (2)  | Yes (2)  | Yes (2)    | Yes (2)          |
+| Docker Compose test/PROD verify | Tests quick-start Docker Compose and 
verify PROD image   | Yes      | Yes      | Yes        | Yes              |
+| Tests Kubernetes                | Run Kubernetes test                        
              | Yes      | Yes      | Yes        |                  |
+| Update constraints              | Upgrade constraints to latest ones         
              | Yes (3)  | Yes (3)  | Yes (3)    | Yes (3)          |
+| Push cache & images             | Pushes cache/images to GitHub Registry (3) 
              |          | Yes (3)  |            | Yes              |
+| Build CI ARM images             | Builds CI images for ARM                   
              | Yes (10) |          | Yes        |                  |
 
 `(1)` Scheduled jobs builds images from scratch - to test if everything
 works properly for clean builds
diff --git a/dev/breeze/doc/ci/06_diagrams.md b/dev/breeze/doc/ci/06_diagrams.md
index c9610ff009..89d6fc772c 100644
--- a/dev/breeze/doc/ci/06_diagrams.md
+++ b/dev/breeze/doc/ci/06_diagrams.md
@@ -363,6 +363,8 @@ sequenceDiagram
         Note over Tests: React WWW tests
     and
         Note over Tests: Test git clone on Windows
+    and
+        Note over Tests: Run upgrade checks
     end
     Note over Tests: Skip waiting for CI images
     par
diff --git a/dev/breeze/doc/images/output_static-checks.svg 
b/dev/breeze/doc/images/output_static-checks.svg
index f05cd5435b..7c119d2cc6 100644
--- a/dev/breeze/doc/images/output_static-checks.svg
+++ b/dev/breeze/doc/images/output_static-checks.svg
@@ -347,14 +347,14 @@
 </text><text class="breeze-static-checks-r5" x="0" y="1020.4" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-41)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1020.4" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-41)">python-no-log-warn&#160;|&#160;replace-bad-characters&#160;|&#160;rst-backticks&#160;|&#160;ruff&#160;|&#160;ruff-format&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1020.4" textLength="12.2" 
clip-path="url(#breez [...]
 </text><text class="breeze-static-checks-r5" x="0" y="1044.8" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-42)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1044.8" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-42)">|&#160;shellcheck&#160;|&#160;trailing-whitespace&#160;|&#160;ts-compile-format-lint-www&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks- [...]
 </text><text class="breeze-static-checks-r5" x="0" y="1069.2" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-43)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1069.2" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-43)">update-black-version&#160;|&#160;update-breeze-cmd-output&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;
 [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1093.6" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-44)">update-breeze-readme-config-hash&#160;|&#160;update-common-sql-api-stubs&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="145 [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-45)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-45)">update-er-diagram&#160;|&#160;update-extras&#160;|&#160;update-in-the-wild-to-be-sorted&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1118" textL [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1142.4" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-46)">update-inlined-dockerfile-scripts&#160;|&#160;update-installed-providers-to-be-sorted&#160;|&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1142.4" textLength="12.2" 
clip-path="url(#breeze-static-c [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1166.8" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-47)">update-local-yml-file&#160;|&#160;update-migration-references&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#
 [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1191.2" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-48)">update-providers-dependencies&#160;|&#160;update-reproducible-source-date-epoch&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1191.2" textLength="12.2" c [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1215.6" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-49)">update-spelling-wordlist-to-be-sorted&#160;|&#160;update-supported-versions&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1215.6" [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-50)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-50)">update-vendored-in-k8s-json-schema&#160;|&#160;update-version&#160;|&#160;validate-operators-init&#160;|&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2" 
clip-path="url(#breeze-static-checks- [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1264.4" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-51)">yamllint)&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&
 [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1093.6" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-44)">update-breeze-readme-config-hash&#160;|&#160;update-build-dependencies&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks- [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-45)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-45)">update-common-sql-api-stubs&#160;|&#160;update-er-diagram&#160;|&#160;update-extras&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="14 [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1142.4" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-46)">update-in-the-wild-to-be-sorted&#160;|&#160;update-inlined-dockerfile-scripts&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1142.4" textLengt [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1166.8" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-47)">update-installed-providers-to-be-sorted&#160;|&#160;update-local-yml-file&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8"  [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1191.2" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-48)">update-migration-references&#160;|&#160;update-providers-dependencies&#160;|&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</text><text
 class="breeze-static-ch [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1215.6" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-49)">update-reproducible-source-date-epoch&#160;|&#160;update-spelling-wordlist-to-be-sorted&#160;|&#160;&#160;</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1215.6" textLength="12.2" 
clip-path="url(#breeze-static-checks-line [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-50)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-50)">update-supported-versions&#160;|&#160;update-vendored-in-k8s-json-schema&#160;|&#160;update-version&#160;|</text><text
 class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-50)"> [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1264.4" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text 
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2" 
clip-path="url(#breeze-static-checks-line-51)">validate-operators-init&#160;|&#160;yamllint)&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&
 [...]
 </text><text class="breeze-static-checks-r5" x="0" y="1288.8" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">│</text><text 
class="breeze-static-checks-r4" x="24.4" y="1288.8" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-52)">-</text><text 
class="breeze-static-checks-r4" x="36.6" y="1288.8" textLength="61" 
clip-path="url(#breeze-static-checks-line-52)">-show</text><text 
class="breeze-static-checks-r4" x="97.6" y="1288.8" textLength="195.2" 
clip-path="url(# [...]
 </text><text class="breeze-static-checks-r5" x="0" y="1313.2" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text 
class="breeze-static-checks-r4" x="24.4" y="1313.2" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-53)">-</text><text 
class="breeze-static-checks-r4" x="36.6" y="1313.2" textLength="134.2" 
clip-path="url(#breeze-static-checks-line-53)">-initialize</text><text 
class="breeze-static-checks-r4" x="170.8" y="1313.2" textLength="146.4" clip-p 
[...]
 </text><text class="breeze-static-checks-r5" x="0" y="1337.6" 
textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">│</text><text 
class="breeze-static-checks-r4" x="24.4" y="1337.6" textLength="12.2" 
clip-path="url(#breeze-static-checks-line-54)">-</text><text 
class="breeze-static-checks-r4" x="36.6" y="1337.6" textLength="48.8" 
clip-path="url(#breeze-static-checks-line-54)">-max</text><text 
class="breeze-static-checks-r4" x="85.4" y="1337.6" textLength="292.8" 
clip-path="url( [...]
diff --git a/dev/breeze/doc/images/output_static-checks.txt 
b/dev/breeze/doc/images/output_static-checks.txt
index 7a064e6df4..c2860a4651 100644
--- a/dev/breeze/doc/images/output_static-checks.txt
+++ b/dev/breeze/doc/images/output_static-checks.txt
@@ -1 +1 @@
-42639ed45de3e438c5e9f95bf3d8b300
+406f5ff089e02996c057276244f98c1d
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py 
b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index ad6614d452..3584779319 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -119,6 +119,7 @@ PRE_COMMIT_LIST = [
     "update-black-version",
     "update-breeze-cmd-output",
     "update-breeze-readme-config-hash",
+    "update-build-dependencies",
     "update-common-sql-api-stubs",
     "update-er-diagram",
     "update-extras",
diff --git a/pyproject.toml b/pyproject.toml
index dd181ddbdf..e4c7002c2b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,22 +19,19 @@
 # build dependencies should be fixed - including all transitive dependencies. 
This way we can ensure
 # reproducibility of the build and make sure that any future releases of any 
dependencies will not
 # break the build of released airflow sources in the future.
-# We should periodically refresh these build dependencies by following this 
procecedure:
-# 1. Create a new virtual environment
-# 2. Run `pip install hatchling gitpython
-# 3. Run `pip freeze`
-# 4. Copy versions you see here (note that tomli needs to have python_version 
< '3.11').
+# The dependencies can be automatically upgraded by running:
+# pre-commit run --hook-stage manual update-build-dependencies --all-files
 requires = [
+    "GitPython==3.1.41",
     "editables==0.5",
     "gitdb==4.0.11",
-    "gitpython==3.1.40",
-    "hatchling==1.21.0",
+    "hatchling==1.21.1",
     "packaging==23.2",
     "pathspec==0.12.1",
-    "pluggy==1.3.0",
+    "pluggy==1.4.0",
     "smmap==5.0.1",
     "tomli==2.0.1; python_version < '3.11'",
-    "trove-classifiers==2023.11.29",
+    "trove-classifiers==2024.1.8",
 ]
 build-backend = "hatchling.build"
 
diff --git a/scripts/ci/pre_commit/pre_commit_update_build_dependencies.py 
b/scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
new file mode 100755
index 0000000000..64c96a90e4
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
+PYPROJECT_TOML_FILE = AIRFLOW_SOURCES / "pyproject.toml"
+
+if __name__ == "__main__":
+    temp_dir = Path(tempfile.mkdtemp())
+    try:
+        subprocess.check_call([sys.executable, "-m", "venv", 
temp_dir.as_posix()])
+        venv_python = temp_dir / "bin" / "python"
+        subprocess.check_call([venv_python, "-m", "pip", "install", 
"gitpython", "hatchling"])
+        frozen_deps = subprocess.check_output([venv_python, "-m", "pip", 
"freeze"], text=True)
+        deps = [dep for dep in sorted(frozen_deps.splitlines()) if not 
dep.startswith("pip==")]
+        pyproject_toml_content = PYPROJECT_TOML_FILE.read_text()
+        result = []
+        skipping = False
+        for line in pyproject_toml_content.splitlines():
+            if not skipping:
+                result.append(line)
+            if line == "requires = [":
+                skipping = True
+                for dep in deps:
+                    # Tomli is only needed for Python < 3.11, otherwise stdlib 
tomllib is used
+                    if dep.startswith("tomli=="):
+                        dep = dep + "; python_version < '3.11'"
+                    result.append(f'    "{dep}",')
+            if skipping and line == "]":
+                skipping = False
+                result.append(line)
+        result.append("")
+        PYPROJECT_TOML_FILE.write_text("\n".join(result))
+    finally:
+        shutil.rmtree(temp_dir)


Reply via email to