This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 62a870bbda Fix caching in CI for pre-commits (#27368)
62a870bbda is described below
commit 62a870bbdaf1cb1b572496e90cf7e9418974870b
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Oct 30 03:56:16 2022 +0100
Fix caching in CI for pre-commits (#27368)
For some time, caching in pre-commits was broken and some of the
pre-commit installation was unnecessary. There were few missing
or mis-sequenced steps when the CI cache was declered and it made
regular PRs us a bit (1-2 minutes) longer time for pre-commit
setup than it was needed. In case of image building, there is no
need to setup all pre-commits because only one pre-commit is
needed (update-generated dependencies) - instead we can simply
install rich and yaml locally and run the pre-comit script directly.
This PR cleans it up.
---
.github/workflows/build-images.yml | 33 ++++-------
.github/workflows/ci.yml | 67 +++++++++-------------
.pre-commit-config.yaml | 2 +-
.../airflow_breeze/commands/ci_image_commands.py | 2 +-
...=> pre_commit_update_providers_dependencies.py} | 0
5 files changed, 41 insertions(+), 63 deletions(-)
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index 78ba06f328..e1186930de 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -216,17 +216,10 @@ jobs:
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze ci free-space
- - name: Cache pre-commit envs
- uses: actions/cache@v3
- with:
- path: ~/.cache/pre-commit
- key:
"pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys:
pre-commit-${{steps.host-python-version.outputs.host-python-version}}
- name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
- run: >
- breeze static-checks --type update-providers-dependencies --all-files
- --show-diff-on-failure --color always || true
+ run: |
+ pip install rich>=12.4.4 pyyaml
+ python
scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
if: needs.build-info.outputs.default-branch == 'main'
- name: >-
Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
@@ -313,17 +306,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze ci free-space
- - name: Cache pre-commit envs
- uses: actions/cache@v3
- with:
- path: ~/.cache/pre-commit
- key:
"pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys:
pre-commit-${{steps.host-python-version.outputs.host-python-version}}
- - name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
+ - name: "Get Python version"
run: >
- breeze static-checks --type update-providers-dependencies --all-files
- --show-diff-on-failure --color always || true
+ echo "host-python-version=$(python -c 'import platform;
print(platform.python_version())')"
+ >> ${GITHUB_OUTPUT}
+ id: host-python-version
+ - name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
+ run: |
+ pip install rich>=12.4.4 pyyaml
+ python
scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
- name: >
Pull CI image for PROD build:
${{ needs.build-info.outputs.default-python-version }}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}
@@ -416,7 +407,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
mv -v "main-airflow/scripts/ci" "scripts"
mv -v "main-airflow/dev" "."
- name: "Setup python"
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v4
with:
python-version: ${{ needs.build-info.outputs.default-python-version
}}
- run: ./scripts/ci/install_breeze.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6d4e91ad84..a775a4ea4b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -383,18 +383,10 @@ jobs:
- name: "Free space"
run: breeze ci free-space
if: needs.build-info.outputs.in-workflow-build == 'true'
- - name: Cache pre-commit envs
- uses: actions/cache@v3
- with:
- path: ~/.cache/pre-commit
- key:
"pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys:
pre-commit-${{steps.host-python-version.outputs.host-python-version}}
- if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
- run: >
- breeze static-checks --type update-providers-dependencies --all-files
- --show-diff-on-failure --color always || true
+ run: |
+ pip install rich>=12.4.4 pyyaml
+ python
scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
if: >
needs.build-info.outputs.in-workflow-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -464,20 +456,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Free space"
run: breeze ci free-space
if: needs.build-info.outputs.in-workflow-build == 'true'
- - name: Cache pre-commit envs
- uses: actions/cache@v3
- with:
- path: ~/.cache/pre-commit
- key:
"pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys:
pre-commit-${{steps.host-python-version.outputs.host-python-version}}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
+ - name: "Get Python version"
run: >
- breeze static-checks --type update-providers-dependencies --all-files
- --show-diff-on-failure --color always || true
+ echo "host-python-version=$(python -c 'import platform;
print(platform.python_version())')"
+ >> ${GITHUB_OUTPUT}
+ id: host-python-version
+ - name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
+ run: |
+ pip install rich>=12.4.4 pyyaml
+ python
scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
if: >
needs.build-info.outputs.in-workflow-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -690,6 +677,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.default-python-version
}}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
+ - name: "Get Python version"
+ run: >
+ echo "host-python-version=$(python -c 'import platform;
print(platform.python_version())')"
+ >> ${GITHUB_OUTPUT}
+ id: host-python-version
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
@@ -705,11 +697,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: breeze ci-image pull --tag-as-latest
env:
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- - name: "Get Python version"
- run: >
- echo "host-python-version=$(python -c 'import platform;
print(platform.python_version())')"
- >> ${GITHUB_OUTPUT}
- id: host-python-version
- name: "Static checks"
run: breeze static-checks --all-files --show-diff-on-failure --color
always
env:
@@ -735,6 +722,17 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Setup python"
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ needs.build-info.outputs.default-python-version
}}
+ cache: 'pip'
+ cache-dependency-path: ./dev/breeze/setup*
+ - name: "Get Python version"
+ run: >
+ echo "host-python-version=$(python -c 'import platform;
print(platform.python_version())')"
+ >> ${GITHUB_OUTPUT}
+ id: host-python-version
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
@@ -748,20 +746,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.default-python-version
}}
- cache: 'pip'
- cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze ci free-space
- - name: "Get Python version"
- run: >
- echo "host-python-version=$(python -c 'import platform;
print(platform.python_version())')"
- >> ${GITHUB_OUTPUT}
- id: host-python-version
- name: "Static checks: basic checks only"
run: >
breeze static-checks --all-files --show-diff-on-failure --color
always
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0e6dd89efc..4b8c2792d5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -374,7 +374,7 @@ repos:
additional_dependencies: ['rich>=12.4.4']
- id: update-providers-dependencies
name: Update cross-dependencies for providers packages
- entry:
./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
+ entry:
./scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
language: python
files:
^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$|^airflow/providers/.*/provider.yaml$
pass_filenames: false
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
index cb21fe2eae..ef070a3c97 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -484,7 +484,7 @@ def run_build_ci_image(
/ "scripts"
/ "ci"
/ "pre_commit"
- / "pre_commit_build_providers_dependencies.py"
+ / "pre_commit_update_providers_dependencies.py"
),
],
check=False,
diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
similarity index 100%
rename from scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
rename to scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
