This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8d8afdf20aa Fix disk space issues on github runners (#46358)
8d8afdf20aa is described below
commit 8d8afdf20aa2d32ea30adddc503216931e29459e
Author: Jens Scheffler <[email protected]>
AuthorDate: Mon Feb 3 09:56:52 2025 +0100
Fix disk space issues on github runners (#46358)
* Triage disk space issues, DO NOT MERGE
* Triage disk space issues, DO NOT MERGE
* Triage disk space issues, DO NOT MERGE
* Triage disk space issues, DO NOT MERGE
* Triage disk space issues, DO NOT MERGE
* Triage disk space issues, DO NOT MERGE
* Triage disk space issues, DO NOT MERGE
* Move docker storage to second drive in general
* Cleanup debug and triaging stuff
* Exception of docker volume for constraints building
---
.github/actions/prepare_all_ci_images/action.yml | 5 +++++
.github/actions/prepare_breeze_and_image/action.yml | 3 +++
.github/workflows/additional-ci-image-checks.yml | 3 +++
.github/workflows/additional-prod-image-tests.yml | 3 +++
.github/workflows/basic-tests.yml | 18 ++++++++++++++++++
.github/workflows/ci-image-build.yml | 3 +++
.github/workflows/ci-image-checks.yml | 6 ++++++
.github/workflows/ci.yml | 3 +++
.github/workflows/finalize-tests.yml | 6 ++++++
.github/workflows/generate-constraints.yml | 1 +
.github/workflows/helm-tests.yml | 3 +++
.github/workflows/prod-image-build.yml | 6 ++++++
.github/workflows/push-image-cache.yml | 6 ++++++
.github/workflows/release_dockerhub_image.yml | 6 ++++++
scripts/ci/cleanup_docker.sh | 6 ++++++
15 files changed, 78 insertions(+)
diff --git a/.github/actions/prepare_all_ci_images/action.yml
b/.github/actions/prepare_all_ci_images/action.yml
index d156818b9b2..25a695b4f97 100644
--- a/.github/actions/prepare_all_ci_images/action.yml
+++ b/.github/actions/prepare_all_ci_images/action.yml
@@ -22,6 +22,9 @@ inputs:
python-versions-list-as-string:
description: 'Stringified array of all Python versions to test - separated
by spaces.'
required: true
+ docker-volume-location:
+ description: File system location where to move docker space to
+ default: /mnt/var-lib-docker
platform:
description: 'Platform for the build - linux/amd64 or linux/arm64'
required: true
@@ -30,6 +33,8 @@ runs:
steps:
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: ${{ inputs.docker-volume-location }}
shell: bash
# TODO: Currently we cannot loop through the list of python versions and
have dynamic list of
# tasks. Instead we hardcode all possible python versions and they -
but
diff --git a/.github/actions/prepare_breeze_and_image/action.yml
b/.github/actions/prepare_breeze_and_image/action.yml
index e5f7bc3dd64..b29e1f7afa3 100644
--- a/.github/actions/prepare_breeze_and_image/action.yml
+++ b/.github/actions/prepare_breeze_and_image/action.yml
@@ -39,7 +39,10 @@ runs:
using: "composite"
steps:
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
shell: bash
- name: "Install Breeze"
uses: ./.github/actions/breeze
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index a6b7bdafcb5..3dba80a024d 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -142,7 +142,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index 7b551215714..86c1c19738a 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -116,7 +116,10 @@ jobs:
fetch-depth: 2
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Prepare breeze & PROD image: ${{ inputs.default-python-version
}}"
uses: ./.github/actions/prepare_breeze_and_image
with:
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 62ac87721a3..f23b9c72463 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -77,7 +77,10 @@ jobs:
fetch-depth: 0
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -98,7 +101,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: Setup pnpm
uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 #
v4.0.0
with:
@@ -165,7 +171,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Setup node"
uses: actions/setup-node@v4
with:
@@ -233,7 +242,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -288,7 +300,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -367,7 +382,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index bd4d90dd02f..39ef0a44f19 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -130,7 +130,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index faa386a2e98..bf4baa0af7f 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -324,7 +324,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Download docs prepared as artifacts"
uses: actions/download-artifact@v4
with:
@@ -405,7 +408,10 @@ jobs:
fetch-depth: 2
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3e41f1ce880..0799ee8397f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -147,7 +147,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index ac13089caf6..e6af5339c52 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -104,7 +104,10 @@ jobs:
# Needed to perform push action
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >>
${GITHUB_OUTPUT}
@@ -192,7 +195,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 47230c556c2..0e6cd9f11ae 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -78,6 +78,7 @@ jobs:
with:
platform: "linux/amd64"
python-versions-list-as-string: ${{
inputs.python-versions-list-as-string }}
+ docker-volume-location: "" # TODO(jscheffl): Understand why it
fails here and fix it
- name: "Verify all CI images ${{ inputs.python-versions-list-as-string
}}"
run: breeze ci-image verify --run-in-parallel
- name: "Source constraints"
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index 1b4aa19cbe5..e45d86ae576 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -97,7 +97,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 0b8951d663a..4bd13547b7f 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -137,7 +137,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
if: inputs.upload-package-artifact == 'true'
- name: "Cleanup dist and context file"
shell: bash
@@ -222,7 +225,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index 7698fc88e53..5d0e3ec1d13 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -126,7 +126,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -202,7 +205,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index b8758146cc1..287c6fd9790 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -60,7 +60,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -99,7 +102,10 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
+ # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
+ env:
+ TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/scripts/ci/cleanup_docker.sh b/scripts/ci/cleanup_docker.sh
index 400cc501541..8f195fe7550 100755
--- a/scripts/ci/cleanup_docker.sh
+++ b/scripts/ci/cleanup_docker.sh
@@ -19,6 +19,12 @@ function cleanup_docker {
# This is faster than docker prune
sudo systemctl stop docker
sudo rm -rf /var/lib/docker
+ # If a path is provided in ENV, bind mount it to /var/lib/docker
+ if [ -n "${TARGET_DOCKER_VOLUME_LOCATION}" ]; then
+ echo "Mounting ${TARGET_DOCKER_VOLUME_LOCATION} to /var/lib/docker"
+ sudo mkdir -p "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker
+ sudo mount --bind "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker
+ fi
sudo systemctl start docker
}
