This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0531e78398 Bring back documentation publishing to Canary builds
(#43509)
0531e78398 is described below
commit 0531e78398c14ca4d1c61dc6c082696c91d5dec1
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Oct 31 20:26:49 2024 +0100
Bring back documentation publishing to Canary builds (#43509)
We are using the same approach as in
https://github.com/apache/airflow-site/pull/1055/ where we utilise
/mnt directory to store checked out airflow-site.
This requires to switch back to public runners for canary builds
as public runners have space available in /mnt
Fixes: #41931
---
.github/workflows/static-checks-mypy-docs.yml | 16 +++++++++-------
dev/breeze/src/airflow_breeze/utils/selective_checks.py | 9 +++++----
dev/breeze/tests/test_selective_checks.py | 5 +----
3 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/static-checks-mypy-docs.yml
b/.github/workflows/static-checks-mypy-docs.yml
index b24aed2c1c..b34ad2c36f 100644
--- a/.github/workflows/static-checks-mypy-docs.yml
+++ b/.github/workflows/static-checks-mypy-docs.yml
@@ -236,8 +236,6 @@ jobs:
timeout-minutes: 150
name: "Publish documentation"
needs: build-docs
- # For canary runs we need to push documentation to AWS S3 and preparing it
takes a lot of space
- # So we should use self-hosted ASF runners for this
runs-on: ${{ fromJSON(inputs.runs-on-as-json-docs-build) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -248,9 +246,7 @@ jobs:
INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
VERBOSE: "true"
- # Temporary disabled it until self-hosted ASF runners will be working
again
- if: false
- # if: inputs.canary-run == 'true' && inputs.branch == 'main'
+ if: inputs.canary-run == 'true' && inputs.branch == 'main'
steps:
- name: "Cleanup repo"
shell: bash
@@ -266,16 +262,22 @@ jobs:
with:
name: airflow-docs
path: './docs/_build'
+ - name: Check disk space available
+ run: df -h
+ - name: Create /mnt/airflow-site directory
+ run: sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}"
/mnt/airflow-site
- name: "Clone airflow-site"
run: >
- git clone https://github.com/apache/airflow-site.git
${GITHUB_WORKSPACE}/airflow-site &&
- echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >>
"$GITHUB_ENV"
+ git clone https://github.com/apache/airflow-site.git
/mnt/airflow-site/airflow-site &&
+ echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/airflow-site" >>
"$GITHUB_ENV"
- name: "Prepare breeze & CI image: ${{ inputs.default-python-version
}}:${{ inputs.image-tag }}"
uses: ./.github/actions/prepare_breeze_and_image
- name: "Publish docs"
run: >
breeze release-management publish-docs --override-versioned
--run-in-parallel
${{ inputs.docs-list-as-string }}
+ - name: Check disk space available
+ run: df -h
- name: "Generate back references for providers"
run: breeze release-management add-back-references all-providers
- name: "Generate back references for apache-airflow"
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 7a1b802fa5..ae344b66c0 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1228,10 +1228,11 @@ class SelectiveChecks:
@cached_property
def runs_on_as_json_docs_build(self) -> str:
- if self._is_canary_run():
- return RUNS_ON_SELF_HOSTED_ASF_RUNNER
- else:
- return RUNS_ON_PUBLIC_RUNNER
+ # We used to run docs build on self-hosted runners because they had
more space, but
+ # It turned out that public runners have a lot of space in /mnt folder
that we can utilise
+ # but in the future we might want to switch back to self-hosted
runners so we have this
+ # separate property to determine that and place to implement different
logic if needed
+ return RUNS_ON_PUBLIC_RUNNER
@cached_property
def runs_on_as_json_public(self) -> str:
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index a6b7963848..d67508eafe 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1909,11 +1909,8 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
(),
dict(),
- # TODO: revert it when we fix self-hosted runners
'["ubuntu-22.04"]',
- '["self-hosted", "asf-runner"]',
- # '["self-hosted", "Linux", "X64"]',
- # TODO: revert it when we fix self-hosted runners
+ '["ubuntu-22.04"]',
"false",
"false",
# "true",
