This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6ddf3ae0a8 Extract docs to a separate workflow (#38211)
6ddf3ae0a8 is described below
commit 6ddf3ae0a874948c9d81aa5f0958b91ffd45233b
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 16 17:40:27 2024 +0100
Extract docs to a separate workflow (#38211)
Similarly to other extranctiosn - doc build can be separated to
a separate workflow, additionally, we can turn both docs and
spelling jobs into a single matrix job with some conditionals
easily.
---
.github/workflows/ci.yml | 122 ++++++-------------------------------------
.github/workflows/docs.yml | 126 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 143 insertions(+), 105 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e65f3a94db..660ef62bae 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -453,111 +453,25 @@ jobs:
needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
if: needs.build-info.outputs.latest-versions-only != 'true'
- build-docs:
- timeout-minutes: 60
- name: "Build docs"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-ci-images]
- if: >
- needs.build-info.outputs.docs-build == 'true' &&
- needs.build-info.outputs.latest-versions-only != 'true'
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: >
- Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/prepare_breeze_and_image
- - uses: actions/cache@v4
- id: cache-doc-inventories
- with:
- path: ./docs/_inventory_cache/
- key: docs-inventory-${{ hashFiles('pyproject.toml;') }}
- restore-keys: |
- docs-inventory-${{ hashFiles('pyproject.toml;') }}
- docs-inventory-
- - name: "Build docs"
- run: >
- breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string
}} --docs-only
- - name: "Clone airflow-site"
- run: >
- git clone https://github.com/apache/airflow-site.git
${GITHUB_WORKSPACE}/airflow-site &&
- echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >>
"$GITHUB_ENV"
- if: needs.build-info.outputs.canary-run == 'true'
- - name: "Publish docs"
- run: >
- breeze release-management publish-docs
- --override-versioned --run-in-parallel
- ${{ needs.build-info.outputs.docs-list-as-string }}
- if: needs.build-info.outputs.canary-run == 'true'
- - name: "Generate back references for providers"
- run: breeze release-management add-back-references all-providers
- if: needs.build-info.outputs.canary-run == 'true'
- - name: "Generate back references for apache-airflow"
- run: breeze release-management add-back-references apache-airflow
- if: needs.build-info.outputs.canary-run == 'true'
- - name: "Generate back references for docker-stack"
- run: breeze release-management add-back-references docker-stack
- if: needs.build-info.outputs.canary-run == 'true'
- - name: "Generate back references for helm-chart"
- run: breeze release-management add-back-references helm-chart
- if: needs.build-info.outputs.canary-run == 'true'
- - name: Configure AWS credentials
- uses:
aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
# v4.0.1
- if: needs.build-info.outputs.canary-run == 'true' &&
needs.build-info.outputs.default-branch == 'main'
- with:
- aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
- aws-region: eu-central-1
- - name: "Upload documentation to AWS S3"
- if: needs.build-info.outputs.canary-run == 'true' &&
needs.build-info.outputs.default-branch == 'main'
- run: aws s3 sync --delete ./files/documentation
s3://apache-airflow-docs
-
- spellcheck-docs:
- timeout-minutes: 120
- name: "Spellcheck docs"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+ docs:
+ name: "Docs"
+ uses: ./.github/workflows/docs.yml
needs: [build-info, wait-for-ci-images]
+ permissions:
+ contents: read
+ packages: read
+ secrets: inherit
if: >
needs.build-info.outputs.docs-build == 'true' &&
needs.build-info.outputs.latest-versions-only != 'true'
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: >
- Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/prepare_breeze_and_image
- - uses: actions/cache@v4
- id: cache-doc-inventories
- with:
- path: ./docs/_inventory_cache/
- key: docs-inventory-${{ hashFiles('pyproject.toml;') }}
- restore-keys: |
- docs-inventory-${{ hashFiles('pyproject.toml;') }}
- docs-inventory-
- - name: "Spellcheck docs"
- run: >
- breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string
}} --spellcheck-only
+ with:
+ runs-on: ${{ needs.build-info.outputs.runs-on }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+ docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
+ default-python-version: ${{
needs.build-info.outputs.default-python-version }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
providers:
name: "Provider checks"
@@ -937,8 +851,7 @@ jobs:
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- - build-docs
- - spellcheck-docs
+ - docs
- wait-for-ci-images
- wait-for-prod-images
- static-checks
@@ -1029,8 +942,7 @@ jobs:
name: Build CI ARM images (in-workflow)
needs:
- build-info
- - build-docs
- - spellcheck-docs
+ - docs
- static-checks
- mypy
- tests-sqlite
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000000..88b74625fa
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Documentation
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ runs-on:
+ description: "The array of labels (in json form) determining type of
the runner to use for the build."
+ required: false
+ default: '["ubuntu-22.04"]'
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
+ required: true
+ type: string
+ default-branch:
+ description: "Default branch of the repository"
+ required: true
+ type: string
+ docs-list-as-string:
+ description: "Stringified list of docs to build (space separated)"
+ required: true
+ type: string
+ default-python-version:
+ description: Which version of python should be used get CI image
+ required: true
+ type: string
+ include-success-outputs:
+ description: "Whether to include success outputs"
+ required: true
+ type: string
+ canary-run:
+ description: "Whether this is a canary run (true/false)"
+ required: true
+ type: string
+jobs:
+ build-docs:
+ timeout-minutes: 60
+ name: "Build documentation"
+ runs-on: ${{ fromJSON(inputs.runs-on) }}
+ strategy:
+ fail-fast: false
+ matrix:
+ flag: ["--docs-only", "--spellcheck-only"]
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: "Prepare breeze & CI image: ${{ inputs.default-python-version
}}:${{ inputs.image-tag }}"
+ uses: ./.github/actions/prepare_breeze_and_image
+ - uses: actions/cache@v4
+ id: cache-doc-inventories
+ with:
+ path: ./docs/_inventory_cache/
+ key: docs-inventory-${{ hashFiles('pyproject.toml;') }}
+ restore-keys: |
+ docs-inventory-${{ hashFiles('pyproject.toml;') }}
+ docs-inventory-
+ - name: "Building docs with ${{ matrix.flag }} flag"
+ run: >
+ breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag
}}
+ - name: "Clone airflow-site"
+ run: >
+ git clone https://github.com/apache/airflow-site.git
${GITHUB_WORKSPACE}/airflow-site &&
+ echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >>
"$GITHUB_ENV"
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Publish docs"
+ run: >
+ breeze release-management publish-docs --override-versioned
--run-in-parallel
+ ${{ inputs.docs-list-as-string }}
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for providers"
+ run: breeze release-management add-back-references all-providers
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for apache-airflow"
+ run: breeze release-management add-back-references apache-airflow
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for docker-stack"
+ run: breeze release-management add-back-references docker-stack
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for helm-chart"
+ run: breeze release-management add-back-references helm-chart
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: Configure AWS credentials
+ uses:
aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
# v4.0.1
+ if: >
+ inputs.canary-run == 'true' &&
+ inputs.default-branch == 'main' &&
+ matrix.flag == '--docs-only'
+ with:
+ aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+ aws-region: eu-central-1
+ - name: "Upload documentation to AWS S3"
+ if: >
+ inputs.canary-run == 'true' &&
+ inputs.default-branch == 'main' &&
+ matrix.flag == '--docs-only'
+ run: aws s3 sync --delete ./files/documentation
s3://apache-airflow-docs
