This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch extract-build-images-to-composite-actions in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 11accb0fb920b2d3f3a5e39e0efc51f7e10722f0 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Oct 30 02:44:00 2022 +0200 Extract Build images to composite actions Follow-up after #27371 --- .github/actions/build-ci-images/action.yml | 58 +++++++++++++++ .github/actions/build-prod-images/action.yml | 71 ++++++++++++++++++ .github/workflows/build-images.yml | 101 +++----------------------- .github/workflows/ci.yml | 103 ++++----------------------- 4 files changed, 151 insertions(+), 182 deletions(-) diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml new file mode 100644 index 0000000000..62e8ead430 --- /dev/null +++ b/.github/actions/build-ci-images/action.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Build CI images' +description: 'Build CI images' +inputs: + regenerate-dependencies: + description: 'Whether to regenerate dependencies before build' + required: true +runs: + using: "composite" + steps: + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + shell: bash + run: | + pip install rich>=12.4.4 pyyaml + python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py + if: ${{ inputs.regenerate-dependencies }} + - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ env.PYTHON_VERSIONS }}" + shell: bash + run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure + - name: "Show dependencies to be upgraded" + shell: bash + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-source-providers + if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' + - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + if: failure() || cancelled() + shell: bash + run: breeze ci-image build --push --empty-image --run-in-parallel + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Candidates for pip resolver backtrack triggers" + shell: bash + run: breeze ci find-newer-dependencies --max-age 1 --python 3.7 + if: failure() || cancelled() + - name: "Fix ownership" + shell: bash + run: breeze ci fix-ownership + if: always() diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml new file mode 100644 index 0000000000..7beb2147c3 --- /dev/null +++ b/.github/actions/build-prod-images/action.yml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Build PROD images' +description: 'Build PROD images' +inputs: + build-provider-packages: + description: 'Whether to build provider packages' + required: true +runs: + using: "composite" + steps: + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + shell: bash + run: | + pip install rich>=12.4.4 pyyaml + python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py + if: ${{ inputs.build-provider-packages }} + - name: "Pull CI image for PROD build: ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}" + shell: bash + run: breeze ci-image pull --tag-as-latest + env: + PYTHON_MAJOR_MINOR_VERSION: "3.7" + - name: "Cleanup dist and context file" + shell: bash + run: rm -fv ./dist/* ./docker-context-files/* + - name: "Prepare providers packages" + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-list-file ./scripts/ci/installed_providers.txt + --package-format wheel --version-suffix-for-pypi dev0 + if: ${{ inputs.build-provider-packages }} + - name: "Prepare airflow package" + shell: bash + run: > + breeze release-management prepare-airflow-package + --package-format wheel --version-suffix-for-pypi dev0 + - name: "Move dist packages to docker-context files" + shell: bash + run: mv -v ./dist/*.whl ./docker-context-files + - name: "Build & Push PROD images ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}" + shell: bash + run: > + breeze prod-image build --tag-as-latest --run-in-parallel --push + --install-packages-from-context --upgrade-on-failure + - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + shell: bash + run: breeze prod-image build --cleanup-context --push --empty-image --run-in-parallel + if: failure() || cancelled() + - name: "Fix ownership" + shell: bash + run: breeze ci fix-ownership + if: always() diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 11a7a61be3..10326d485c 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -173,7 +173,6 @@ jobs: DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} RUNS_ON: ${{ needs.build-info.outputs.runs-on }} BACKEND: sqlite - outputs: ${{toJSON(needs.build-info.outputs) }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -182,10 +181,6 @@ jobs: ref: ${{ needs.build-info.outputs.target-commit-sha }} persist-credentials: false submodules: recursive - - name: "Setup python" - uses: actions/setup-python@v4 - with: - python-version: ${{ needs.build-info.outputs.default-python-version }} - name: > Checkout "main branch to 'main-airflow' folder to use ci/scripts from there. @@ -206,42 +201,16 @@ jobs: rm -rfv "dev" mv -v "main-airflow/scripts/ci" "scripts" mv -v "main-airflow/dev" "." - - name: "Regenerate dependencies in case they was modified manually so that we can build an image" - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py - if: needs.build-info.outputs.default-branch == 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: >- - Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure + - name: Build CI Images + uses: ./.github/action/build-ci-images + with: + regenerate-dependencies: ${{ needs.build-info.outputs.default-branch == 'main' }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - - name: "Show dependencies to be upgraded" - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-source-providers - env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' - - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - if: failure() || cancelled() - run: breeze ci-image build --push --empty-image --run-in-parallel - env: - IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Candidates for pip resolver backtrack triggers" - if: failure() || cancelled() - run: > - breeze ci find-newer-dependencies --max-age 1 - --python "${{ needs.build-info.outputs.default-python-version }}" - - name: "Fix ownership" - run: breeze ci fix-ownership - if: always() + DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} build-prod-images: permissions: @@ -286,66 +255,17 @@ jobs: rm -rfv "dev" mv -v "main-airflow/scripts/ci" "scripts" mv -v "main-airflow/dev" "." - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: Cache pre-commit envs - uses: actions/cache@v3 + - name: Build PROD Images + uses: ./.github/action/build-prod-images + if: needs.build-info.outputs.in-workflow-build == 'true' with: - path: ~/.cache/pre-commit - key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - - name: "Regenerate dependencies in case they was modified manually so that we can build an image" - run: > - breeze static-checks --type update-providers-dependencies --all-files - --show-diff-on-failure --color always || true - - name: > - Pull CI image for PROD build: - ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze ci-image pull --tag-as-latest - env: - # Always use default Python version of CI image for preparing packages - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} - IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Prepare providers packages" - run: > - breeze release-management prepare-provider-packages - --package-list-file ./scripts/ci/installed_providers.txt - --package-format wheel - --version-suffix-for-pypi dev0 - if: needs.build-info.outputs.default-branch == 'main' - - name: "Prepare airflow package" - run: > - breeze release-management prepare-airflow-package - --package-format wheel --version-suffix-for-pypi dev0 - - name: "Move dist packages to docker-context files" - run: mv -v ./dist/*.whl ./docker-context-files - - name: >- - Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - run: > - breeze prod-image build - --run-in-parallel - --tag-as-latest - --push - --install-packages-from-context - --upgrade-on-failure + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - if: failure() || cancelled() - run: breeze prod-image build --cleanup-context --push --empty-image --run-in-parallel - env: - IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Fix ownership" - run: breeze ci fix-ownership - if: always() - + DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} build-ci-images-arm: timeout-minutes: 50 @@ -390,7 +310,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" rm -rfv "dev" mv -v "main-airflow/scripts/ci" "scripts" mv -v "main-airflow/dev" "." - - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: > diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65d0d1138a..2a734045e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -349,44 +349,19 @@ jobs: persist-credentials: false submodules: recursive if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Install Breeze" - uses: ./.github/actions/breeze + - name: Build CI Images + uses: ./.github/action/build-ci-images if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Regenerate dependencies in case they was modified manually so that we can build an image" - run: | - pip install rich>=12.4.4 pyyaml - python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: > - Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure + with: + regenerate-dependencies: > + ${{ needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Show dependencies to be upgraded" - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-source-providers - env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - if: > - needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && - needs.build-info.outputs.in-workflow-build == 'true' - - name: "Candidates for pip resolver backtrack triggers" - if: failure() || cancelled() - run: > - breeze ci find-newer-dependencies --max-age 1 - --python "${{ needs.build-info.outputs.default-python-version }}" - - name: "Fix ownership" - run: breeze ci fix-ownership - if: always() && needs.build-info.outputs.in-workflow-build == 'true' build-prod-images: permissions: @@ -415,73 +390,19 @@ jobs: persist-credentials: false submodules: recursive if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Install Breeze" - uses: ./.github/actions/breeze + - name: Build PROD Images + uses: ./.github/action/build-prod-images if: needs.build-info.outputs.in-workflow-build == 'true' - - name: Cache pre-commit envs - uses: actions/cache@v3 with: - path: ~/.cache/pre-commit - key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: "Regenerate dependencies in case they was modified manually so that we can build an image" - run: > - breeze static-checks --type update-providers-dependencies --all-files - --show-diff-on-failure --color always || true - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: > - Pull CI image for PROD build: - ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" - run: breeze ci-image pull --tag-as-latest - env: - # Always use default Python version of CI image for preparing packages - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} - IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Prepare providers packages" - run: > - breeze release-management prepare-provider-packages - --package-list-file ./scripts/ci/installed_providers.txt - --package-format wheel --version-suffix-for-pypi dev0 - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: "Prepare airflow package" - run: > - breeze release-management prepare-airflow-package - --package-format wheel --version-suffix-for-pypi dev0 - if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Move dist packages to docker-context files" - run: mv -v ./dist/*.whl ./docker-context-files - if: needs.build-info.outputs.in-workflow-build == 'true' - - name: > - Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - run: > - breeze prod-image build - --tag-as-latest - --run-in-parallel - --push - --install-packages-from-context - --upgrade-on-failure + build-provider-packages: > + ${{ needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' }} env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - if: needs.build-info.outputs.in-workflow-build == 'true' - - name: "Fix ownership" - run: breeze ci fix-ownership - if: always() && needs.build-info.outputs.in-workflow-build == 'true' + DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} run-new-breeze-tests: timeout-minutes: 10
