This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a4977c14cf Extract production image tests to a separate workflow
(#38210)
a4977c14cf is described below
commit a4977c14cf70b02dffb083ceae8fc527b61c2a08
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Mar 16 16:57:49 2024 +0100
Extract production image tests to a separate workflow (#38210)
Another common part of the CI workflow that could be extracted to
a separate workflow. Those are all PROD image tests that can be
run in the same foldable section of the CI
---
.github/actions/build-prod-images/action.yml | 2 +-
.github/workflows/all-prod-image-tests.yml | 158 +++++++++++++++++++++++++++
.github/workflows/build-images.yml | 2 +-
.github/workflows/ci.yml | 125 +++------------------
.github/workflows/prod-image-build.yml | 2 +-
5 files changed, 179 insertions(+), 110 deletions(-)
diff --git a/.github/actions/build-prod-images/action.yml
b/.github/actions/build-prod-images/action.yml
index 0493c148c0..e50cc29e57 100644
--- a/.github/actions/build-prod-images/action.yml
+++ b/.github/actions/build-prod-images/action.yml
@@ -77,7 +77,7 @@ runs:
name: source-constraints-${{ inputs.python-version }}
path: ./docker-context-files
if: ${{ inputs.build-provider-packages == 'true' }}
- - name: "Download constraints from the Generate & Verify build"
+ - name: "Download constraints"
uses: actions/download-artifact@v4
with:
name: constraints
diff --git a/.github/workflows/all-prod-image-tests.yml
b/.github/workflows/all-prod-image-tests.yml
new file mode 100644
index 0000000000..57ebd000d3
--- /dev/null
+++ b/.github/workflows/all-prod-image-tests.yml
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: All PROD image tests
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ runs-on:
+ description: "The array of labels (in json form) determining type of
the runner to use for the build."
+ required: false
+ default: '["ubuntu-22.04"]'
+ type: string
+ default-branch:
+ description: "The default branch for the repository"
+ required: true
+ type: string
+ constraints-branch:
+ description: "Branch used to construct constraints URL from."
+ required: true
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
+ required: true
+ type: string
+ upgrade-to-newer-dependencies:
+ description: "Whether to upgrade to newer dependencies (true/false)"
+ required: true
+ type: string
+ chicken-egg-providers:
+ description: "Whether to build chicken-egg provider packages in the
same run (true/false)"
+ required: true
+ type: string
+ docker-cache:
+ description: "Whether to use Docker cache (true/false)"
+ required: true
+ type: string
+ canary-run:
+ description: "Whether to run the canary run (true/false)"
+ required: true
+ type: string
+ default-python-version:
+ description: Which version of python should be used get CI image
+ required: true
+ type: string
+ breeze-python-version:
+ description: >
+ Which version of python should be used to install Breeze (3.9 is
minimum for reproducible builds)
+ required: true
+ type: string
+jobs:
+ prod-image-extra-checks-main:
+ name: PROD image extra checks (main)
+ uses: ./.github/workflows/prod-image-extra-checks.yml
+ with:
+ python-versions: "[ '${{ inputs.default-python-version }}' ]"
+ branch: ${{ inputs.default-branch }}
+ image-tag: ${{ inputs.image-tag }}
+ build-provider-packages: ${{ inputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
+ breeze-python-version: ${{ inputs.breeze-python-version }}
+ chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.default-branch == 'main' && inputs.canary-run == 'true'
+
+ prod-image-extra-checks-release-branch:
+ name: PROD image extra checks (release)
+ uses: ./.github/workflows/prod-image-extra-checks.yml
+ with:
+ python-versions: "[ '${{ inputs.default-python-version }}' ]"
+ branch: ${{ inputs.default-branch }}
+ image-tag: ${{ inputs.image-tag }}
+ build-provider-packages: ${{ inputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
+ breeze-python-version: ${{ inputs.breeze-python-version }}
+ chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.default-branch != 'main' && inputs.canary-run == 'true'
+
+ test-examples-of-prod-image-building:
+ timeout-minutes: 60
+ name: "Test examples of POD image building"
+ runs-on: ["ubuntu-22.04"]
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 2
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ - name: Pull PROD image ${{ inputs.default-python-version}}:${{
inputs.image-tag }}
+ run: breeze prod-image pull --tag-as-latest
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ - name: "Setup python"
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ inputs.default-python-version }}
+ cache: 'pip'
+ cache-dependency-path: ./dev/requirements.txt
+ - name: "Test examples of PROD image building"
+ run: "
+ cd ./docker_tests && \
+ python -m pip install -r requirements.txt && \
+ TEST_IMAGE=\"ghcr.io/${{ github.repository }}/${{
inputs.default-branch }}\
+ /prod/python${{ inputs.default-python-version }}:${{
inputs.image-tag }}\" \
+ python -m pytest test_examples_of_prod_image_building.py -n auto
--color=yes"
+
+ test-docker-compose-quick-start:
+ timeout-minutes: 60
+ name: "Docker-compose quick start with PROD image verifying"
+ runs-on: ["ubuntu-22.04"]
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 2
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ - name: "Pull image ${{ inputs.default-python-version}}:${{
inputs.image-tag }}"
+ run: breeze prod-image pull --tag-as-latest
+ - name: "Test docker-compose quick start"
+ run: breeze testing docker-compose-tests
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index 2b4abed5ee..1ee19fb76a 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -72,7 +72,7 @@ jobs:
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
- default-constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
+ constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
runs-on: ${{steps.selective-checks.outputs.runs-on}}
is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
is-committer-build: ${{
steps.selective-checks.outputs.is-committer-build }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 331782f1db..7dea803dd0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -945,40 +945,6 @@ jobs:
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- prod-image-extra-checks-main:
- name: PROD image extra checks (main)
- # Here we just need to wait for CI images: we build provider packages and
can use source constraints
- needs: [build-info, build-ci-images]
- uses: ./.github/workflows/prod-image-extra-checks.yml
- with:
- python-versions: "[ '${{ needs.build-info.outputs.default-python-version
}}' ]"
- branch: ${{ needs.build-info.outputs.default-branch }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- build-provider-packages: ${{ needs.build-info.outputs.default-branch ==
'main' }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
- chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- if: needs.build-info.outputs.default-branch == 'main' &&
needs.build-info.outputs.canary-run == 'true'
-
- prod-image-extra-checks-release-branch:
- name: PROD image extra checks (release)
- # Here we need to wait for generate-constraints to complete because we
have to use PyPI constraints
- needs: [build-info, generate-constraints]
- uses: ./.github/workflows/prod-image-extra-checks.yml
- with:
- python-versions: "[ '${{ needs.build-info.outputs.default-python-version
}}' ]"
- branch: ${{ needs.build-info.outputs.default-branch }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- build-provider-packages: ${{ needs.build-info.outputs.default-branch ==
'main' }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
- chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- if: needs.build-info.outputs.default-branch != 'main' &&
needs.build-info.outputs.canary-run == 'true'
-
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for PROD images"
@@ -992,6 +958,7 @@ jobs:
# Force more parallelism for pull on public images
PARALLELISM: 6
INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
+ IMAGE_TAG: ${{ needs.build-info.outputs.image-tag }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -1018,84 +985,28 @@ jobs:
PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
-
- test-examples-of-prod-image-building:
- timeout-minutes: 60
- name: "Test examples of production image building"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-prod-images]
- if: needs.build-info.outputs.prod-image-build == 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- - name: >
- Pull PROD image
- "${{needs.build-info.outputs.default-python-version}}":${{
needs.build-info.outputs.image-tag }}
- run: breeze prod-image pull --tag-as-latest
- env:
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- - name: "Setup python"
- uses: actions/setup-python@v5
- with:
- python-version: ${{needs.build-info.outputs.default-python-version}}
- cache: 'pip'
- cache-dependency-path: ./dev/requirements.txt
- - name: "Test examples of PROD image building"
- # yamllint disable-line rule:line-length
- run: >
- cd ./docker_tests &&
- python -m pip install -r requirements.txt &&
-
TEST_IMAGE="ghcr.io/${{env.GITHUB_REPOSITORY}}/${{needs.build-info.outputs.default-branch}}/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
- python -m pytest test_examples_of_prod_image_building.py -n auto
--color=yes
- env:
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
-
-
- test-docker-compose-quick-start:
- timeout-minutes: 60
- name: "Docker-compose quick start with PROD image verifying"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-prod-images]
- if: needs.build-info.outputs.prod-image-build == 'true'
- env:
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{
needs.build-info.outputs.image-tag }}
- run: breeze prod-image pull --run-in-parallel --tag-as-latest
- env:
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- # Force more parallelism for pull even on public images
- PARALLELISM: 6
- name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{
needs.build-info.outputs.image-tag }}
run: breeze prod-image verify --run-in-parallel
env:
PYTHON_VERSIONS:
${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- - name: "Test docker-compose quick start"
- run: breeze testing docker-compose-tests
+
+ all-prod-image-tests:
+ name: "All PROD image tests"
+ needs: [build-info, build-prod-images, generate-constraints]
+ uses: ./.github/workflows/all-prod-image-tests.yml
+ with:
+ runs-on: ${{ needs.build-info.outputs.runs-on }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+ constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+ breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
+ default-python-version: ${{
needs.build-info.outputs.default-python-version }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ if: needs.build-info.outputs.prod-image-build == 'true'
tests-kubernetes:
name: "Kubernetes tests"
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index f9768eca1a..ad3c947ec3 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -167,7 +167,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
name: source-constraints-${{ matrix.python-version }}
path: ./docker-context-files
if: inputs.do-build == 'true' && inputs.build-provider-packages ==
'true'
- - name: "Download constraints from the Generate & Verify build"
+ - name: "Download constraints"
uses: actions/download-artifact@v4
with:
name: constraints
