This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 654038af5d Extract provider checks as separate workflow (from CI
workflow) (#38130)
654038af5d is described below
commit 654038af5dbea30f2d630746390aebdcd11afd65
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Mar 15 21:59:32 2024 +0100
Extract provider checks as separate workflow (from CI workflow) (#38130)
---
.github/workflows/check-providers.yml | 250 ++++++++++++++++++++++++++++++++++
.github/workflows/ci.yml | 207 +++-------------------------
2 files changed, 268 insertions(+), 189 deletions(-)
diff --git a/.github/workflows/check-providers.yml
b/.github/workflows/check-providers.yml
new file mode 100644
index 0000000000..23855aa510
--- /dev/null
+++ b/.github/workflows/check-providers.yml
@@ -0,0 +1,250 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Provider tests
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ runs-on:
+ description: "The array of labels (in json form) determining type of
the runner to use for the build."
+ required: false
+ default: '["ubuntu-22.04"]'
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
+ required: true
+ type: string
+ default-python-version:
+ description: Which version of python should be used get CI image
+ required: true
+ type: string
+ upgrade-to-newer-dependencies:
+ description: "Whether to upgrade to newer dependencies"
+ required: true
+ type: string
+ affected-providers-list-as-string:
+ description: "List of affected providers as string"
+ required: false
+ type: string
+ providers-compatibility-checks:
+ description: >
+ JSON-formatted array of providers compatibility checks in the form
of array of dicts
+ (airflow-version, python-versions, remove-providers)
+ required: true
+ type: string
+ skip-provider-tests:
+ description: "Whether to skip provider tests (true/false)"
+ required: true
+ type: string
+ python-versions:
+ description: "JSON-formatted array of Python versions to build images
from"
+ required: true
+ type: string
+ breeze-python-version:
+ description: >
+ Which version of python should be used to install Breeze (3.9 is
minimum for reproducible builds)
+ required: true
+ type: string
+jobs:
+ prepare-install-verify-provider-packages-wheel:
+ timeout-minutes: 80
+ name: "Provider packages wheel build and verify"
+ runs-on: ${{fromJSON(inputs.runs-on)}}
+ env:
+ RUNS_ON: "${{ inputs.runs-on }}"
+ PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: >
+ Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{
inputs.image-tag }}
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ python-version: ${{ inputs.breeze-python-version }}
+ - name: "Cleanup dist files"
+ run: rm -fv ./dist/*
+ - name: "Prepare provider documentation"
+ run: >
+ breeze release-management prepare-provider-documentation
--include-not-ready-providers
+ --non-interactive
+ - name: "Prepare provider packages: wheel"
+ run: >
+ breeze release-management prepare-provider-packages
--include-not-ready-providers
+ --version-suffix-for-pypi dev0 --package-format wheel
+ - name: "Prepare airflow package: wheel"
+ run: breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0
+ - name: "Verify wheel packages with twine"
+ run: |
+ pipx uninstall twine || true
+ pipx install twine && twine check dist/*.whl
+ - name: "Test providers issue generation automatically"
+ run: >
+ breeze release-management generate-issue-content-providers
+ --only-available-in-dist --disable-progress
+ - name: "Generate source constraints from CI image"
+ shell: bash
+ run: >
+ breeze release-management generate-constraints
+ --airflow-constraints-mode constraints-source-providers --answer yes
+ - name: "Install and verify all provider packages and airflow via wheel
files"
+ run: >
+ breeze release-management verify-provider-packages
+ --use-packages-from-dist
+ --package-format wheel
+ --use-airflow-version wheel
+ --airflow-constraints-reference default
+ --providers-constraints-location
+
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+ env:
+ AIRFLOW_SKIP_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies
}}"
+ - name: "Prepare airflow package: wheel without suffix and skipping the
tag check"
+ run: >
+ breeze release-management prepare-provider-packages --skip-tag-check
--package-format wheel
+
+ prepare-install-provider-packages-sdist:
+ timeout-minutes: 80
+ name: "Provider packages sdist build and install"
+ runs-on: ${{fromJSON(inputs.runs-on)}}
+ env:
+ RUNS_ON: "${{ inputs.runs-on }}"
+ PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: >
+ Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{
inputs.image-tag }}
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ python-version: ${{ inputs.breeze-python-version }}
+ - name: "Cleanup dist files"
+ run: rm -fv ./dist/*
+ - name: "Prepare provider packages: sdist"
+ run: >
+ breeze release-management prepare-provider-packages
--include-not-ready-providers
+ --version-suffix-for-pypi dev0 --package-format sdist
+ ${{ inputs.affected-providers-list-as-string }}
+ - name: "Prepare airflow package: sdist"
+ run: >
+ breeze release-management prepare-airflow-package
+ --version-suffix-for-pypi dev0 --package-format sdist
+ - name: "Verify sdist packages with twine"
+ run: |
+ pipx uninstall twine || true
+ pipx install twine && twine check dist/*.tar.gz
+ - name: "Generate source constraints from CI image"
+ shell: bash
+ run: >
+ breeze release-management generate-constraints
+ --airflow-constraints-mode constraints-source-providers --answer yes
+ - name: "Install all provider packages and airflow via sdist files"
+ run: >
+ breeze release-management install-provider-packages
+ --use-packages-from-dist
+ --package-format sdist
+ --use-airflow-version sdist
+ --airflow-constraints-reference default
+ --providers-constraints-location
+
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+ --run-in-parallel
+ if: inputs.affected-providers-list-as-string == ''
+ - name: "Install affected provider packages and airflow via sdist files"
+ run: >
+ breeze release-management install-provider-packages
+ --use-packages-from-dist
+ --package-format sdist
+ --use-airflow-version sdist
+ --airflow-constraints-reference default
+ --providers-constraints-location
+
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+ --run-in-parallel
+ if: inputs.affected-providers-list-as-string != ''
+
+ providers-compatibility-checks:
+ timeout-minutes: 80
+ name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }}
provider check
+ runs-on: ${{fromJSON(inputs.runs-on)}}
+ strategy:
+ fail-fast: false
+ matrix:
+ include: ${{fromJson(inputs.providers-compatibility-checks)}}
+ env:
+ RUNS_ON: "${{ inputs.runs-on }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
+ VERSION_SUFFIX_FOR_PYPI: "dev0"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ if: inputs.skip-provider-tests != 'true'
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: "Prepare breeze & CI image: ${{ matrix.python-version }}:${{
inputs.image-tag }}"
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ python-version: ${{ inputs.breeze-python-version }}
+ - name: "Cleanup dist files"
+ run: rm -fv ./dist/*
+ - name: "Prepare provider packages: wheel"
+ run: >
+ breeze release-management prepare-provider-packages
--include-not-ready-providers
+ --package-format wheel
+ - name: >
+ Remove incompatible Airflow
+ ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }}
provider packages
+ run: |
+ rm -vf ${{ matrix.remove-providers }}
+ working-directory: ./dist
+ if: matrix.remove-providers != ''
+ - name: "Download airflow package: wheel"
+ run: |
+ pip download "apache-airflow==${{ matrix.airflow-version }}" -d dist
--no-deps
+ - name: >
+ Install and verify all provider packages and airflow on
+ Airflow ${{ matrix.airflow-version }}:Python ${{
matrix.python-version }}
+ run: >
+ breeze release-management verify-provider-packages
+ --use-packages-from-dist
+ --package-format wheel
+ --use-airflow-version wheel
+ --airflow-constraints-reference
constraints-${{matrix.airflow-version}}
+ --providers-skip-constraints
+ --install-airflow-with-constraints
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 64ecdb93a1..26e66bb070 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -818,198 +818,27 @@ jobs:
run: >
breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string
}} --spellcheck-only
- prepare-test-provider-packages-wheel:
- timeout-minutes: 80
- name: "Provider packages wheel (verify)"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+ providers:
+ name: "Provider checks"
+ uses: ./.github/workflows/check-providers.yml
needs: [build-info, wait-for-ci-images]
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ permissions:
+ contents: read
+ packages: read
+ secrets: inherit
if: >
- needs.build-info.outputs.skip-provider-tests != 'true' &&
+ needs.build-info.outputs.skip-providers-tests != 'true' &&
needs.build-info.outputs.latest-versions-only != 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: >
- Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/prepare_breeze_and_image
- with:
- python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- - name: "Cleanup dist files"
- run: rm -fv ./dist/*
- - name: "Prepare provider documentation"
- run: >
- breeze release-management prepare-provider-documentation
--include-not-ready-providers
- --non-interactive
- - name: "Prepare provider packages: wheel"
- run: >
- breeze release-management prepare-provider-packages
--include-not-ready-providers
- --version-suffix-for-pypi dev0 --package-format wheel
- - name: "Prepare airflow package: wheel"
- run: breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0
- - name: "Verify wheel packages with twine"
- run: |
- pipx uninstall twine || true
- pipx install twine && twine check dist/*.whl
- - name: "Test providers issue generation automatically"
- run: >
- breeze release-management generate-issue-content-providers
- --only-available-in-dist --disable-progress
- - name: "Generate source constraints from CI image"
- shell: bash
- run: >
- breeze release-management generate-constraints
- --airflow-constraints-mode constraints-source-providers
- - name: "Install and verify all provider packages and airflow via wheel
files"
- run: >
- breeze release-management verify-provider-packages
- --use-packages-from-dist
- --package-format wheel
- --use-airflow-version wheel
- --airflow-constraints-reference default
- --providers-constraints-location
-
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
- env:
- AIRFLOW_SKIP_CONSTRAINTS: "${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- - name: "Prepare airflow package: wheel without suffix and skipping the
tag check"
- run: >
- breeze release-management prepare-provider-packages --skip-tag-check
--package-format wheel
-
- prepare-install-provider-packages-sdist:
- timeout-minutes: 80
- name: "Provider packages sdist (install)"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-ci-images]
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- if: >
- needs.build-info.outputs.canary-run == 'true' &&
- needs.build-info.outputs.skip-provider-tests != 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: >
- Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/prepare_breeze_and_image
- with:
- python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- - name: "Cleanup dist files"
- run: rm -fv ./dist/*
- - name: "Prepare provider packages: sdist"
- run: >
- breeze release-management prepare-provider-packages
--include-not-ready-providers
- --version-suffix-for-pypi dev0 --package-format sdist
- ${{ needs.build-info.outputs.affected-providers-list-as-string }}
- - name: "Prepare airflow package: sdist"
- run: >
- breeze release-management prepare-airflow-package
- --version-suffix-for-pypi dev0 --package-format sdist
- - name: "Verify sdist packages with twine"
- run: |
- pipx uninstall twine || true
- pipx install twine && twine check dist/*.tar.gz
- - name: "Generate source constraints from CI image"
- shell: bash
- run: >
- breeze release-management generate-constraints
- --airflow-constraints-mode constraints-source-providers
- - name: "Install all provider packages and airflow via sdist files"
- run: >
- breeze release-management install-provider-packages
- --use-packages-from-dist
- --package-format sdist
- --use-airflow-version sdist
- --airflow-constraints-reference default
- --providers-constraints-location
-
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
- --run-in-parallel
- if: needs.build-info.outputs.affected-providers-list-as-string == ''
- - name: "Install affected provider packages and airflow via sdist files"
- run: >
- breeze release-management install-provider-packages
- --use-packages-from-dist
- --package-format sdist
- --use-airflow-version sdist
- --airflow-constraints-reference default
- --providers-constraints-location
-
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
- --run-in-parallel
- if: needs.build-info.outputs.affected-providers-list-as-string != ''
-
- providers-compatibility-checks:
- timeout-minutes: 80
- name: >
- Compat ${{matrix.airflow-version}}:P${{matrix.python-version}} provider
check
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-ci-images]
- strategy:
- fail-fast: false
- matrix:
- include:
${{fromJson(needs.build-info.outputs.providers-compatibility-checks)}}
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- if: needs.build-info.outputs.skip-provider-tests != 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: >
- Prepare breeze & CI image:
${{matrix.python-version}}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/prepare_breeze_and_image
- with:
- python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- - name: "Cleanup dist files"
- run: rm -fv ./dist/*
- - name: "Prepare provider packages: wheel"
- run: >
- breeze release-management prepare-provider-packages
--include-not-ready-providers
- --package-format wheel
- - name: >
- Remove incompatible Airflow
- ${{matrix.airflow-version}}:Python ${{matrix.python-version}}
provider packages
- run: |
- rm -vf ${{ matrix.remove-providers }}
- working-directory: ./dist
- if: matrix.remove-providers != ''
- - name: "Download airflow package: wheel"
- run: |
- pip download "apache-airflow==${{matrix.airflow-version}}" -d dist
--no-deps
- - name: >
- Install and verify all provider packages and airflow on
- Airflow ${{matrix.airflow-version}}:Python ${{matrix.python-version}}
- run: >
- breeze release-management verify-provider-packages
- --use-packages-from-dist
- --package-format wheel
- --use-airflow-version wheel
- --airflow-constraints-reference
constraints-${{matrix.airflow-version}}
- --providers-skip-constraints
- --install-airflow-with-constraints
+ with:
+ runs-on: ${{ needs.build-info.outputs.runs-on }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
+ default-python-version: ${{
needs.build-info.outputs.default-python-version }}
+ upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ affected-providers-list-as-string: ${{
needs.build-info.outputs.affected-providers-list-as-string }}
+ providers-compatibility-checks: ${{
needs.build-info.outputs.providers-compatibility-checks }}
+ skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
test-airflow-release-commands:
timeout-minutes: 80