This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 654038af5d Extract provider checks as separate workflow (from CI 
workflow) (#38130)
654038af5d is described below

commit 654038af5dbea30f2d630746390aebdcd11afd65
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Mar 15 21:59:32 2024 +0100

    Extract provider checks as separate workflow (from CI workflow) (#38130)
---
 .github/workflows/check-providers.yml | 250 ++++++++++++++++++++++++++++++++++
 .github/workflows/ci.yml              | 207 +++-------------------------
 2 files changed, 268 insertions(+), 189 deletions(-)

diff --git a/.github/workflows/check-providers.yml 
b/.github/workflows/check-providers.yml
new file mode 100644
index 0000000000..23855aa510
--- /dev/null
+++ b/.github/workflows/check-providers.yml
@@ -0,0 +1,250 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Provider tests
+on:  # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      runs-on:
+        description: "The array of labels (in json form) determining type of 
the runner to use for the build."
+        required: false
+        default: '["ubuntu-22.04"]'
+        type: string
+      image-tag:
+        description: "Tag to set for the image"
+        required: true
+        type: string
+      default-python-version:
+        description: Which version of python should be used get CI image
+        required: true
+        type: string
+      upgrade-to-newer-dependencies:
+        description: "Whether to upgrade to newer dependencies"
+        required: true
+        type: string
+      affected-providers-list-as-string:
+        description: "List of affected providers as string"
+        required: false
+        type: string
+      providers-compatibility-checks:
+        description: >
+          JSON-formatted array of providers compatibility checks in the form 
of array of dicts
+          (airflow-version, python-versions, remove-providers)
+        required: true
+        type: string
+      skip-provider-tests:
+        description: "Whether to skip provider tests (true/false)"
+        required: true
+        type: string
+      python-versions:
+        description: "JSON-formatted array of Python versions to build images 
from"
+        required: true
+        type: string
+      breeze-python-version:
+        description: >
+          Which version of python should be used to install Breeze (3.9 is 
minimum for reproducible builds)
+        required: true
+        type: string
+jobs:
+  prepare-install-verify-provider-packages-wheel:
+    timeout-minutes: 80
+    name: "Provider packages wheel build and verify"
+    runs-on: ${{fromJSON(inputs.runs-on)}}
+    env:
+      RUNS_ON: "${{ inputs.runs-on }}"
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      IMAGE_TAG: "${{ inputs.image-tag }}"
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
+      - name: >
+          Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ 
inputs.image-tag }}
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          python-version: ${{ inputs.breeze-python-version }}
+      - name: "Cleanup dist files"
+        run: rm -fv ./dist/*
+      - name: "Prepare provider documentation"
+        run: >
+          breeze release-management prepare-provider-documentation 
--include-not-ready-providers
+          --non-interactive
+      - name: "Prepare provider packages: wheel"
+        run: >
+          breeze release-management prepare-provider-packages 
--include-not-ready-providers
+          --version-suffix-for-pypi dev0 --package-format wheel
+      - name: "Prepare airflow package: wheel"
+        run: breeze release-management prepare-airflow-package 
--version-suffix-for-pypi dev0
+      - name: "Verify wheel packages with twine"
+        run: |
+          pipx uninstall twine || true
+          pipx install twine && twine check dist/*.whl
+      - name: "Test providers issue generation automatically"
+        run: >
+          breeze release-management generate-issue-content-providers
+          --only-available-in-dist --disable-progress
+      - name: "Generate source constraints from CI image"
+        shell: bash
+        run: >
+          breeze release-management generate-constraints
+          --airflow-constraints-mode constraints-source-providers --answer yes
+      - name: "Install and verify all provider packages and airflow via wheel 
files"
+        run: >
+          breeze release-management verify-provider-packages
+          --use-packages-from-dist
+          --package-format wheel
+          --use-airflow-version wheel
+          --airflow-constraints-reference default
+          --providers-constraints-location
+          
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+        env:
+          AIRFLOW_SKIP_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies 
}}"
+      - name: "Prepare airflow package: wheel without suffix and skipping the 
tag check"
+        run: >
+          breeze release-management prepare-provider-packages --skip-tag-check 
--package-format wheel
+
+  prepare-install-provider-packages-sdist:
+    timeout-minutes: 80
+    name: "Provider packages sdist build and install"
+    runs-on: ${{fromJSON(inputs.runs-on)}}
+    env:
+      RUNS_ON: "${{ inputs.runs-on }}"
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      IMAGE_TAG: "${{ inputs.image-tag }}"
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
+      - name: >
+          Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ 
inputs.image-tag }}
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          python-version: ${{ inputs.breeze-python-version }}
+      - name: "Cleanup dist files"
+        run: rm -fv ./dist/*
+      - name: "Prepare provider packages: sdist"
+        run: >
+          breeze release-management prepare-provider-packages 
--include-not-ready-providers
+          --version-suffix-for-pypi dev0 --package-format sdist
+          ${{ inputs.affected-providers-list-as-string }}
+      - name: "Prepare airflow package: sdist"
+        run: >
+          breeze release-management prepare-airflow-package
+          --version-suffix-for-pypi dev0 --package-format sdist
+      - name: "Verify sdist packages with twine"
+        run: |
+          pipx uninstall twine || true
+          pipx install twine && twine check dist/*.tar.gz
+      - name: "Generate source constraints from CI image"
+        shell: bash
+        run: >
+          breeze release-management generate-constraints
+          --airflow-constraints-mode constraints-source-providers --answer yes
+      - name: "Install all provider packages and airflow via sdist files"
+        run: >
+          breeze release-management install-provider-packages
+          --use-packages-from-dist
+          --package-format sdist
+          --use-airflow-version sdist
+          --airflow-constraints-reference default
+          --providers-constraints-location
+          
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+          --run-in-parallel
+        if: inputs.affected-providers-list-as-string == ''
+      - name: "Install affected provider packages and airflow via sdist files"
+        run: >
+          breeze release-management install-provider-packages
+          --use-packages-from-dist
+          --package-format sdist
+          --use-airflow-version sdist
+          --airflow-constraints-reference default
+          --providers-constraints-location
+          
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+          --run-in-parallel
+        if: inputs.affected-providers-list-as-string != ''
+
+  providers-compatibility-checks:
+    timeout-minutes: 80
+    name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} 
provider check
+    runs-on: ${{fromJSON(inputs.runs-on)}}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{fromJson(inputs.providers-compatibility-checks)}}
+    env:
+      RUNS_ON: "${{ inputs.runs-on }}"
+      IMAGE_TAG: "${{ inputs.image-tag }}"
+      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
+      VERSION_SUFFIX_FOR_PYPI: "dev0"
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    if: inputs.skip-provider-tests != 'true'
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Cleanup docker
+        uses: ./.github/actions/cleanup-docker
+      - name: "Prepare breeze & CI image: ${{ matrix.python-version }}:${{ 
inputs.image-tag }}"
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          python-version: ${{ inputs.breeze-python-version }}
+      - name: "Cleanup dist files"
+        run: rm -fv ./dist/*
+      - name: "Prepare provider packages: wheel"
+        run: >
+          breeze release-management prepare-provider-packages 
--include-not-ready-providers
+          --package-format wheel
+      - name: >
+          Remove incompatible Airflow
+          ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }} 
provider packages
+        run: |
+          rm -vf ${{ matrix.remove-providers }}
+        working-directory: ./dist
+        if: matrix.remove-providers != ''
+      - name: "Download airflow package: wheel"
+        run: |
+          pip download "apache-airflow==${{ matrix.airflow-version }}" -d dist 
--no-deps
+      - name: >
+          Install and verify all provider packages and airflow on
+          Airflow ${{ matrix.airflow-version }}:Python ${{ 
matrix.python-version }}
+        run: >
+          breeze release-management verify-provider-packages
+          --use-packages-from-dist
+          --package-format wheel
+          --use-airflow-version wheel
+          --airflow-constraints-reference 
constraints-${{matrix.airflow-version}}
+          --providers-skip-constraints
+          --install-airflow-with-constraints
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 64ecdb93a1..26e66bb070 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -818,198 +818,27 @@ jobs:
         run: >
           breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string 
}} --spellcheck-only
 
-  prepare-test-provider-packages-wheel:
-    timeout-minutes: 80
-    name: "Provider packages wheel (verify)"
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+  providers:
+    name: "Provider checks"
+    uses: ./.github/workflows/check-providers.yml
     needs: [build-info, wait-for-ci-images]
-    env:
-      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
-      PYTHON_MAJOR_MINOR_VERSION: 
"${{needs.build-info.outputs.default-python-version}}"
+    permissions:
+      contents: read
+      packages: read
+    secrets: inherit
     if: >
-      needs.build-info.outputs.skip-provider-tests != 'true' &&
+      needs.build-info.outputs.skip-providers-tests != 'true' &&
       needs.build-info.outputs.latest-versions-only != 'true'
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: Cleanup docker
-        uses: ./.github/actions/cleanup-docker
-      - name: >
-          Prepare breeze & CI image: 
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
-        uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          python-version: ${{ needs.build-info.outputs.breeze-python-version }}
-      - name: "Cleanup dist files"
-        run: rm -fv ./dist/*
-      - name: "Prepare provider documentation"
-        run: >
-          breeze release-management prepare-provider-documentation 
--include-not-ready-providers
-          --non-interactive
-      - name: "Prepare provider packages: wheel"
-        run: >
-          breeze release-management prepare-provider-packages 
--include-not-ready-providers
-          --version-suffix-for-pypi dev0 --package-format wheel
-      - name: "Prepare airflow package: wheel"
-        run: breeze release-management prepare-airflow-package 
--version-suffix-for-pypi dev0
-      - name: "Verify wheel packages with twine"
-        run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.whl
-      - name: "Test providers issue generation automatically"
-        run: >
-          breeze release-management generate-issue-content-providers
-          --only-available-in-dist --disable-progress
-      - name: "Generate source constraints from CI image"
-        shell: bash
-        run: >
-          breeze release-management generate-constraints
-          --airflow-constraints-mode constraints-source-providers
-      - name: "Install and verify all provider packages and airflow via wheel 
files"
-        run: >
-          breeze release-management verify-provider-packages
-          --use-packages-from-dist
-          --package-format wheel
-          --use-airflow-version wheel
-          --airflow-constraints-reference default
-          --providers-constraints-location
-          
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
-        env:
-          AIRFLOW_SKIP_CONSTRAINTS: "${{ 
needs.build-info.outputs.upgrade-to-newer-dependencies }}"
-      - name: "Prepare airflow package: wheel without suffix and skipping the 
tag check"
-        run: >
-          breeze release-management prepare-provider-packages --skip-tag-check 
--package-format wheel
-
-  prepare-install-provider-packages-sdist:
-    timeout-minutes: 80
-    name: "Provider packages sdist (install)"
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
-    needs: [build-info, wait-for-ci-images]
-    env:
-      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
-      PYTHON_MAJOR_MINOR_VERSION: 
"${{needs.build-info.outputs.default-python-version}}"
-    if: >
-      needs.build-info.outputs.canary-run == 'true' &&
-      needs.build-info.outputs.skip-provider-tests != 'true'
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: Cleanup docker
-        uses: ./.github/actions/cleanup-docker
-      - name: >
-          Prepare breeze & CI image: 
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
-        uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          python-version: ${{ needs.build-info.outputs.breeze-python-version }}
-      - name: "Cleanup dist files"
-        run: rm -fv ./dist/*
-      - name: "Prepare provider packages: sdist"
-        run: >
-          breeze release-management prepare-provider-packages 
--include-not-ready-providers
-          --version-suffix-for-pypi dev0 --package-format sdist
-          ${{ needs.build-info.outputs.affected-providers-list-as-string }}
-      - name: "Prepare airflow package: sdist"
-        run: >
-          breeze release-management prepare-airflow-package
-          --version-suffix-for-pypi dev0 --package-format sdist
-      - name: "Verify sdist packages with twine"
-        run: |
-          pipx uninstall twine || true
-          pipx install twine && twine check dist/*.tar.gz
-      - name: "Generate source constraints from CI image"
-        shell: bash
-        run: >
-          breeze release-management generate-constraints
-          --airflow-constraints-mode constraints-source-providers
-      - name: "Install all provider packages and airflow via sdist files"
-        run: >
-          breeze release-management install-provider-packages
-          --use-packages-from-dist
-          --package-format sdist
-          --use-airflow-version sdist
-          --airflow-constraints-reference default
-          --providers-constraints-location
-          
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
-          --run-in-parallel
-        if: needs.build-info.outputs.affected-providers-list-as-string == ''
-      - name: "Install affected provider packages and airflow via sdist files"
-        run: >
-          breeze release-management install-provider-packages
-          --use-packages-from-dist
-          --package-format sdist
-          --use-airflow-version sdist
-          --airflow-constraints-reference default
-          --providers-constraints-location
-          
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
-          --run-in-parallel
-        if: needs.build-info.outputs.affected-providers-list-as-string != ''
-
-  providers-compatibility-checks:
-    timeout-minutes: 80
-    name: >
-      Compat ${{matrix.airflow-version}}:P${{matrix.python-version}} provider 
check
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
-    needs: [build-info, wait-for-ci-images]
-    strategy:
-      fail-fast: false
-      matrix:
-        include: 
${{fromJson(needs.build-info.outputs.providers-compatibility-checks)}}
-    env:
-      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
-      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
-      VERSION_SUFFIX_FOR_PYPI: "dev0"
-    if: needs.build-info.outputs.skip-provider-tests != 'true'
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm 
-rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: Cleanup docker
-        uses: ./.github/actions/cleanup-docker
-      - name: >
-          Prepare breeze & CI image: 
${{matrix.python-version}}:${{env.IMAGE_TAG}}
-        uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          python-version: ${{ needs.build-info.outputs.breeze-python-version }}
-      - name: "Cleanup dist files"
-        run: rm -fv ./dist/*
-      - name: "Prepare provider packages: wheel"
-        run: >
-          breeze release-management prepare-provider-packages 
--include-not-ready-providers
-          --package-format wheel
-      - name: >
-          Remove incompatible Airflow
-          ${{matrix.airflow-version}}:Python ${{matrix.python-version}} 
provider packages
-        run: |
-          rm -vf ${{ matrix.remove-providers }}
-        working-directory: ./dist
-        if: matrix.remove-providers != ''
-      - name: "Download airflow package: wheel"
-        run: |
-          pip download "apache-airflow==${{matrix.airflow-version}}" -d dist 
--no-deps
-      - name: >
-          Install and verify all provider packages and airflow on
-          Airflow ${{matrix.airflow-version}}:Python ${{matrix.python-version}}
-        run: >
-          breeze release-management verify-provider-packages
-          --use-packages-from-dist
-          --package-format wheel
-          --use-airflow-version wheel
-          --airflow-constraints-reference 
constraints-${{matrix.airflow-version}}
-          --providers-skip-constraints
-          --install-airflow-with-constraints
+    with:
+      runs-on: ${{ needs.build-info.outputs.runs-on }}
+      image-tag: ${{ needs.build-info.outputs.image-tag }}
+      breeze-python-version: ${{ 
needs.build-info.outputs.breeze-python-version }}
+      default-python-version: ${{ 
needs.build-info.outputs.default-python-version }}
+      upgrade-to-newer-dependencies: ${{ 
needs.build-info.outputs.upgrade-to-newer-dependencies }}
+      affected-providers-list-as-string: ${{ 
needs.build-info.outputs.affected-providers-list-as-string }}
+      providers-compatibility-checks: ${{ 
needs.build-info.outputs.providers-compatibility-checks }}
+      skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
 
   test-airflow-release-commands:
     timeout-minutes: 80

Reply via email to