This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4af69f9777 Extract integration tests to a spearate workflow (#38208)
4af69f9777 is described below
commit 4af69f9777264e98af6a602d3edf6d5dda031a25
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Mar 16 17:09:12 2024 +0100
Extract integration tests to a spearate workflow (#38208)
Another step of ci workflow refactoring. This one extracts the
integration tests to a separate workflow, and turn the
integration tests there into a matrix strategy single job with
a bit more conditionals.
---
.github/workflows/ci.yml | 145 +++++-------------------------
.github/workflows/integration-tests.yml | 155 ++++++++++++++++++++++++++++++++
2 files changed, 176 insertions(+), 124 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7dea803dd0..e65f3a94db 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -759,126 +759,26 @@ jobs:
breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
if: needs.build-info.outputs.run-tests == 'true'
- tests-integration-postgres:
- timeout-minutes: 130
- name: Integration Tests Postgres
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-ci-images]
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
- FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
- DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
- BACKEND: "postgres"
- ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
- JOB_ID: "integration-postgres"
- SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: "Prepare breeze & CI image:
${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
- uses: ./.github/actions/prepare_breeze_and_image
- - name: "Integration Tests Postgres: cassandra"
- run: |
- breeze testing integration-tests --integration cassandra
- breeze down
- if: needs.build-info.outputs.is-airflow-runner != 'true'
- - name: "Integration Tests Postgres: mongo"
- run: |
- breeze testing integration-tests --integration mongo
- breeze down
- if: needs.build-info.outputs.is-airflow-runner != 'true'
- - name: "Integration Tests Postgres: pinot"
- run: |
- breeze testing integration-tests --integration pinot
- breeze down
- if: needs.build-info.outputs.is-airflow-runner != 'true'
- - name: "Integration Tests Postgres: celery"
- run: |
- breeze testing integration-tests --integration celery
- breeze down
- if: needs.build-info.outputs.is-airflow-runner != 'true'
- - name: "Integration Tests Postgres: trino, kerberos"
- run: |
- breeze testing integration-tests --integration trino --integration
kerberos
- breeze down
- if: needs.build-info.outputs.is-airflow-runner != 'true'
- - name: "Integration Tests Postgres: Kafka"
- run: |
- breeze testing integration-tests --integration kafka
- breeze down
- if: needs.build-info.outputs.is-airflow-runner != 'true'
- - name: "Integration Tests Postgres: Qdrant"
- run: breeze testing integration-tests --integration qdrant
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: "Integration Tests Postgres: all-testable"
- run: breeze testing integration-tests --integration all-testable
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: >
- Post Tests success: Integration Postgres"
- uses: ./.github/actions/post_tests_success
- if: success()
- - name: >
- Post Tests failure: Integration Postgres"
- uses: ./.github/actions/post_tests_failure
- if: failure()
-
- tests-integration-mysql:
- timeout-minutes: 130
- name: Integration Tests MySQL
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+ tests-integration:
+ name: Integration Tests
needs: [build-info, wait-for-ci-images]
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
- FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
- DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
- BACKEND: "mysql"
- ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
- JOB_ID: "integration-mysql"
- SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
+ uses: ./.github/workflows/integration-tests.yml
+ permissions:
+ contents: read
+ packages: read
+ secrets: inherit
+ with:
+ runs-on: ${{ needs.build-info.outputs.runs-on }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
+ default-python-version: ${{
needs.build-info.outputs.default-python-version }}
+ default-postgres-version: ${{
needs.build-info.outputs.default-postgres-version }}
+ default-mysql-version: ${{
needs.build-info.outputs.default-mysql-version }}
+ skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
+ is-airflow-runner: ${{ needs.build-info.outputs.is-airflow-runner }}
+ run-coverage: ${{ needs.build-info.outputs.run-coverage }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: "Prepare breeze & CI image:
${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
- uses: ./.github/actions/prepare_breeze_and_image
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: "Integration Tests MySQL: all-testable"
- run: breeze testing integration-tests --integration all-testable
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: >
- Post Tests success: Integration MySQL"
- uses: ./.github/actions/post_tests_success
- if: needs.build-info.outputs.is-airflow-runner == 'true'
- - name: >
- Post Tests failure: Integration MySQL"
- uses: ./.github/actions/post_tests_failure
- if: failure()
summarize-warnings:
timeout-minutes: 15
@@ -891,8 +791,7 @@ jobs:
- tests-mysql
- tests-non-db
- tests-special
- - tests-integration-postgres
- - tests-integration-mysql
+ - tests-integration
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
steps:
@@ -1049,8 +948,7 @@ jobs:
- tests-postgres
- tests-non-db
- tests-special
- - tests-integration-postgres
- - tests-integration-mysql
+ - tests-integration
- generate-constraints
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
@@ -1139,8 +1037,7 @@ jobs:
- tests-mysql
- tests-postgres
- tests-non-db
- - tests-integration-postgres
- - tests-integration-mysql
+ - tests-integration
uses: ./.github/workflows/ci-image-build.yml
permissions:
contents: read
diff --git a/.github/workflows/integration-tests.yml
b/.github/workflows/integration-tests.yml
new file mode 100644
index 0000000000..5d6c92bbbf
--- /dev/null
+++ b/.github/workflows/integration-tests.yml
@@ -0,0 +1,155 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Integration tests
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ runs-on:
+ description: "The array of labels (in json form) determining type of
the runner to use for the build."
+ required: false
+ default: '["ubuntu-22.04"]'
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
+ required: true
+ type: string
+ parallel-test-types-list-as-string:
+ description: "The list of parallel test types to run separated by
spaces"
+ required: true
+ type: string
+ default-postgres-version:
+ description: "Default version of Postgres to use"
+ required: true
+ type: string
+ default-mysql-version:
+ description: "Default version of MySQL to use"
+ required: true
+ type: string
+ skip-provider-tests:
+ description: "Skip provider tests (true/false)"
+ required: true
+ type: string
+ is-airflow-runner:
+ description: "Is this an Airflow self-hosted runner (true/false)"
+ required: true
+ type: string
+ run-coverage:
+ description: "Run coverage (true/false)"
+ required: true
+ type: string
+ default-python-version:
+ description: Which version of python should be used get CI image
+ required: true
+ type: string
+ debug-resources:
+ description: "Debug resources (true/false)"
+ required: true
+ type: string
+jobs:
+ tests-integration:
+ timeout-minutes: 130
+ name: Integration Tests ${{ matrix.backend }}:${{ matrix.backend-version}}
+ runs-on: ${{fromJSON(inputs.runs-on)}}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - backend: postgres
+ backend-version: ${{ inputs.default-postgres-version }}
+ - backend: mysql
+ backend-version: ${{ inputs.default-mysql-version }}
+ env:
+ RUNS_ON: "${{ inputs.runs-on }}"
+ IMAGE_TAG: "${{ inputs.image-tag }}"
+ PARALLEL_TEST_TYPES: "${{ inputs.parallel-test-types-list-as-string }}"
+ BACKEND: "${{ matrix.backend }}"
+ BACKEND_VERSION: "${{ matrix.backend-version}}"
+ PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ JOB_ID: "integration-${{ matrix.backend }}-${{ matrix.backend-version }}"
+ SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}"
+ ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
+ DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
+ - name: "Prepare breeze & CI image: ${{ inputs.default-python-version
}}:${{ inputs.image-tag }}"
+ uses: ./.github/actions/prepare_breeze_and_image
+ if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
+ # For public runners only run Integration tests one-by-one to avoid too
much resource use
+ # and skip MySQL tests for Public Runners altogether as they are too
flaky with mysql resource use
+ - name: "Integration Tests Postgres: cassandra"
+ run: |
+ breeze testing integration-tests --integration cassandra
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ - name: "Integration Tests Postgres: mongo"
+ run: |
+ breeze testing integration-tests --integration mongo
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ - name: "Integration Tests Postgres: pinot"
+ run: |
+ breeze testing integration-tests --integration pinot
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ - name: "Integration Tests Postgres: celery"
+ run: |
+ breeze testing integration-tests --integration celery
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ - name: "Integration Tests Postgres: trino, kerberos"
+ run: |
+ breeze testing integration-tests --integration trino --integration
kerberos
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ - name: "Integration Tests Postgres: Kafka"
+ run: |
+ breeze testing integration-tests --integration kafka
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ - name: "Integration Tests Postgres: Qdrant"
+ run: |
+ breeze testing integration-tests --integration qdrant
+ breeze down
+ if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
+ # For Self Hosted runners we run complete set of integration tests
+ # in one big test - as we have enough resources to start all integrations
+ # and we do that for both - postgres and mysql
+ - name: "Integration Tests Postgres: all-testable"
+ run: |
+ breeze testing integration-tests --integration all-testable
+ breeze down
+ if: inputs.is-airflow-runner == 'true'
+ - name: "Post Tests success: Integration"
+ uses: ./.github/actions/post_tests_success
+ if: success() && inputs.is-airflow-runner == 'true' || matrix.backend
== 'postgres'
+ - name: "Post Tests failure: Integration"
+ uses: ./.github/actions/post_tests_failure
+ if: failure() && inputs.is-airflow-runner == 'true' || matrix.backend
== 'postgres'
