This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7cefd0acac Simplify Integration test execution in CI (#39866)
7cefd0acac is described below
commit 7cefd0acacce9836e71046bf8178ec41f086095f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon May 27 13:01:52 2024 +0200
Simplify Integration test execution in CI (#39866)
When we run all integration tests in CI together on self-hosted
runners they are somewhat flaky, however when those tests are run
individually on public runners, they are both fast and stable. Also
the integration test list has been somewhat duplicated - you had to
remember about adding individual integration tests to the separate
integration-tests workflow when you added such integration tests.
This PR solves all those problems:
* we always run integration tests individually
* they are all run on public runners no matter what build they are
on
* they are run as matrix of jobs and list of tests is extracted
via selective checks outputs - no more duplication
---
.github/workflows/ci.yml | 6 +-
.github/workflows/integration-tests.yml | 89 ++++------------------
dev/breeze/doc/ci/04_selective_checks.md | 1 +
.../src/airflow_breeze/utils/selective_checks.py | 5 ++
scripts/ci/docker-compose/integration-mssql.yml | 2 +-
5 files changed, 26 insertions(+), 77 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0502df3368..f11487bdd0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -120,6 +120,7 @@ jobs:
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
build-job-description: ${{
steps.source-run-info.outputs.build-job-description }}
+ testable-integrations: ${{
steps.selective-checks.outputs.testable-integrations }}
canary-run: ${{ steps.source-run-info.outputs.canary-run }}
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
steps:
@@ -470,14 +471,13 @@ jobs:
packages: read
secrets: inherit
with:
- runs-on-as-json-default: ${{
needs.build-info.outputs.runs-on-as-json-default }}
+ runs-on-as-json-public: ${{
needs.build-info.outputs.runs-on-as-json-public }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
+ testable-integrations: ${{
needs.build-info.outputs.testable-integrations }}
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
default-postgres-version: ${{
needs.build-info.outputs.default-postgres-version }}
default-mysql-version: ${{
needs.build-info.outputs.default-mysql-version }}
skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
- is-airflow-runner: ${{ needs.build-info.outputs.is-airflow-runner }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.run-tests == 'true'
diff --git a/.github/workflows/integration-tests.yml
b/.github/workflows/integration-tests.yml
index bc7d58d099..8dd4cae6a2 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -20,16 +20,16 @@ name: Integration tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-default:
- description: "The array of labels (in json form) determining default
runner used for the build."
+ runs-on-as-json-public:
+ description: "The array of labels (in json form) determining public
runners."
required: true
type: string
image-tag:
description: "Tag to set for the image"
required: true
type: string
- parallel-test-types-list-as-string:
- description: "The list of parallel test types to run separated by
spaces"
+ testable-integrations:
+ description: "The list of testable integrations as JSON array."
required: true
type: string
default-postgres-version:
@@ -44,10 +44,6 @@ on: # yamllint disable-line rule:truthy
description: "Skip provider tests (true/false)"
required: true
type: string
- is-airflow-runner:
- description: "Is this an Airflow self-hosted runner (true/false)"
- required: true
- type: string
run-coverage:
description: "Run coverage (true/false)"
required: true
@@ -63,23 +59,20 @@ on: # yamllint disable-line rule:truthy
jobs:
tests-integration:
timeout-minutes: 130
- name: Integration Tests ${{ matrix.backend }}:${{ matrix.backend-version}}
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ name: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
strategy:
fail-fast: false
matrix:
- include:
- - backend: postgres
- backend-version: ${{ inputs.default-postgres-version }}
- - backend: mysql
- backend-version: ${{ inputs.default-mysql-version }}
+ backend: ["postgres", "mysql"]
+ integration: ${{ fromJSON(inputs.testable-integrations) }}
env:
IMAGE_TAG: "${{ inputs.image-tag }}"
- PARALLEL_TEST_TYPES: "${{ inputs.parallel-test-types-list-as-string }}"
BACKEND: "${{ matrix.backend }}"
- BACKEND_VERSION: "${{ matrix.backend-version}}"
+ # yamllint disable-line rule:line-length
+ BACKEND_VERSION: ${{ matrix.backend == 'postgres' &&
inputs.default-postgres-version || inputs.default-mysql-version }}"
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
- JOB_ID: "integration-${{ matrix.backend }}-${{ matrix.backend-version }}"
+ JOB_ID: "integration-${{ matrix.backend }}-${{ matrix.integration }}"
SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}"
ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
@@ -91,71 +84,21 @@ jobs:
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
- if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
- name: "Prepare breeze & CI image: ${{ inputs.default-python-version
}}:${{ inputs.image-tag }}"
uses: ./.github/actions/prepare_breeze_and_image
- if: inputs.is-airflow-runner == 'true' || matrix.backend == 'postgres'
- # For public runners only run Integration tests one-by-one to avoid too
much resource use
- # and skip MySQL tests for Public Runners altogether as they are too
flaky with mysql resource use
- - name: "Integration Tests Postgres: cassandra"
+ - name: "Integration Tests ${{ matrix.backend }}:${{ matrix.integration
}}"
run: |
- breeze testing integration-tests --integration cassandra
breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: mongo"
- run: |
- breeze testing integration-tests --integration mongo
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: pinot"
- run: |
- breeze testing integration-tests --integration pinot
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: celery"
- run: |
- breeze testing integration-tests --integration celery
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: trino, kerberos"
- run: |
- breeze testing integration-tests --integration trino --integration
kerberos
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: Kafka"
- run: |
- breeze testing integration-tests --integration kafka
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: Qdrant"
- run: |
- breeze testing integration-tests --integration qdrant
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- - name: "Integration Tests Postgres: mssql"
- run: |
- breeze testing integration-tests --integration mssql
- breeze down
- if: inputs.is-airflow-runner != 'true' && matrix.backend == 'postgres'
- # For Self Hosted runners we run complete set of integration tests
- # in one big test - as we have enough resources to start all integrations
- # and we do that for both - postgres and mysql
- - name: "Integration Tests Postgres: all-testable"
- run: |
- breeze testing integration-tests --integration all-testable
+ breeze testing integration-tests --integration ${{
matrix.integration }}
breeze down
- if: inputs.is-airflow-runner == 'true'
- - name: "Post Tests success: Integration"
+ - name: "Post Tests success: Integration Tests ${{ matrix.backend }}-${{
matrix.integration }}"
uses: ./.github/actions/post_tests_success
- if: success() && (inputs.is-airflow-runner == 'true' || matrix.backend
== 'postgres')
- - name: "Post Tests failure: Integration"
+ - name: "Post Tests failure: Integration Tests ${{ matrix.backend }}-${{
matrix.integration }}"
uses: ./.github/actions/post_tests_failure
- if: failure() && (inputs.is-airflow-runner == 'true' || matrix.backend
== 'postgres')
+ if: failure()
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index e6c59eee8c..819633d4c5 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -229,6 +229,7 @@ Github Actions to pass the list of parameters to a command
to execute
| skip-pre-commits | Which pre-commits should be skipped
during the static-checks run |
check-provider-yaml-valid,flynt,identity | |
| skip-provider-tests | When provider tests should be
skipped (on non-main branch or when no provider changes detected) | true
| |
| sqlite-exclude | Which versions of Sqlite to exclude
for tests as JSON array | []
| |
+| testable-integrations | List of integrations that are
testable in the build as JSON array |
['mongo', 'kafka', 'mssql'] | |
| upgrade-to-newer-dependencies | Whether the image build should
attempt to upgrade all dependencies (true/false or commit hash) | false
| |
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 193271f34c..893e0d2f99 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -45,6 +45,7 @@ from airflow_breeze.global_constants import (
KIND_VERSION,
RUNS_ON_PUBLIC_RUNNER,
RUNS_ON_SELF_HOSTED_RUNNER,
+ TESTABLE_INTEGRATIONS,
GithubEvents,
SelectiveUnitTestTypes,
all_helm_test_packages,
@@ -1205,6 +1206,10 @@ class SelectiveChecks:
]
)
+ @cached_property
+ def testable_integrations(self) -> list[str]:
+ return TESTABLE_INTEGRATIONS
+
@cached_property
def is_committer_build(self):
if NON_COMMITTER_BUILD_LABEL in self._pr_labels:
diff --git a/scripts/ci/docker-compose/integration-mssql.yml
b/scripts/ci/docker-compose/integration-mssql.yml
index 4b9f9bf4da..ffe171e050 100644
--- a/scripts/ci/docker-compose/integration-mssql.yml
+++ b/scripts/ci/docker-compose/integration-mssql.yml
@@ -35,7 +35,7 @@ services:
test: /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U sa -P
"airflow123" -Q "SELECT 1" -b -o /dev/null
interval: 10s
timeout: 3s
- retries: 10
+ retries: 30
start_period: 10s
airflow:
depends_on:
