This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 63bc63c190 Better diangostics for parallel test execution in CI
(#38200)
63bc63c190 is described below
commit 63bc63c190a4648ee71301759dd1e959f2095f70
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 16 09:25:48 2024 +0100
Better diangostics for parallel test execution in CI (#38200)
When running tests in parallel in our CI we switched off showing
success outputs by default in #38157. This is generally a good
thing, because it only shows outputs for the test types that failed
and makes the CI log output far snappier in case only one or few
test types failed.
However it also has a side effect that it's
not really easy to see what kind of commands were run in each
parallel test type, so we might get accidentally some mistakes
where we miss that our tests are not doing what we think they are
doing - for example not running tests for various Python versions,
only for one, or not running them for various database versions
(both cases happened in the past). By hiding the success outputs,
we make it harder to spot such issues.
This PR introduces two ways to improve it:
* we can now set a label on PR to `include success outputs` and
any committer can set the label to show success outputs as well.
* When we run parallel unit tests we summarize what is going
to be executed - showing all the ShellParams passed to the
parallel test execution. This is the best place to show it
because this ShellParams object contains all the actual values
passed - after processing throught options, special environment
variables and so-on - so those are the actual values used to
run the parallel tests.
Also it will be printed in a prominent place - just before test
execution in the main console output, which means it will be
visible without the need to unfold any output.
---
.github/workflows/ci.yml | 12 +++++++++++-
.github/workflows/run-unit-tests.yml | 6 ++++++
.github/workflows/special-tests.yml | 8 ++++++++
dev/breeze/doc/ci/04_selective_checks.md | 8 +++++++-
.../src/airflow_breeze/commands/testing_commands.py | 17 ++++++++++++++---
dev/breeze/src/airflow_breeze/utils/selective_checks.py | 7 +++++++
6 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 394ddf1553..620b0bfac4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,7 +48,6 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
USE_SUDO: "true"
- INCLUDE_SUCCESS_OUTPUTS: "true"
INCLUDE_NOT_READY_PROVIDERS: "true"
AIRFLOW_ENABLE_AIP_44: "true"
MOUNT_SOURCES: "skip"
@@ -98,6 +97,7 @@ jobs:
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed
}}
parallel-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.parallel-test-types-list-as-string
}}
+ include-success-outputs: ${{
steps.selective-checks.outputs.include-success-outputs }}
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
@@ -421,6 +421,7 @@ jobs:
BACKEND: sqlite
# Force more parallelism for pull even on public images
PARALLELISM: 6
+ INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
steps:
- name: "Cleanup repo"
shell: bash
@@ -459,6 +460,7 @@ jobs:
PYTHON_VERSIONS:
${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
VERSION_SUFFIX_FOR_PYPI: "dev0"
+ INCLUDE_SUCCESS_OUTPUTS: "true"
if: needs.build-info.outputs.ci-image-build == 'true'
steps:
- name: "Cleanup repo"
@@ -724,6 +726,7 @@ jobs:
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
steps:
- name: "Cleanup repo"
shell: bash
@@ -914,6 +917,7 @@ jobs:
backend-versions: ${{ needs.build-info.outputs.postgres-versions }}
excludes: ${{ needs.build-info.outputs.postgres-exclude }}
parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-migration-tests: "true"
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
@@ -938,6 +942,7 @@ jobs:
backend-versions: ${{ needs.build-info.outputs.mysql-versions }}
excludes: ${{ needs.build-info.outputs.mysql-exclude }}
parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
@@ -964,6 +969,7 @@ jobs:
backend-versions: "['']"
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
@@ -990,6 +996,7 @@ jobs:
backend-versions: "['']"
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
@@ -1246,6 +1253,7 @@ jobs:
PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
# Force more parallelism for pull on public images
PARALLELISM: 6
+ INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
steps:
- name: "Cleanup repo"
shell: bash
@@ -1323,6 +1331,7 @@ jobs:
if: needs.build-info.outputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
steps:
- name: "Cleanup repo"
shell: bash
@@ -1365,6 +1374,7 @@ jobs:
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+ INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.needs-helm-tests == 'true')
diff --git a/.github/workflows/run-unit-tests.yml
b/.github/workflows/run-unit-tests.yml
index ba95cfef17..c996a6338c 100644
--- a/.github/workflows/run-unit-tests.yml
+++ b/.github/workflows/run-unit-tests.yml
@@ -80,6 +80,11 @@ on: # yamllint disable-line rule:truthy
Which version of python should be used to install Breeze (3.9 is
minimum for reproducible builds)
required: true
type: string
+ include-success-outputs:
+ description: "Whether to include success outputs or not (true/false)"
+ required: false
+ default: "false"
+ type: string
downgrade-sqlalchemy:
description: "Whether to downgrade SQLAlchemy or not (true/false)"
required: false
@@ -129,6 +134,7 @@ jobs:
DOWN_PENDULUM: "${{ inputs.downgrade-pendulum }}"
ENABLE_COVERAGE: "${{ inputs.run-coverage }}"
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+ INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }}
# yamllint disable rule:line-length
JOB_ID: "${{ inputs.test-scope }}-${{ inputs.test-name
}}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version
}}"
MOUNT_SOURCES: "skip"
diff --git a/.github/workflows/special-tests.yml
b/.github/workflows/special-tests.yml
index 4550191ae7..1bfbdd6c39 100644
--- a/.github/workflows/special-tests.yml
+++ b/.github/workflows/special-tests.yml
@@ -73,6 +73,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -95,6 +96,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -117,6 +119,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -139,6 +142,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -161,6 +165,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -183,6 +188,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -204,6 +210,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
@@ -225,6 +232,7 @@ jobs:
backend-versions: "['${{ inputs.default-postgres-version }}']"
excludes: "[]"
parallel-test-types-list-as-string: ${{
inputs.parallel-test-types-list-as-string }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ inputs.run-coverage }}
debug-resources: ${{ inputs.debug-resources }}
breeze-python-version: ${{ inputs.breeze-python-version }}
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index e53d7c0448..061cb1644b 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -275,6 +275,11 @@ used in the matrix to the latest ones (latest Python
version and latest Kubernet
You can also disable cache if you want to make sure your tests will run with
image that does not have
left-over package installed from the past cached image - by setting `disable
image cache` label in the PR.
+By default all outputs of successful parallel tests are not shown. You can
enable them by setting
+`include success outputs` label in the PR. This makes the logs of mostly
successful tests a lot longer
+and more difficult to sift through, but it might be useful in case you want to
compare successful and
+unsuccessful runs of the tests.
+
## PR labels
As mentioned below, you can influence the outputs of selected checks by
setting labels to the PR. Here is
@@ -285,7 +290,8 @@ am overview of possible labels and their meaning:
| canary | is-canary-run |
If set, the PR run from apache/airflow repo behaves as `canary` run (can only
be run by maintainer). |
| debug ci resources | debug-ci-resources |
If set, then debugging resources is enabled during parallel tests and you can
see them in the output. |
| default versions only | python-versions-*, kubernetes-versions-* |
If set, the number of Python and Kubernetes versions used by the build will be
limited to the default ones. |
-| disable image cache | | If set, the number of Python and
Kubernetes versions used by the build will be limited to the latest ones. |
+| disable image cache | cache-directive |
If set, the image cache is disables when building the image.
|
+| include success outputs | include-success-outputs |
By default, outputs of successful parallel tests are not shown - enabling this
flag will make then shown. |
| latest versions only | python-versions-*, kubernetes-versions-* |
If set, the number of Python and Kubernetes versions used by the build will be
limited to the latest ones. |
| full tests needed | full-tests-needed |
Run complete set of tests, including all Python all DB versions, and all test
types. |
| non committer build | is-committer-build |
If set then even for non-committer builds, the scripts used for images are used
from target branch. |
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index cb8bd5723e..e0bf83cd63 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -336,8 +336,19 @@ def run_tests_in_parallel(
debug_resources: bool,
parallelism: int,
skip_cleanup: bool,
- skio_docker_compose_down: bool,
+ skip_docker_compose_down: bool,
) -> None:
+ get_console().print("\n[info]Summary of the tests to run\n")
+ get_console().print(f"[info]Running tests in parallel with
parallelism={parallelism}")
+ get_console().print(f"[info]Extra pytest args: {extra_pytest_args}")
+ get_console().print(f"[info]DB reset: {db_reset}")
+ get_console().print(f"[info]Test timeout: {test_timeout}")
+ get_console().print(f"[info]Include success outputs:
{include_success_outputs}")
+ get_console().print(f"[info]Debug resources: {debug_resources}")
+ get_console().print(f"[info]Skip cleanup: {skip_cleanup}")
+ get_console().print(f"[info]Skip docker-compose down:
{skip_docker_compose_down}")
+ get_console().print("[info]Shell params:")
+ get_console().print(shell_params.__dict__)
_run_tests_in_pool(
tests_to_run=shell_params.parallel_test_types_list,
parallelism=parallelism,
@@ -348,7 +359,7 @@ def run_tests_in_parallel(
include_success_outputs=include_success_outputs,
debug_resources=debug_resources,
skip_cleanup=skip_cleanup,
- skip_docker_compose_down=skio_docker_compose_down,
+ skip_docker_compose_down=skip_docker_compose_down,
)
@@ -684,7 +695,7 @@ def _run_test_command(
parallelism=parallelism,
skip_cleanup=skip_cleanup,
debug_resources=debug_resources,
- skio_docker_compose_down=skip_docker_compose_down,
+ skip_docker_compose_down=skip_docker_compose_down,
)
else:
if shell_params.test_type == "Default":
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index eebee874bd..6214a7e3b3 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -70,6 +70,7 @@ NON_COMMITTER_BUILD_LABEL = "non committer build"
DEFAULT_VERSIONS_ONLY_LABEL = "default versions only"
LATEST_VERSIONS_ONLY_LABEL = "latest versions only"
DISABLE_IMAGE_CACHE_LABEL = "disable image cache"
+INCLUDE_SUCCESS_OUTPUTS_LABEL = "include success outputs"
UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies"
@@ -833,6 +834,12 @@ class SelectiveChecks:
self._extract_long_provider_tests(current_test_types)
return " ".join(sorted(current_test_types))
+ @cached_property
+ def include_success_outputs(
+ self,
+ ) -> bool:
+ return INCLUDE_SUCCESS_OUTPUTS_LABEL in self._pr_labels
+
@cached_property
def basic_checks_only(self) -> bool:
return not self.ci_image_build
