This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 63bc63c190 Better diangostics for parallel test execution in CI (#38200) 63bc63c190 is described below commit 63bc63c190a4648ee71301759dd1e959f2095f70 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sat Mar 16 09:25:48 2024 +0100 Better diangostics for parallel test execution in CI (#38200) When running tests in parallel in our CI we switched off showing success outputs by default in #38157. This is generally a good thing, because it only shows outputs for the test types that failed and makes the CI log output far snappier in case only one or few test types failed. However it also has a side effect that it's not really easy to see what kind of commands were run in each parallel test type, so we might get accidentally some mistakes where we miss that our tests are not doing what we think they are doing - for example not running tests for various Python versions, only for one, or not running them for various database versions (both cases happened in the past). By hiding the success outputs, we make it harder to spot such issues. This PR introduces two ways to improve it: * we can now set a label on PR to `include success outputs` and any committer can set the label to show success outputs as well. * When we run parallel unit tests we summarize what is going to be executed - showing all the ShellParams passed to the parallel test execution. This is the best place to show it because this ShellParams object contains all the actual values passed - after processing throught options, special environment variables and so-on - so those are the actual values used to run the parallel tests. Also it will be printed in a prominent place - just before test execution in the main console output, which means it will be visible without the need to unfold any output. --- .github/workflows/ci.yml | 12 +++++++++++- .github/workflows/run-unit-tests.yml | 6 ++++++ .github/workflows/special-tests.yml | 8 ++++++++ dev/breeze/doc/ci/04_selective_checks.md | 8 +++++++- .../src/airflow_breeze/commands/testing_commands.py | 17 ++++++++++++++--- dev/breeze/src/airflow_breeze/utils/selective_checks.py | 7 +++++++ 6 files changed, 53 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 394ddf1553..620b0bfac4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,6 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" USE_SUDO: "true" - INCLUDE_SUCCESS_OUTPUTS: "true" INCLUDE_NOT_READY_PROVIDERS: "true" AIRFLOW_ENABLE_AIP_44: "true" MOUNT_SOURCES: "skip" @@ -98,6 +97,7 @@ jobs: full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} parallel-test-types-list-as-string: >- ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} @@ -421,6 +421,7 @@ jobs: BACKEND: sqlite # Force more parallelism for pull even on public images PARALLELISM: 6 + INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" steps: - name: "Cleanup repo" shell: bash @@ -459,6 +460,7 @@ jobs: PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} VERSION_SUFFIX_FOR_PYPI: "dev0" + INCLUDE_SUCCESS_OUTPUTS: "true" if: needs.build-info.outputs.ci-image-build == 'true' steps: - name: "Cleanup repo" @@ -724,6 +726,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" steps: - name: "Cleanup repo" shell: bash @@ -914,6 +917,7 @@ jobs: backend-versions: ${{ needs.build-info.outputs.postgres-versions }} excludes: ${{ needs.build-info.outputs.postgres-exclude }} parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -938,6 +942,7 @@ jobs: backend-versions: ${{ needs.build-info.outputs.mysql-versions }} excludes: ${{ needs.build-info.outputs.mysql-exclude }} parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -964,6 +969,7 @@ jobs: backend-versions: "['']" excludes: ${{ needs.build-info.outputs.sqlite-exclude }} parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -990,6 +996,7 @@ jobs: backend-versions: "['']" excludes: ${{ needs.build-info.outputs.sqlite-exclude }} parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} @@ -1246,6 +1253,7 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" # Force more parallelism for pull on public images PARALLELISM: 6 + INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" steps: - name: "Cleanup repo" shell: bash @@ -1323,6 +1331,7 @@ jobs: if: needs.build-info.outputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" steps: - name: "Cleanup repo" shell: bash @@ -1365,6 +1374,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" if: > ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.needs-helm-tests == 'true') diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index ba95cfef17..c996a6338c 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -80,6 +80,11 @@ on: # yamllint disable-line rule:truthy Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds) required: true type: string + include-success-outputs: + description: "Whether to include success outputs or not (true/false)" + required: false + default: "false" + type: string downgrade-sqlalchemy: description: "Whether to downgrade SQLAlchemy or not (true/false)" required: false @@ -129,6 +134,7 @@ jobs: DOWN_PENDULUM: "${{ inputs.downgrade-pendulum }}" ENABLE_COVERAGE: "${{ inputs.run-coverage }}" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }} # yamllint disable rule:line-length JOB_ID: "${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}" MOUNT_SOURCES: "skip" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 4550191ae7..1bfbdd6c39 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -73,6 +73,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -95,6 +96,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -117,6 +119,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -139,6 +142,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -161,6 +165,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -183,6 +188,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -204,6 +210,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} @@ -225,6 +232,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excludes: "[]" parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} breeze-python-version: ${{ inputs.breeze-python-version }} diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index e53d7c0448..061cb1644b 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -275,6 +275,11 @@ used in the matrix to the latest ones (latest Python version and latest Kubernet You can also disable cache if you want to make sure your tests will run with image that does not have left-over package installed from the past cached image - by setting `disable image cache` label in the PR. +By default all outputs of successful parallel tests are not shown. You can enable them by setting +`include success outputs` label in the PR. This makes the logs of mostly successful tests a lot longer +and more difficult to sift through, but it might be useful in case you want to compare successful and +unsuccessful runs of the tests. + ## PR labels As mentioned below, you can influence the outputs of selected checks by setting labels to the PR. Here is @@ -285,7 +290,8 @@ am overview of possible labels and their meaning: | canary | is-canary-run | If set, the PR run from apache/airflow repo behaves as `canary` run (can only be run by maintainer). | | debug ci resources | debug-ci-resources | If set, then debugging resources is enabled during parallel tests and you can see them in the output. | | default versions only | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limited to the default ones. | -| disable image cache | | If set, the number of Python and Kubernetes versions used by the build will be limited to the latest ones. | +| disable image cache | cache-directive | If set, the image cache is disables when building the image. | +| include success outputs | include-success-outputs | By default, outputs of successful parallel tests are not shown - enabling this flag will make then shown. | | latest versions only | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limited to the latest ones. | | full tests needed | full-tests-needed | Run complete set of tests, including all Python all DB versions, and all test types. | | non committer build | is-committer-build | If set then even for non-committer builds, the scripts used for images are used from target branch. | diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index cb8bd5723e..e0bf83cd63 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -336,8 +336,19 @@ def run_tests_in_parallel( debug_resources: bool, parallelism: int, skip_cleanup: bool, - skio_docker_compose_down: bool, + skip_docker_compose_down: bool, ) -> None: + get_console().print("\n[info]Summary of the tests to run\n") + get_console().print(f"[info]Running tests in parallel with parallelism={parallelism}") + get_console().print(f"[info]Extra pytest args: {extra_pytest_args}") + get_console().print(f"[info]DB reset: {db_reset}") + get_console().print(f"[info]Test timeout: {test_timeout}") + get_console().print(f"[info]Include success outputs: {include_success_outputs}") + get_console().print(f"[info]Debug resources: {debug_resources}") + get_console().print(f"[info]Skip cleanup: {skip_cleanup}") + get_console().print(f"[info]Skip docker-compose down: {skip_docker_compose_down}") + get_console().print("[info]Shell params:") + get_console().print(shell_params.__dict__) _run_tests_in_pool( tests_to_run=shell_params.parallel_test_types_list, parallelism=parallelism, @@ -348,7 +359,7 @@ def run_tests_in_parallel( include_success_outputs=include_success_outputs, debug_resources=debug_resources, skip_cleanup=skip_cleanup, - skip_docker_compose_down=skio_docker_compose_down, + skip_docker_compose_down=skip_docker_compose_down, ) @@ -684,7 +695,7 @@ def _run_test_command( parallelism=parallelism, skip_cleanup=skip_cleanup, debug_resources=debug_resources, - skio_docker_compose_down=skip_docker_compose_down, + skip_docker_compose_down=skip_docker_compose_down, ) else: if shell_params.test_type == "Default": diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index eebee874bd..6214a7e3b3 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -70,6 +70,7 @@ NON_COMMITTER_BUILD_LABEL = "non committer build" DEFAULT_VERSIONS_ONLY_LABEL = "default versions only" LATEST_VERSIONS_ONLY_LABEL = "latest versions only" DISABLE_IMAGE_CACHE_LABEL = "disable image cache" +INCLUDE_SUCCESS_OUTPUTS_LABEL = "include success outputs" UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies" @@ -833,6 +834,12 @@ class SelectiveChecks: self._extract_long_provider_tests(current_test_types) return " ".join(sorted(current_test_types)) + @cached_property + def include_success_outputs( + self, + ) -> bool: + return INCLUDE_SUCCESS_OUTPUTS_LABEL in self._pr_labels + @cached_property def basic_checks_only(self) -> bool: return not self.ci_image_build