This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 94749b0 Do not display resource statistics during parallell jobs
(#18344)
94749b0 is described below
commit 94749b056999d31ef728864a44a9016d0e24f7ea
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Sep 19 00:27:21 2021 +0200
Do not display resource statistics during parallell jobs (#18344)
Those stats are useful to analyse problems but they are
verbose and misleading for "normal" runs. The stats
after this PR will only be displayed in `push` events or
in case committers manually set "debug-ci-resources" label
on PR
---
.github/workflows/ci.yml | 21 +++++++++++++++++++++
scripts/ci/libraries/_parallel.sh | 9 ++++++---
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 257dc77..617a1e3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -320,6 +320,7 @@ jobs:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
VERIFY_IMAGE: "true"
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
static-checks:
timeout-minutes: 30
@@ -593,6 +594,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: Helm"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -651,6 +654,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -708,6 +713,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -764,6 +771,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -818,6 +827,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -881,6 +892,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: Quarantined"
run: ./scripts/ci/testing/ci_run_quarantined_tests.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload Quarantine test results"
uses: actions/upload-artifact@v2
if: always()
@@ -985,6 +998,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
VERIFY_IMAGE: "true"
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
tests-kubernetes:
timeout-minutes: 50
@@ -1037,6 +1051,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: "kubernetes-binaries"
- name: "Kubernetes Tests"
run:
./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload KinD logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -1103,6 +1119,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: "kubernetes-binaries"
- name: "Kubernetes Helm Chart Executor Upgrade Tests"
run:
./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
+ env:
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Upload KinD logs"
uses: actions/upload-artifact@v2
if: failure()
@@ -1155,14 +1173,17 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
env:
GENERATE_CONSTRAINTS_MODE: "pypi-providers"
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Generate constraints with source providers"
run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
env:
GENERATE_CONSTRAINTS_MODE: "source-providers"
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Generate constraints without providers"
run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
env:
GENERATE_CONSTRAINTS_MODE: "no-providers"
+ PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
diff --git a/scripts/ci/libraries/_parallel.sh
b/scripts/ci/libraries/_parallel.sh
index 2d4b96f..d5ff807 100644
--- a/scripts/ci/libraries/_parallel.sh
+++ b/scripts/ci/libraries/_parallel.sh
@@ -75,9 +75,12 @@ function parallel::monitor_loop() {
echo
echo "${COLOR_YELLOW}########## Monitoring progress start:
${progress_report_number} ##########${COLOR_RESET}"
echo
- echo "${COLOR_BLUE}########### STATISTICS #################"
- docker_engine_resources::print_overall_stats
- echo "########### STATISTICS #################${COLOR_RESET}"
+ if [[ ${PR_LABELS} == *debug-ci-resources* || ${GITHUB_EVENT_NAME} ==
"push" ]]; then
+ # Only print stats in `main` or when "debug-ci-resources" label is
set on PR.
+ echo "${COLOR_BLUE}########### STATISTICS #################"
+ docker_engine_resources::print_overall_stats
+ echo "########### STATISTICS #################${COLOR_RESET}"
+ fi
for directory in "${PARALLEL_MONITORED_DIR}"/*/*
do
parallel_process=$(basename "${directory}")
