This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6db73a6 Reduce number of lines in "monitoring job" parallel group
(#17995)
6db73a6 is described below
commit 6db73a60c9f6e0ba46712cf44b9f1e506450fcae
Author: Ash Berlin-Taylor <[email protected]>
AuthorDate: Fri Sep 3 09:31:08 2021 +0100
Reduce number of lines in "monitoring job" parallel group (#17995)
Although this group is collapsed when the job is finished it it is large
it can cause the logs view in GitHub to bog down the browser, so this PR
does a few things:
- Stops showing disk information for "overlay" FS types -- i.e. docker
containers as they share the same disk usage as the docker root
- Removes a few "duplicated" messages to further reduce output
- Sleep for 15s instead of 10 between poll attempts
---
scripts/ci/libraries/_docker_engine_resources.sh | 18 +++++++++---------
scripts/ci/libraries/_parallel.sh | 21 ++++++---------------
.../testing/ci_run_single_airflow_test_in_docker.sh | 2 --
3 files changed, 15 insertions(+), 26 deletions(-)
diff --git a/scripts/ci/libraries/_docker_engine_resources.sh
b/scripts/ci/libraries/_docker_engine_resources.sh
index 0f66740..7bcf427 100644
--- a/scripts/ci/libraries/_docker_engine_resources.sh
+++ b/scripts/ci/libraries/_docker_engine_resources.sh
@@ -18,18 +18,18 @@
function docker_engine_resources::print_overall_stats() {
- echo
- echo "Docker statistics"
- echo
- docker stats --all --no-stream --no-trunc
- echo
- echo "Memory statistics"
+ docker stats --all --no-stream
echo
docker run --rm --entrypoint /bin/sh "alpine:latest" -c "free -m"
echo
- echo "Disk statistics"
- echo
- df -h || true
+ df -h \
+ --exclude-type devtmpfs \
+ --exclude-type overlay \
+ --exclude-type squashfs \
+ | grep -v " /run" \
+ | grep -v " /sys" \
+ | grep -v "/dev/shm" \
+ || true
}
function docker_engine_resources::get_available_cpus_in_docker() {
diff --git a/scripts/ci/libraries/_parallel.sh
b/scripts/ci/libraries/_parallel.sh
index d5b99c7..2d4b96f 100644
--- a/scripts/ci/libraries/_parallel.sh
+++ b/scripts/ci/libraries/_parallel.sh
@@ -73,7 +73,7 @@ function parallel::monitor_loop() {
while true
do
echo
- echo "${COLOR_YELLOW}########### Monitoring progress start:
${progress_report_number} #################${COLOR_RESET}"
+ echo "${COLOR_YELLOW}########## Monitoring progress start:
${progress_report_number} ##########${COLOR_RESET}"
echo
echo "${COLOR_BLUE}########### STATISTICS #################"
docker_engine_resources::print_overall_stats
@@ -87,31 +87,22 @@ function parallel::monitor_loop() {
fi
echo "${COLOR_BLUE}### The last ${PARALLEL_TAIL_LENGTH} lines for
${parallel_process} process: ${directory}/stdout ###${COLOR_RESET}"
- echo
tail "-${PARALLEL_TAIL_LENGTH}" "${directory}/stdout" || true
echo
if [[ -s "${directory}/status" ]]; then
finished_jobs+=("$parallel_process")
- status=$(cat "${directory}/status")
-
- if [[ $status == 0 ]]; then
- local color="$COLOR_GREEN"
- else
- local color="$COLOR_RED"
- fi
- echo "${color}### Test ${parallel_process} exited with
${status}${COLOR_RESET}"
+ # The last line of output (which we've already shown) will be a
line about the success/failure
+ # of this job
fi
echo
done
- echo
- echo "${COLOR_YELLOW}########### Monitoring progress end:
${progress_report_number} #################${COLOR_RESET}"
- echo
+
end_time=${SECONDS}
- echo "${COLOR_YELLOW}############## $((end_time - start_time)) seconds
passed since start ####################### ${COLOR_RESET}"
- sleep 10
+ echo "${COLOR_YELLOW}########## $((end_time - start_time)) seconds
passed since start ##########${COLOR_RESET}"
+ sleep 15
progress_report_number=$((progress_report_number + 1))
done
}
diff --git a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
index 16ae041..19983f0 100755
--- a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
+++ b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
@@ -176,11 +176,9 @@ function run_airflow_testing_in_docker() {
if [[ ${exit_code} == 0 ]]; then
echo
echo "${COLOR_GREEN}Test type: ${TEST_TYPE} succeeded.${COLOR_RESET}"
- echo
else
echo
echo "${COLOR_RED}Test type: ${TEST_TYPE} failed.${COLOR_RESET}"
- echo
fi
return "${exit_code}"
}
