potiuk commented on a change in pull request #14531:
URL: https://github.com/apache/airflow/pull/14531#discussion_r586588251
##########
File path: scripts/ci/testing/ci_run_airflow_testing.sh
##########
@@ -170,5 +161,282 @@ do
echo
"**********************************************************************************************"
run_airflow_testing_in_docker "${@}"
+}
+
+
+# Cleans up Docker before test execution. System prune should clean all the
temporary/unnamed images
+# And left-over volumes
+function cleanup_docker() {
+ echo
+ echo "System-prune docker"
+ echo
+ docker system prune --force --volumes
+ echo
+ echo "Check available space"
+ echo
+ df --human
+ echo
+ echo "Check available memory"
+ echo
+ free --human
+}
+
+# Prints status of a parallel test type. Returns 0 if the test type is still
running, 1 otherwise
+#
+# $1 - Test type
+#
+# The test_pids variable should contain mapping of test types to pids
+# NOTE: It might be surprise that local variables (like test_pids) are visible
to called functions
+# But this is how local variables in bash work (by design).
+function print_test_type_status() {
+ local test_type=$1
+ local pid="${test_pids[${test_type}]}"
+ local log_file="${CACHE_TMP_FILE_DIR}/output-${test_type}.log"
+ if ps -p "${pid}" >/dev/null 2>/dev/null; then
+ if [[ ${TEST_SHOW_ALL_OUTPUT_LINES_AS_THEY_APPEAR} != "true" ]]; then
+ echo "${COLOR_BLUE}Test type ${test_type} in progress (last
${TEST_SHOW_OUTPUT_LINES} lines of the output shown).${COLOR_RESET}"
+ echo
+ tail -${TEST_SHOW_OUTPUT_LINES} "${log_file}"
+ echo
+ echo
+ fi
+ return 0
+ else
+ if wait "${pid}"; then
+ echo
+ echo "${COLOR_GREEN}Test type: ${test_type}
succeeded.${COLOR_RESET}"
+ echo
+ return 1
+ else
+ echo
+ echo "${COLOR_RED}Test type: ${test_type} failed.${COLOR_RESET}"
+ echo
+ return 1
+ fi
+ fi
+}
+
+# Prints status of all parallel test types. Returns 0 if any of the tests are
still running, 1 otherwise
+function print_all_test_types_status() {
+ local still_running="false"
+ local test_type
+ echo "${COLOR_YELLOW}Progress for all tests:
${COLOR_BLUE}${TEST_TYPES}${COLOR_RESET}"
+ echo
+ for test_type in ${TEST_TYPES}
+ do
+ if print_test_type_status "${test_type}" ; then
+ still_running="true"
+ fi
+ done
+ if [[ ${still_running} == "true" ]]; then
+ return 0
+ fi
+ return 1
+}
+
+# Starts all test types in parallel
+# TEST_TYPES - list of all test types (it's not an array, it is space-separate
list)
+# ${@} - additional arguments to pass to test execution
+function start_test_types_in_parallel() {
+ local test_type
+ local database_port
+ start_end::group_start "Starting parallel tests: ${TEST_TYPES}"
+ for test_type in ${TEST_TYPES}
+ do
+ # Grabbing a free random port to use for database forwarding
+ database_port="$(python -c 'import socket; s=socket.socket();
s.bind(("", 0)); print(s.getsockname()[1])')";
+ echo
+ echo "Grabbed database port for forwarding: ${database_port}"
+ log_file="${CACHE_TMP_FILE_DIR}/output-${test_type}.log"
+ TEST_TYPE=${test_type} MYSQL_HOST_PORT="${database_port}"
POSTGRES_HOST_PORT="${database_port}" \
+ run_single_test_type "${@}" >"${log_file}" 2>&1 &
+ pid=$!
+ echo "The process started for ${test_type}. Log file ${log_file}"
+ echo
+ test_pids[${test_type}]=${pid}
+ done
+ start_end::group_end
+}
+
+# Monitors progress of all test types running
+# TEST_TYPES - all test types to monitor
+# test_pids - map of test types to PIDs
+function monitor_test_types() {
+ start_end::group_start "Monitoring progress of running parallel tests:
${TEST_TYPES}"
+ while true;
+ do
+ sleep "${TEST_SLEEP_TIME}"
+ if ! print_all_test_types_status; then
+ break
+ fi
+ done
+ start_end::group_end
+}
+
+
+# Monitors progress of single test type running
+# $1 - test type to monitor
+# test_pids - map of test types to PIDs
+function monitor_sequential_test_type() {
+ local test_type=$1
+ start_end::group_start "Monitoring progress of running sequential test
type: $1"
+ while true;
+ do
+ sleep "${TEST_SLEEP_TIME}"
+ if ! print_test_type_status "${test_type}"; then
+ break
+ fi
+ done
+ start_end::group_end
+}
+
+# Checks status for all test types running. Returns 0 if any of the tests
failed (with the exception of
+# Quarantined test.
+#
+# TEST_TYPES - all test types to run
+# test_pids - map of test types to PIDs
+#
+# output:
+# successful_tests - array of test types that succeeded
+# failed_tests - array of test types that failed
+function check_test_types_status() {
+ start_end::group_start "Checking status for parallel tests: ${TEST_TYPES}"
+ local test_type_exit_code="0"
+ for test_type in "${!test_pids[@]}"; do
+ local pid="${test_pids[${test_type}]}"
+ wait "${pid}"
+ local temp_exit_code=$?
+ if [[ ${temp_exit_code} == "0" ]]; then
+ successful_tests+=("${test_type}")
+ else
+ failed_tests+=("${test_type}")
+ if [[ ${test_type} != "Quarantined" ]]; then
+ test_type_exit_code=${temp_exit_code}``
+ fi
+ fi
+ done
+ start_end::group_end
+ return "${test_type_exit_code}"
+}
+
+# Outputs logs for failed test type
+# $1 test type
+function output_log_for_failed_test_type(){
+ local failed_test_type=$1
+ local log_file="${CACHE_TMP_FILE_DIR}/output-${failed_test_type}.log"
+ start_end::group_start "${COLOR_RED}Output for failed ${failed_test_type}
from log: ${log_file}${COLOR_RESET}"
+ echo "${COLOR_RED}##### Test type ${failed_test_type} failed #####
${COLOR_RESET}"
+ echo
+ cat "${log_file}"
+ echo
+ echo "${COLOR_RED}##### Test type ${failed_test_type} failed #####
${COLOR_RESET}"
+ echo
+ start_end::group_end
+}
+
+function print_test_summary() {
+ if [[ ${#successful_tests[@]} != 0 ]]; then
+ echo
+ echo "${COLOR_GREEN}Successful test types: ${successful_tests[*]}
${COLOR_RESET}"
+ echo
Review comment:
Yeah. That's a good reason to keep it - and on CI those are folded so
yeah . we can always show it no problem.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
