This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7b777ba54bfeedadea91374897f34a92a67c88ab Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Dec 1 17:44:05 2020 +0100 User-friendly output of Breeze and CI scripts (#12735) (cherry picked from commit a02e0f746f85e6ac8786bc1d99bf78994e66b5cf) --- breeze | 41 +++++------ scripts/ci/images/ci_verify_ci_image.sh | 4 +- scripts/ci/images/ci_verify_prod_image.sh | 15 ++-- scripts/ci/libraries/_build_images.sh | 86 ++++++++++++---------- scripts/ci/libraries/_initialization.sh | 20 +++++ scripts/ci/libraries/_kind.sh | 30 ++++---- scripts/ci/libraries/_parameters.sh | 17 +++-- scripts/ci/libraries/_push_pull_remove_images.sh | 38 +++++----- scripts/ci/libraries/_repeats.sh | 13 ++-- scripts/ci/libraries/_sanity_checks.sh | 68 +++++++++-------- .../ci/pre_commit/pre_commit_check_pre_commits.sh | 40 +++++----- scripts/ci/pre_commit/pre_commit_mermaid.sh | 6 +- scripts/ci/selective_ci_checks.sh | 8 +- scripts/ci/static_checks/check_license.sh | 15 ++-- scripts/in_container/_in_container_script_init.sh | 6 +- scripts/in_container/_in_container_utils.sh | 51 ++++++++++--- scripts/in_container/check_environment.sh | 5 +- scripts/in_container/entrypoint_ci.sh | 6 +- scripts/in_container/run_ci_tests.sh | 83 +++++++++++---------- scripts/in_container/run_cli_tool.sh | 20 +++-- scripts/in_container/run_system_tests.sh | 2 + tests/bats/breeze/test_breeze_params.bats | 17 +---- tests/bats/in_container/test_in_container.bats | 2 + 23 files changed, 339 insertions(+), 254 deletions(-) diff --git a/breeze b/breeze index 6f73ad7..d99ba02 100755 --- a/breeze +++ b/breeze @@ -215,11 +215,11 @@ function breeze::initialize_virtualenv() { local res=$? set -e if [[ ${res} != "0" ]]; then - echo >&2 - echo >&2 "ERROR: Initializing local virtualenv only works when you have virtualenv activated" - echo >&2 - echo >&2 "Please enter your local virtualenv before (for example using 'pyenv activate' or 'workon') " - echo >&2 + echo + echo "${COLOR_RED_ERROR} Initializing local virtualenv only works when you have virtualenv activated ${COLOR_RESET}" + echo + echo "Please enter your local virtualenv before (for example using 'pyenv activate' or 'workon') " + echo exit 1 else echo @@ -1138,9 +1138,9 @@ function breeze::parse_arguments() { ;; *) breeze::flags - echo >&2 - echo >&2 "ERROR: Unknown flag ${1}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Unknown flag ${COLOR_RESET}" + echo exit 1 ;; esac @@ -1323,9 +1323,9 @@ function breeze::parse_arguments() { ;; *) breeze::usage - echo >&2 - echo >&2 "ERROR: Unknown command ${1}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Unknown command ${COLOR_RESET}" + echo exit 1 ;; esac @@ -2769,10 +2769,9 @@ function breeze::make_sure_precommit_is_installed() { elif command -v pip >/dev/null; then pip_binary=pip else - echo >&2 - echo >&2 "ERROR: You need to have pip or pip3 in your PATH" - echo >&2 - S + echo + echo "${COLOR_RED_ERROR} You need to have pip or pip3 in your PATH ${COLOR_RESET}" + echo exit 1 fi "${pip_binary}" install --upgrade pre-commit >/dev/null 2>&1 @@ -2934,9 +2933,9 @@ function breeze::run_build_command() { fi ;; *) - echo >&2 - echo >&2 "ERROR: Unknown command to run ${command_to_run}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Unknown command to run ${command_to_run} ${COLOR_RESET}" + echo exit 1 ;; esac @@ -3059,9 +3058,9 @@ function breeze::run_breeze_command() { fi ;; *) - echo >&2 - echo >&2 "ERROR: Unknown command to run ${command_to_run}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Unknown command to run ${command_to_run} ${COLOR_RESET}" + echo ;; esac set -u diff --git a/scripts/ci/images/ci_verify_ci_image.sh b/scripts/ci/images/ci_verify_ci_image.sh index e1f2b98..004eac0 100755 --- a/scripts/ci/images/ci_verify_ci_image.sh +++ b/scripts/ci/images/ci_verify_ci_image.sh @@ -28,12 +28,12 @@ function verify_ci_image_dependencies() { docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c 'pip check' local res=$? if [[ ${res} != "0" ]]; then - echo -e " \e[31mERROR: ^^^ Some dependencies are conflicting. See instructions below on how to deal with it.\e[0m" + echo "${COLOR_RED_ERROR} ^^^ Some dependencies are conflicting. See instructions below on how to deal with it. ${COLOR_RESET}" echo build_images::inform_about_pip_check "" else echo - echo -e " \e[32mOK. The ${AIRFLOW_PROD_IMAGE} image dependencies are consistent.\e[0m" + echo "${COLOR_GREEN_OK} The ${AIRFLOW_PROD_IMAGE} image dependencies are consistent. ${COLOR_RESET}" echo fi set -e diff --git a/scripts/ci/images/ci_verify_prod_image.sh b/scripts/ci/images/ci_verify_prod_image.sh index 9718a48..274261b 100755 --- a/scripts/ci/images/ci_verify_prod_image.sh +++ b/scripts/ci/images/ci_verify_prod_image.sh @@ -39,14 +39,15 @@ function verify_prod_image_has_airflow { echo if [[ "${COUNT_AIRFLOW_DIRS}" -lt "${EXPECTED_MIN_AIRFLOW_DIRS_COUNT}" ]]; then - >&2 echo - >&2 echo Number of airflow folders installed is less than ${EXPECTED_MIN_AIRFLOW_DIRS_COUNT} - >&2 echo This is unexpected. Please investigate, looking at the output above! - >&2 echo + echo + echo "${COLOR_RED_ERROR} Number of airflow folders installed is less than ${EXPECTED_MIN_AIRFLOW_DIRS_COUNT} ${COLOR_RESET}" + echo + echo "This is unexpected. Please investigate, looking at the output above!" + echo exit 1 else echo - echo -e " \e[32mOK. Airflow is installed.\e[0m" + echo "${COLOR_GREEN_OK} Airflow is installed. ${COLOR_RESET}" echo fi } @@ -62,14 +63,14 @@ function verify_prod_image_dependencies { docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check' local res=$? if [[ ${res} != "0" ]]; then - echo -e " \e[31mERROR: ^^^ Some dependencies are conflicting. See instructions below on how to deal with it.\e[0m" + echo "${COLOR_RED_ERROR} ^^^ Some dependencies are conflicting. See instructions below on how to deal with it. ${COLOR_RESET}" echo build_images::inform_about_pip_check "--production " # TODO(potiuk) - enable the comment once https://github.com/apache/airflow/pull/12188 is merged # exit ${res} else echo - echo " \e[32mOK. The ${AIRFLOW_PROD_IMAGE} image dependencies are consistent.\e[0m" + echo " \e[32mOK. The ${AIRFLOW_PROD_IMAGE} image dependencies are consistent. ${COLOR_RESET}" echo fi set -e diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 30e7a85..17889ce 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -180,14 +180,17 @@ function build_images::confirm_image_rebuild() { export FORCE_ANSWER_TO_QUESTIONS="no" echo 'export FORCE_ANSWER_TO_QUESTIONS="no"' >"${LAST_FORCE_ANSWER_FILE}" elif [[ ${RES} == "2" ]]; then - echo >&2 - echo >&2 "ERROR: The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. " - echo >&2 " Make sure you build the images bu running" - echo >&2 - echo >&2 " ./breeze --python ${PYTHON_MAJOR_MINOR_VERSION}" build-image - echo >&2 - echo >&2 " If you run it via pre-commit as individual hook, you can run 'pre-commit run build'." - echo >&2 + echo + echo "${COLOR_RED_ERROR} The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. ${COLOR_RESET}" + echo """ + + Make sure you build the images bu running + + ./breeze --python ${PYTHON_MAJOR_MINOR_VERSION} build-image + + If you run it via pre-commit as individual hook, you can run 'pre-commit run build'. + +""" exit 1 else # Force "yes" also to subsequent questions @@ -535,9 +538,9 @@ function build_images::build_ci_image() { "--cache-from" "${AIRFLOW_CI_IMAGE}" ) else - echo >&2 - echo >&2 "Error - the ${DOCKER_CACHE} cache is unknown!" - echo >&2 + echo + echo "${COLOR_RED_ERROR} The ${DOCKER_CACHE} cache is unknown! ${COLOR_RESET}" + echo exit 1 fi EXTRA_DOCKER_CI_BUILD_FLAGS=( @@ -711,9 +714,10 @@ function build_images::build_prod_images() { "--cache-from" "${AIRFLOW_PROD_BUILD_IMAGE}" ) else - echo >&2 - echo >&2 "Error - thee ${DOCKER_CACHE} cache is unknown!" - echo >&2 + echo + echo "${COLOR_RED_ERROR} The ${DOCKER_CACHE} cache is unknown ${COLOR_RESET}" + echo + echo exit 1 fi set +u @@ -853,60 +857,64 @@ function build_images::determine_docker_cache_strategy() { # Useful information for people who stumble upon a pip check failure function build_images::inform_about_pip_check() { - >&2 echo """ + echo """ +${COLOR_BLUE}***** Beginning of the instructions ****${COLOR_RESET} The image did not pass 'pip check' verification. This means that there are some conflicting dependencies -in the image. Usually it means that some setup.py or setup.cfg limits need to be adjusted to fix it. +in the image. + +It can mean one of those: + +1) The master is currently broken (other PRs will fail with the same error) +2) You changed some dependencies in setup.py or setup.cfg and they are conflicting. -Usually it happens when one of the dependencies gets upgraded and it has more strict requirements -than the other dependencies and they are conflicting. -In case you did not update setup.py or any of your dependencies, this error might happen in case -someone accidentally merges conflicting dependencies in master. This -should not happen as we are running 'pip check' as dependency before we upgrade the constrained -dependencies, but we could miss some edge cases (thank you for your patience). Please let committer now -and apologies for the troubles. You do not have to do anything in this case. You might be asked to -rebase to the latest master after the problem is fixed. +In case 1) - apologies for the trouble.Please let committers know and they will fix it. You might +be asked to rebase to the latest master after the problem is fixed. -In case you actually updated setup.py, there are some steps you can take to address that: +In case 2) - Follow the steps below: -* first of all ask the committer to set 'upgrade to newer dependencies' and 'full tests needed' labels - for your PR. This will turn your PR in mode where all the dependencies are upgraded to latest matching - dependencies and the checks will run for all python versions +* consult the committers if you are unsure what to do. Just comment in the PR that you need help, if you do, + but try to follow those instructions first! -* run locally the image that is failing with Breeze - this will make it easy to manually try to update - the setup.py and test the consequences of changing constraints. You can do it by checking out your PR - and running this command: +* ask the committer to set 'upgrade to newer dependencies'. All dependencies in your PR will be updated + to latest 'good' versions and you will be able to check if they are not conflicting. + +* run locally the image that is failing with Breeze: ./breeze ${1}--github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} --python ${PYTHON_MAJOR_MINOR_VERSION} -* your setup.py and setup.cfg will be mounted to the container and you will be able to iterate with +* your setup.py and setup.cfg will be mounted to the container. You will be able to iterate with different setup.py versions. -* run 'pipdeptree' to figure out where the dependency conflict comes from. Useful commands that can help you - to find out dependencies you have are: +* in container your can run 'pipdeptree' to figure out where the dependency conflict comes from. + +* Some useful commands that can help yoy to find out dependencies you have: + * 'pipdeptree | less' (you can then search through the dependencies with vim-like shortcuts) + * 'pipdeptree > /files/pipdeptree.txt' - this will produce a pipdeptree.txt file in your source 'files' directory and you can open it in editor of your choice, + * 'pipdeptree | grep YOUR_DEPENDENCY' - to see all the requirements your dependency has as specified by other packages -* figure out which dependency limits should be upgraded. First try to upgrade them in setup.py extras +* figure out which dependency limits should be upgraded. Upgrade them in corresponding setup.py extras and run pip to upgrade your dependencies accordingly: pip install '.[all]' --upgrade --upgrade-strategy eager -* run pip check to figure out if the dependencies have been fixed (it should let you know which dependencies +* run pip check to figure out if the dependencies have been fixed. It should let you know which dependencies are conflicting or (hurray!) if there are no conflicts: pip check * in some, rare, cases, pip will not limit the requirement in case you specify it in extras, you might - need to add such requirement in 'install_requires' section of setup.cfg in order to have pip take it into - account. This will happen if higher version of your dependency is already installed in 'install_requires' - section. In such case update 'setup.cfg' and run pip install/pip check from the previous steps + need to add such requirement in 'install_requires' section of setup.cfg instead of extras in setup.py. * iterate until all such dependency conflicts are fixed. +${COLOR_BLUE}***** End of the instructions ****${COLOR_RESET} + """ } diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 5890834..9c3ab67 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -458,8 +458,28 @@ function initialization::initialize_build_image_variables() { REMOTE_IMAGE_BUILD_CACHE_HASH_FILE="${AIRFLOW_SOURCES}/manifests/remote-build-cache-hash" } +function initialization::set_output_color_variables() { + COLOR_BLUE=$'\e[37m' + COLOR_GREEN=$'\e[32m' + COLOR_GREEN_OK=$'\e[32mOK.' + COLOR_RED=$'\e[31m' + COLOR_RED_ERROR=$'\e[31mERROR:' + COLOR_RESET=$'\e[0m' + COLOR_YELLOW=$'\e[33m' + COLOR_YELLOW_WARNING=$'\e[33mWARNING:' + export COLOR_BLUE + export COLOR_GREEN + export COLOR_GREEN_OK + export COLOR_RED + export COLOR_RED_ERROR + export COLOR_RESET + export COLOR_YELLOW + export COLOR_YELLOW_WARNING +} + # Common environment that is initialized by both Breeze and CI scripts function initialization::initialize_common_environment() { + initialization::set_output_color_variables initialization::create_directories initialization::initialize_base_variables initialization::initialize_branch_variables diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index 3a170cb..ab48007 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -121,9 +121,9 @@ function kind::perform_kind_cluster_operation() { ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell recreate k9s]" set +u if [[ -z "${1=}" ]]; then - echo >&2 - echo >&2 "Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS} ${COLOR_RESET}" + echo exit 1 fi set -u @@ -202,9 +202,9 @@ function kind::perform_kind_cluster_operation() { -e EDITOR -e K9S_EDITOR \ -v "${KUBECONFIG}:/root/.kube/config" quay.io/derailed/k9s else - echo >&2 - echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of: ${ALLOWED_KIND_OPERATIONS}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Wrong cluster operation: ${OPERATION}. Should be one of: ${ALLOWED_KIND_OPERATIONS} ${COLOR_RESET}" + echo exit 1 fi else @@ -220,14 +220,14 @@ function kind::perform_kind_cluster_operation() { echo kind::create_cluster elif [[ ${OPERATION} == "stop" || ${OPERATION} == "deploy" || ${OPERATION} == "test" || ${OPERATION} == "shell" ]]; then - echo >&2 - echo >&2 "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation ${COLOR_RESET}" + echo exit 1 else - echo >&2 - echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of ${ALLOWED_KIND_OPERATIONS}" - echo >&2 + echo + echo "${COLOR_RED_ERROR} Wrong cluster operation: ${OPERATION}. Should be one of ${ALLOWED_KIND_OPERATIONS} ${COLOR_RESET}" + echo exit 1 fi fi @@ -293,9 +293,9 @@ function kind::wait_for_webserver_healthy() { sleep "${SLEEP_TIME_FOR_HEALTH_CHECK}" num_tries=$((num_tries + 1)) if [[ ${num_tries} == "${MAX_NUM_TRIES_FOR_HEALTH_CHECK}" ]]; then - >&2 echo - >&2 echo "Timeout while waiting for the webserver health check" - >&2 echo + echo + echo "${COLOR_RED_ERROR} Timeout while waiting for the webserver health check ${COLOR_RESET}" + echo fi done echo diff --git a/scripts/ci/libraries/_parameters.sh b/scripts/ci/libraries/_parameters.sh index d655853..566585e 100644 --- a/scripts/ci/libraries/_parameters.sh +++ b/scripts/ci/libraries/_parameters.sh @@ -40,15 +40,16 @@ function parameters::check_allowed_param() { _ALLOWED_VALUES=" ${!_ALLOWED_VALUES_ENV_NAME//$'\n'/ } " _VALUE=${!_VARIABLE_NAME} if [[ ${_ALLOWED_VALUES:=} != *" ${_VALUE} "* ]]; then - echo >&2 - echo >&2 "ERROR: Allowed ${_VARIABLE_DESCRIPTIVE_NAME}: [${_ALLOWED_VALUES}]. Passed: '${!_VARIABLE_NAME}'." - echo >&2 - echo >&2 "Switch to supported value with ${_FLAG} flag." - + echo + echo "${COLOR_RED_ERROR} Allowed ${_VARIABLE_DESCRIPTIVE_NAME}: [${_ALLOWED_VALUES}]. Passed: '${!_VARIABLE_NAME}' ${COLOR_RESET}" + echo + echo "Switch to supported value with ${_FLAG} flag." + echo if [[ -n ${!_VARIABLE_NAME} && -f "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}" && ${!_VARIABLE_NAME} == $(cat "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}") ]]; then - echo >&2 - echo >&2 "Removing ${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}. Next time you run it, it should be OK." - echo >&2 + echo + echo "${COLOR_YELLOW_WARNING}: Removing ${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}. Next time you run it, it should be OK. ${COLOR_RESET}" + echo + echo rm -f "${BUILD_CACHE_DIR}/.${_VARIABLE_NAME}" fi exit 1 diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index 216e025..fff08ad 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -29,17 +29,17 @@ function push_pull_remove_images::push_image_with_retries() { local res=$? set -e if [[ ${res} != "0" ]]; then - >&2 echo - >&2 echo "Error ${res} when pushing image on ${try_num} try" - >&2 echo + echo + echo "${COLOR_YELLOW_WARNING}: Error ${res} when pushing image on ${try_num} try ${COLOR_RESET}" + echo continue else return 0 fi done - >&2 echo - >&2 echo "Error ${res} when pushing image on ${try_num} try. Giving up!" - >&2 echo + echo + echo "${COLOR_RED_ERROR} Error ${res} when pushing image on ${try_num} try. Giving up! ${COLOR_RESET}" + echo return 1 } @@ -64,17 +64,19 @@ function push_pull_remove_images::pull_image_if_not_present_or_forced() { docker pull "${IMAGE_TO_PULL}" EXIT_VALUE="$?" if [[ ${EXIT_VALUE} != "0" && ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} == "true" ]]; then - >&2 echo - >&2 echo "ERROR! Exiting on docker pull error" - >&2 echo - >&2 echo "If you have authorisation problems, you might want to run:" - >&2 echo - >&2 echo "docker login ${IMAGE_TO_PULL%%\/*}" - >&2 echo - >&2 echo "You need to use generate token as the password, not your personal password." - >&2 echo "You can generete one at https://github.com/settings/tokens"; - >&2 echo "Make sure to choose 'read:packages' scope". - >&2 echo + echo + echo """ +${COLOR_RED_ERROR} Exiting on docker pull error + +If you have authorisation problems, you might want to run: + +docker login ${IMAGE_TO_PULL%%\/*} + +You need to use generate token as the password, not your personal password. +You can generate one at https://github.com/settings/tokens +Make sure to choose 'read:packages' scope. +${COLOR_RESET} +""" exit ${EXIT_VALUE} fi echo @@ -282,7 +284,7 @@ function push_pull_remove_images::wait_for_github_registry_image() { digest=$(jq '.config.digest' < "${OUTPUT_LOG}") echo -n "." if [[ ${digest} != "null" ]]; then - echo -e " \e[32mOK.\e[0m" + echo "${COLOR_GREEN_OK} ${COLOR_RESET}" break fi sleep 10 diff --git a/scripts/ci/libraries/_repeats.sh b/scripts/ci/libraries/_repeats.sh index 96e4582..b245c5b 100644 --- a/scripts/ci/libraries/_repeats.sh +++ b/scripts/ci/libraries/_repeats.sh @@ -34,12 +34,13 @@ function repeats::run_with_retry() { if [[ ${res} == "0" ]]; then return 0 fi - >&2 echo - >&2 echo "Unsuccessful attempt no. ${n}. Result: ${res}" - >&2 echo + echo + echo "${COLOR_YELLOW_WARNING}: Unsuccessful attempt no. ${n}. Result: ${res} ${COLOR_RESET}" + echo + echo done - >&2 echo - >&2 echo "Giving up after ${num_repeats} attempts!" - >&2 echo + echo + echo "${COLOR_RED_ERROR} Giving up after ${num_repeats} attempts! ${COLOR_RESET}" + echo return ${res} } diff --git a/scripts/ci/libraries/_sanity_checks.sh b/scripts/ci/libraries/_sanity_checks.sh index 9347bf4..3138776 100644 --- a/scripts/ci/libraries/_sanity_checks.sh +++ b/scripts/ci/libraries/_sanity_checks.sh @@ -77,32 +77,36 @@ function sanity_checks::check_if_coreutils_installed() { if [[ ${GETOPT_RETVAL} != 4 || "${STAT_PRESENT}" != "0" || "${MD5SUM_PRESENT}" != "0" ]]; then verbosity::print_info if [[ $(uname -s) == 'Darwin' ]] ; then - echo >&2 "You are running ${CMDNAME} in OSX environment" - echo >&2 "And you need to install gnu commands" - echo >&2 - echo >&2 "Run 'brew install gnu-getopt coreutils'" - echo >&2 - echo >&2 "Then link the gnu-getopt to become default as suggested by brew." - echo >&2 - echo >&2 "If you use bash, you should run these commands:" - echo >&2 - echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.bash_profile" - echo >&2 ". ~/.bash_profile" - echo >&2 - echo >&2 "If you use zsh, you should run these commands:" - echo >&2 - echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.zprofile" - echo >&2 ". ~/.zprofile" - echo >&2 - echo >&2 "Either source the profile file as shown above, or re-login afterwards." - echo >&2 - echo >&2 "After that, your PATH variable should start with \"/usr/local/opt/gnu-getopt/bin\"" - echo >&2 "Your current path is ${PATH}" - echo >&2 + echo """ +${COLOR_RED_ERROR} You are running ${CMDNAME} in OSX environment and ou need to install gnu commands + +Run 'brew install gnu-getopt coreutils' + +Then link the gnu-getopt to become default as suggested by brew. + +If you use bash, you should run these commands: + +echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.bash_profile +. ~/.bash_profile + +If you use zsh, you should run these commands: + +echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.zprofile +. ~/.zprofile + +Either source the profile file as shown above, or re-login afterwards. + +After that, your PATH variable should start with \"/usr/local/opt/gnu-getopt/bin\" +Your current path is ${PATH} +${COLOR_RESET} +""" else - echo >&2 "You do not have necessary tools in your path (getopt, stat, md5sum)." - echo >&2 "Please install latest/GNU version of getopt and coreutils." - echo >&2 "This can usually be done with 'apt install util-linux coreutils'" + echo """ +${COLOR_RED_ERROR} You do not have necessary tools in your path (getopt, stat, md5sum). +Please install latest/GNU version of getopt and coreutils. +This can usually be done with 'apt install util-linux coreutils' +${COLOR_RESET} +""" fi verbosity::print_info exit 1 @@ -117,12 +121,14 @@ function sanity_checks::assert_not_in_container() { return fi if [[ -f /.dockerenv ]]; then - echo >&2 - echo >&2 "You are inside the Airflow docker container!" - echo >&2 "You should only run this script from the host." - echo >&2 "Learn more about how we develop and test airflow in:" - echo >&2 "https://github.com/apache/airflow/blob/master/TESTING.rst"; - echo >&2 + echo """ +\e[31mERROR: You are inside the Airflow docker container + +You should only run this script from the host. +Learn more about how we develop and test airflow at: +https://github.com/apache/airflow/blob/master/TESTING.rst + +""" exit 1 fi } diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh b/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh index 1b37ca5..59165b2 100755 --- a/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh +++ b/scripts/ci/pre_commit/pre_commit_check_pre_commits.sh @@ -38,29 +38,35 @@ for pre_commit in ${all_pre_commits} do if ! grep -q "${pre_commit}" "${STATIC_CODE_CHECKS_FILE}"; then error="true" - >&2 echo - >&2 echo "ERROR: Pre-commit ${pre_commit} is not described in ${STATIC_CODE_CHECKS_FILE}" - >&2 echo - >&2 echo "FIX: Please add ${pre_commit} in the table in the 'Pre-commit hooks' chapter in ${STATIC_CODE_CHECKS_FILE}" - >&2 echo + echo + echo """ +${COLOR_RED_ERROR} Pre-commit ${pre_commit} is not described in ${STATIC_CODE_CHECKS_FILE} +ERROR: Pre-commit ${pre_commit} is not described in ${STATIC_CODE_CHECKS_FILE} + +FIX: Please add ${pre_commit} in the table in the 'Pre-commit hooks' chapter in ${STATIC_CODE_CHECKS_FILE} +${COLOR_RESET} +""" + echo fi + # shellcheck disable=SC2154 if [[ ! ${_breeze_allowed_static_checks} == *${pre_commit}* ]]; then error="true" - >&2 echo - >&2 echo "ERROR: Pre-commit ${pre_commit} is missing in _breeze_allowed_static_checks variable in breeze-complete" - >&2 echo - >&2 echo "FIX: Please add ${pre_commit} in the table in the '_breeze_allowed_static_checks' constant in ${AIRFLOW_SOURCES}/breeze-complete" - >&2 echo + echo """ +${COLOR_RED_ERROR}: Pre-commit ${pre_commit} is missing in _breeze_allowed_static_checks variable in breeze-complete + +FIX: Please add ${pre_commit} in the table in the '_breeze_allowed_static_checks' constant in ${AIRFLOW_SOURCES}/breeze-complete +${COLOR_RESET} +""" fi done if [[ ${error} == "true" ]]; then - >&2 echo - >&2 echo "Some pre-commits are not synchronized! Please fix the errors above!" - >&2 echo - exit 1 + echo + echo "${COLOR_RED_ERROR} Some pre-commits are not synchronized! Please fix the errors above! ${COLOR_RESET}" + echo + exit 1 else - echo - echo "All pre-commits are synchronized!" - echo + echo + echo "${COLOR_GREEN_OK} All pre-commits are synchronized! ${COLOR_RESET}" + echo fi diff --git a/scripts/ci/pre_commit/pre_commit_mermaid.sh b/scripts/ci/pre_commit/pre_commit_mermaid.sh index fc52c91..60db93e 100755 --- a/scripts/ci/pre_commit/pre_commit_mermaid.sh +++ b/scripts/ci/pre_commit/pre_commit_mermaid.sh @@ -86,9 +86,9 @@ EOF echo "Please add both files and commit them to repository" echo else - 1>&2 echo - 1>&2 echo "ERROR: Could not generate ${basename_file}.png" - 1>&2 echo + echo + echo "\e[31mERROR: Could not generate ${basename_file}.png ${COLOR_RESET}" + echo exit 1 fi else diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index 8696d56..f8f43cc 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -145,10 +145,10 @@ function get_changed_files() { CHANGED_FILES=$(git diff-tree --no-commit-id --name-only \ -r "${INCOMING_COMMIT_SHA}^" "${INCOMING_COMMIT_SHA}" || true) if [[ ${CHANGED_FILES} == "" ]]; then - >&2 echo - >&2 echo Warning! Could not find any changed files - >&2 echo Assuming that we should run all tests in this case - >&2 echo + echo + echo "${COLOR_YELLOW_WARNING}: Could not find any changed files ${COLOR_RESET}" + echo Assuming that we should run all tests in this case + echo set_outputs_run_everything_and_exit fi echo diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh index 642cfc9..a185a5a 100755 --- a/scripts/ci/static_checks/check_license.sh +++ b/scripts/ci/static_checks/check_license.sh @@ -37,7 +37,9 @@ function run_check_license() { apache/airflow:apache-rat-2020.07.10-0.13 \ --exclude-file /opt/airflow/.rat-excludes \ --d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then - echo >&2 "RAT exited abnormally" + echo + echo "${COLOR_RED_ERROR} RAT exited abnormally ${COLOR_RESET}" + echo exit 1 fi @@ -46,13 +48,14 @@ function run_check_license() { errors=$(grep -F "??" "${AIRFLOW_SOURCES}/logs/rat-results.txt") set -e if test ! -z "${errors}"; then - echo >&2 - echo >&2 "Could not find Apache license headers in the following files:" - echo >&2 "${errors}" + echo + echo "${COLOR_RED_ERROR} Could not find Apache license headers in the following files: ${COLOR_RESET}" + echo + echo "${errors}" exit 1 - echo >&2 else - echo "RAT checks passed." + echo + echo "${COLOR_GREEN_OK} RAT checks passed. ${COLOR_RESET}" echo fi } diff --git a/scripts/in_container/_in_container_script_init.sh b/scripts/in_container/_in_container_script_init.sh index 96d99e5..562de97 100755 --- a/scripts/in_container/_in_container_script_init.sh +++ b/scripts/in_container/_in_container_script_init.sh @@ -24,10 +24,12 @@ IN_CONTAINER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=scripts/in_container/_in_container_utils.sh . "${IN_CONTAINER_DIR}/_in_container_utils.sh" +in_container_set_colors + in_container_basic_sanity_check in_container_script_start -add_trap "in_container_script_end" EXIT HUP INT TERM -add_trap "in_container_clear_tmp" EXIT HUP INT TERM add_trap "in_container_fix_ownership" EXIT HUP INT TERM +add_trap "in_container_clear_tmp" EXIT HUP INT TERM +add_trap "in_container_script_end" EXIT HUP INT TERM diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 6e41f38..a5a827a 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -40,17 +40,19 @@ function add_trap() { function assert_in_container() { export VERBOSE=${VERBOSE:="false"} if [[ ! -f /.dockerenv ]]; then - echo >&2 - echo >&2 "You are not inside the Airflow docker container!" - echo >&2 "You should only run this script in the Airflow docker container as it may override your files." - echo >&2 "Learn more about how we develop and test airflow in:" - echo >&2 "https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst"; - echo >&2 + echo + echo "${COLOR_RED_ERROR} You are not inside the Airflow docker container! ${COLOR_RESET}" + echo + echo "You should only run this script in the Airflow docker container as it may override your files." + echo "Learn more about how we develop and test airflow in:" + echo "https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst"; + echo exit 1 fi } function in_container_script_start() { + OUT_FILE_PRINTED_ON_ERROR=$(mktemp) if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then set -x fi @@ -61,17 +63,23 @@ function in_container_script_end() { EXIT_CODE=$? if [[ ${EXIT_CODE} != 0 ]]; then if [[ "${PRINT_INFO_FROM_SCRIPTS=="true"}" == "true" ]] ;then - if [[ -n ${OUT_FILE_PRINTED_ON_ERROR=} ]]; then - echo " ERROR ENCOUNTERED!" + if [[ -f ${OUT_FILE_PRINTED_ON_ERROR} ]]; then + echo "###########################################################################################" echo - echo " Output:" + echo "${COLOR_BLUE} EXIT CODE: ${EXIT_CODE} in container (See above for error message). Below is the output of the last action! ${COLOR_RESET}" echo + echo "${COLOR_BLUE}*** BEGINNING OF THE LAST COMMAND OUTPUT *** ${COLOR_RESET}" cat "${OUT_FILE_PRINTED_ON_ERROR}" + echo "${COLOR_BLUE}*** END OF THE LAST COMMAND OUTPUT *** ${COLOR_RESET}" + echo + echo "${COLOR_BLUE} EXIT CODE: ${EXIT_CODE} in container. The actual error might be above the output! ${COLOR_RESET}" + echo echo "###########################################################################################" + else + echo "########################################################################################################################" + echo "${COLOR_BLUE} [IN CONTAINER] EXITING ${0} WITH EXIT CODE ${EXIT_CODE} ${COLOR_RESET}" + echo "########################################################################################################################" fi - echo "###########################################################################################" - echo " [IN CONTAINER] EXITING ${0} WITH STATUS CODE ${EXIT_CODE}" - echo "###########################################################################################" fi fi @@ -241,5 +249,24 @@ function install_released_airflow_version() { } +function in_container_set_colors() { + COLOR_BLUE=$'\e[34m' + COLOR_GREEN=$'\e[32m' + COLOR_GREEN_OK=$'\e[32mOK.' + COLOR_RED=$'\e[31m' + COLOR_RED_ERROR=$'\e[31mERROR:' + COLOR_RESET=$'\e[0m' + COLOR_YELLOW=$'\e[33m' + COLOR_YELLOW_WARNING=$'\e[33mWARNING:' + export COLOR_BLUE + export COLOR_GREEN + export COLOR_GREEN_OK + export COLOR_RED + export COLOR_RED_ERROR + export COLOR_RESET + export COLOR_YELLOW + export COLOR_YELLOW_WARNING +} + export CI=${CI:="false"} export GITHUB_ACTIONS=${GITHUB_ACTIONS:="false"} diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index 7052628..2282304 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -35,15 +35,14 @@ function check_service() { RES=$? set -e if [[ ${RES} == 0 ]]; then - echo -e " \e[32mOK.\e[0m" + echo "${COLOR_GREEN_OK} ${COLOR_RESET}" break else echo -n "." MAX_CHECK=$((MAX_CHECK - 1)) fi if [[ ${MAX_CHECK} == 0 ]]; then - echo -e " \e[31mERROR!\e[0m" - echo "Maximum number of retries while checking service. Exiting" + echo "${COLOR_RED_ERROR} Maximum number of retries while checking service. Exiting ${COLOR_RESET}" break else sleep 1 diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index a645922..7ef1e7a 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -252,9 +252,9 @@ else ${TEST_TYPE} == "Integration" ]]; then SELECTED_TESTS=("${ALL_TESTS[@]}") else - >&2 echo - >&2 echo "Wrong test type ${TEST_TYPE}" - >&2 echo + echo + echo "${COLOR_RED_ERROR} Wrong test type ${TEST_TYPE} ${COLOR_RESET}" + echo exit 1 fi diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh index 7f2be4c..bebcce8 100755 --- a/scripts/in_container/run_ci_tests.sh +++ b/scripts/in_container/run_ci_tests.sh @@ -38,46 +38,53 @@ elif [[ "${RES}" != "0" ]]; then elif [[ ${BACKEND} == "mysql" ]]; then EXTRA_ARGS="--mysql-version ${MYSQL_VERSION} " fi - - >&2 echo "***********************************************************************************************" - >&2 echo "*" - >&2 echo "* ERROR! Some tests failed, unfortunately. Those might be transient errors," - >&2 echo "* but usually you have to fix something." - >&2 echo "* See the above log for details." - >&2 echo "*" - >&2 echo "***********************************************************************************************" - >&2 echo "* You can easily reproduce the failed tests on your dev machine/" - >&2 echo "*" - >&2 echo "* When you have the source branch checked out locally:" - >&2 echo "*" - >&2 echo "* Run all tests:" - >&2 echo "*" - >&2 echo "* ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} tests" - >&2 echo "*" - >&2 echo "* Enter docker shell:" - >&2 echo "*" - >&2 echo "* ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} shell" - >&2 echo "*" + echo """ +${COLOR_RED_ERROR} +*********************************************************************************************** +* +* ERROR! Some tests failed, unfortunately. Those might be transient errors, +* but usually you have to fix something. +* See the above log for details. +* +*********************************************************************************************** +* You can easily reproduce the failed tests on your dev machine/ +* +* When you have the source branch checked out locally: +* +* Run all tests: +* +* ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} tests +* +* Enter docker shell: +* +* ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} shell +* +""" if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG=} != "" ]]; then - >&2 echo "* When you do not have sources:" - >&2 echo "*" - >&2 echo "* Run all tests:" - >&2 echo "*" - >&2 echo "* ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} tests" - >&2 echo "*" - >&2 echo "* Enter docker shell:" - >&2 echo "*" - >&2 echo "* ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} shell" - >&2 echo "*" + echo """ +* When you do not have sources: +* +* Run all tests: +* +* ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} tests +* +* Enter docker shell: +* +* ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} shell +* +""" fi - >&2 echo "*" - >&2 echo "* NOTE! Once you are in the docker shell, you can run failed test with:" - >&2 echo "*" - >&2 echo "* pytest [TEST_NAME]" - >&2 echo "*" - >&2 echo "* You can copy the test name from the output above" - >&2 echo "*" - >&2 echo "***********************************************************************************************" + echo """ +* +* NOTE! Once you are in the docker shell, you can run failed test with: +* +* pytest [TEST_NAME] +* +* You can copy the test name from the output above +* +*********************************************************************************************** +${COLOR_RESET} +""" fi MAIN_GITHUB_REPOSITORY="apache/airflow" diff --git a/scripts/in_container/run_cli_tool.sh b/scripts/in_container/run_cli_tool.sh index 706098e..6a4c67c 100755 --- a/scripts/in_container/run_cli_tool.sh +++ b/scripts/in_container/run_cli_tool.sh @@ -19,19 +19,27 @@ set -euo pipefail if [ -z "${AIRFLOW_CI_IMAGE=}" ]; then - >&2 echo "Missing environment variable AIRFLOW_CI_IMAGE" + echo + echo "${COLOR_RED_ERROR} Missing environment variable AIRFLOW_CI_IMAGE ${COLOR_RESET}" + echo exit 1 fi if [ -z "${HOST_AIRFLOW_SOURCES=}" ]; then - >&2 echo "Missing environment variable HOST_AIRFLOW_SOURCES" + echo + echo "${COLOR_RED_ERROR} Missing environment variable HOST_AIRFLOW_SOURCES ${COLOR_RESET}" + echo exit 1 fi if [ -z "${HOST_USER_ID=}" ]; then - >&2 echo "Missing environment variable HOST_USER_ID" + echo + echo "${COLOR_RED_ERROR} Missing environment variable HOST_USER_ID ${COLOR_RESET}" + echo exit 1 fi if [ -z "${HOST_GROUP_ID=}" ]; then - >&2 echo "Missing environment variable HOST_GROUP_ID" + echo + echo "${COLOR_RED_ERROR} Missing environment variable HOST_GROUP_ID ${COLOR_RESET}" + echo exit 1 fi @@ -122,7 +130,9 @@ case "${TOOL_NAME}" in COMMAND=("/usr/local/openjdk-8/bin/java" "${@}") ;; * ) - >&2 echo "Unsupported tool name: ${TOOL_NAME}" + echo + echo "${COLOR_RED_ERROR} Unsupported tool name: ${TOOL_NAME} ${COLOR_RESET}" + echo exit 1 ;; esac diff --git a/scripts/in_container/run_system_tests.sh b/scripts/in_container/run_system_tests.sh index e8c11c4..2b1181c 100755 --- a/scripts/in_container/run_system_tests.sh +++ b/scripts/in_container/run_system_tests.sh @@ -25,6 +25,8 @@ IN_CONTAINER_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) # shellcheck source=scripts/in_container/_in_container_utils.sh . "${IN_CONTAINER_DIR}/_in_container_utils.sh" +in_container_set_colors + in_container_basic_sanity_check in_container_script_start diff --git a/tests/bats/breeze/test_breeze_params.bats b/tests/bats/breeze/test_breeze_params.bats index c103d6d..92a2686 100644 --- a/tests/bats/breeze/test_breeze_params.bats +++ b/tests/bats/breeze/test_breeze_params.bats @@ -31,10 +31,7 @@ teardown() { @test "Test missing value for a parameter" { export _breeze_allowed_test_params="a b c" run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message" - assert_output " -ERROR: Allowed Test Param: [ a b c ]. Passed: ''. - -Switch to supported value with --message flag." + assert_output --regexp "Allowed Test Param: \[ a b c \]\. Passed: ''" assert_failure } @@ -43,10 +40,7 @@ Switch to supported value with --message flag." export TEST_PARAM=x echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message" - assert_output " -ERROR: Allowed Test Param: [ a b c ]. Passed: 'x'. - -Switch to supported value with --message flag." + assert_output --regexp "Allowed Test Param: \[ a b c \]\. Passed: 'x" assert_exist "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" assert_file_contains "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" "^a$" assert_failure 1 @@ -57,12 +51,7 @@ Switch to supported value with --message flag." export TEST_PARAM=x echo "x" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message" - assert_output " -ERROR: Allowed Test Param: [ a b c ]. Passed: 'x'. - -Switch to supported value with --message flag. - -Removing ${AIRFLOW_SOURCES}/.build/.TEST_PARAM. Next time you run it, it should be OK." + assert_output --regexp "Allowed Test Param: \[ a b c \]\. Passed: 'x'" assert_not_exist "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" assert_failure 1 } diff --git a/tests/bats/in_container/test_in_container.bats b/tests/bats/in_container/test_in_container.bats index fe2e0c3..8386ad2 100644 --- a/tests/bats/in_container/test_in_container.bats +++ b/tests/bats/in_container/test_in_container.bats @@ -23,6 +23,8 @@ source "/opt/bats/lib/load.bash" setup() { # shellcheck source=scripts/in_container/_in_container_utils.sh source "${AIRFLOW_SOURCES}/scripts/in_container/_in_container_utils.sh" + + in_container_set_colors } @test "test in_container" {
