feluelle commented on a change in pull request #10368:
URL: https://github.com/apache/airflow/pull/10368#discussion_r472848834
##########
File path: scripts/ci/libraries/_push_pull_remove_images.sh
##########
@@ -33,41 +33,43 @@ function pull_image_if_needed() {
echo
echo "Pulling the image ${IMAGE_TO_PULL}"
echo
- verbose_docker pull "${IMAGE_TO_PULL}" | tee -a "${OUTPUT_LOG}"
+ # need eto be explicitly verbose in order to have pre-commit spinner
working
Review comment:
```suggestion
# need to be explicitly verbose in order to have pre-commit spinner
working
```
##########
File path: scripts/ci/in_container/_in_container_utils.sh
##########
@@ -39,17 +39,19 @@ function in_container_script_end() {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
- if [[ -n ${OUT_FILE:=} ]]; then
- echo " ERROR ENCOUNTERED!"
- echo
- echo " Output:"
- echo
- cat "${OUT_FILE}"
+ if [[ "${PRINT_INFO_FROM_SCRIPTS=="ture"}" == "true" ]] ;then
Review comment:
```suggestion
if [[ "${PRINT_INFO_FROM_SCRIPTS=="true"}" == "true" ]] ;then
```
##########
File path: scripts/ci/libraries/_push_pull_remove_images.sh
##########
@@ -103,75 +111,122 @@ function pull_prod_images_if_needed() {
echo
echo "Force pull base image ${PYTHON_BASE_IMAGE}"
echo
- if [[ ${PULL_PYTHON_BASE_IMAGES_FROM_CACHE:="true"} == "true" ]];
then
- pull_image_possibly_from_cache "${PYTHON_BASE_IMAGE}"
"${CACHED_PYTHON_BASE_IMAGE}"
+ if [[ ${USE_GITHUB_REGISTRY:="false"} == "true" ]]; then
+ PYTHON_TAG_SUFFIX=""
+ if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then
+ PYTHON_TAG_SUFFIX="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ fi
+ pull_image_github_dockerhub "${PYTHON_BASE_IMAGE}"
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
else
- verbose_docker pull "${PYTHON_BASE_IMAGE}" | tee -a
"${OUTPUT_LOG}"
+ # need eto be explicitly verbose in order to have pre-commit
spinner working
+ # No aliases in pre-commit non-interactive shells :(
+ verbose_docker pull "${PYTHON_BASE_IMAGE}"
fi
echo
fi
# "Build" segment of production image
- pull_image_possibly_from_cache "${AIRFLOW_PROD_BUILD_IMAGE}"
"${CACHED_AIRFLOW_PROD_BUILD_IMAGE}"
- # we never pull the main segment of production image - we always build
it locally = this is
- # usually very fast this way and it is much nicer for rebuilds and
development
+ pull_image_github_dockerhub "${AIRFLOW_PROD_BUILD_IMAGE}"
"${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ # "Main" segment of production image
+ pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}"
"${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
}
-# Pushes Ci image and it's manifest to the registry. In case the image was
taken from cache registry
-# it is pushed to the cache, not to the main registry. Manifest is only pushed
to the main registry
-function push_ci_image() {
- if [[ ${CACHED_AIRFLOW_CI_IMAGE:=} != "" ]]; then
- verbose_docker tag "${AIRFLOW_CI_IMAGE}" "${CACHED_AIRFLOW_CI_IMAGE}"
- IMAGE_TO_PUSH="${CACHED_AIRFLOW_CI_IMAGE}"
- else
- IMAGE_TO_PUSH="${AIRFLOW_CI_IMAGE}"
+# Pushes Ci images and the manifest to the registry in DockerHub.
+function push_ci_images_to_dockerhub() {
+ docker push "${AIRFLOW_CI_IMAGE}"
+ docker tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}"
"${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
+ docker push "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
+ if [[ -n ${DEFAULT_CI_IMAGE:=""} ]]; then
+ # Only push default image to DockerHub registry if it is defined and
Review comment:
```suggestion
# Only push default image to DockerHub registry if it is defined
```
##########
File path: scripts/ci/libraries/_verbosity.sh
##########
@@ -28,58 +27,87 @@ function check_verbose_setup {
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker
commands run will be
# printed before execution
function verbose_docker {
- if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]];
then
- # do not print echo if VERBOSE_COMMAND is set (set -x does it already)
- echo "docker" "${@}"
+ if [[ ${VERBOSE:="false"} == "true" && \
+ # do not print echo if VERBOSE_COMMAND is set (set -x does it already)
+ ${VERBOSE_COMMANDS:=} != "true" && \
+ # And when generally printing info is disabled
+ ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then
+ >&2 echo "docker" "${@}"
+ fi
+ if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then
+ docker "${@}" >>"${OUTPUT_LOG}" 2>&1
+ else
+ docker "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
+ fi
+ EXIT_CODE="$?"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
+ # This way in the output log we only see the most recent failed
command and what was echoed before
+ rm -f "${OUTPUT_LOG}"
fi
- docker "${@}"
+ return "${EXIT_CODE}"
}
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all helm
commands run will be
# printed before execution
function verbose_helm {
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]];
then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
- echo "helm" "${@}"
+ >&2 echo "helm" "${@}"
+ fi
+ helm "${@}" | tee -a "${OUTPUT_LOG}"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
+ rm -f "${OUTPUT_LOG}"
fi
- helm "${@}"
}
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kubectl
commands run will be
# printed before execution
function verbose_kubectl {
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]];
then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
- echo "kubectl" "${@}"
+ >&2 echo "kubectl" "${@}"
+ fi
+ kubectl "${@}" | tee -a "${OUTPUT_LOG}"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
+ rm -f "${OUTPUT_LOG}"
fi
- kubectl "${@}"
}
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kind
commands run will be
# printed before execution
function verbose_kind {
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]];
then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
- echo "kind" "${@}"
+ >&2 echo "kind" "${@}"
fi
+ # kind outputs nice output on terminal.
kind "${@}"
}
-
-# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker
commands run will be
-# printed before execution
-function verbose_docker_hide_output_on_success {
- if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]];
then
- # do not print echo if VERBOSE_COMMAND is set (set -x does it already)
- echo "docker" "${@}"
- fi
- docker "${@}" >>"${OUTPUT_LOG}" 2>&1
-}
-
-
# Prints verbose information in case VERBOSE variable is set
function print_info() {
- if [[ ${VERBOSE:="false"} == "true" ]]; then
+ if [[ ${VERBOSE:="false"} == "true" && ${PRINT_INFO_FROM_SCRIPTS} ==
"true" ]]; then
echo "$@"
fi
}
+
+function set_verbosity() {
+ # whether verbose output should be produced
+ export VERBOSE=${VERBOSE:="false"}
+
+ # whether every bash statement should be printed as they are executed
+ export VERBOSE_COMMANDS=${VERBOSE_COMMANDS:="false"}
+
+ # whether the output from script should be printed at all
+ export PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS:="true"}
+}
+
+set_verbosity
+
+alias docker=verbose_docker
+alias kubectl=verbose_kubectl
+alias helm=verbose_helm
+alias kind=verbose_kind
Review comment:
For how long do these alias' exist? I mean if you simply run docker
commands after a breeze session. Does it still use the verbose version?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
