This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new ebb0a97 Make ci/scripts/pre-commit Google Shell Guide compatible
(#10748)
ebb0a97 is described below
commit ebb0a97018a16efbf96ef08afe4f0bab25eb29c2
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Sep 6 20:00:54 2020 +0200
Make ci/scripts/pre-commit Google Shell Guide compatible (#10748)
Part of #10576
---
scripts/ci/pre_commit/pre_commit_bat_tests.sh | 6 ++---
.../ci/pre_commit/pre_commit_breeze_cmd_line.sh | 13 ++++++-----
.../pre_commit_build_providers_dependencies.sh | 5 ++++
.../ci/pre_commit/pre_commit_local_yml_mounts.sh | 10 ++++----
scripts/ci/pre_commit/pre_commit_mermaid.sh | 27 ++++++++++++++--------
scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh | 24 +++++++++++++------
6 files changed, 56 insertions(+), 29 deletions(-)
diff --git a/scripts/ci/pre_commit/pre_commit_bat_tests.sh
b/scripts/ci/pre_commit/pre_commit_bat_tests.sh
index 9531d5e..5807862 100755
--- a/scripts/ci/pre_commit/pre_commit_bat_tests.sh
+++ b/scripts/ci/pre_commit/pre_commit_bat_tests.sh
@@ -21,10 +21,10 @@ export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
if [[ $# -eq 0 ]]; then
- PARAMS=("tests/bats")
+ params=("tests/bats")
else
- PARAMS=("${@}")
+ params=("${@}")
fi
# shellcheck source=scripts/ci/static_checks/bats_tests.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/bats_tests.sh"
"${PARAMS[@]}"
+. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/bats_tests.sh"
"${params[@]}"
diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
index 92ec950..66955a8 100755
--- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
@@ -64,15 +64,16 @@ if (( MAX_LEN > MAX_SCREEN_WIDTH + 2 )); then
fi
BREEZE_RST_FILE="${AIRFLOW_SOURCES}/BREEZE.rst"
+readonly BREEZE_RST_FILE
-LEAD='^ \.\. START BREEZE HELP MARKER$'
-TAIL='^ \.\. END BREEZE HELP MARKER$'
+lead_marker='^ \.\. START BREEZE HELP MARKER$'
+tail_marker='^ \.\. END BREEZE HELP MARKER$'
-BEGIN_GEN=$(grep -n "${LEAD}" <"${BREEZE_RST_FILE}" | sed 's/\(.*\):.*/\1/g')
-END_GEN=$(grep -n "${TAIL}" <"${BREEZE_RST_FILE}" | sed 's/\(.*\):.*/\1/g')
-cat <(head -n "${BEGIN_GEN}" "${BREEZE_RST_FILE}") \
+beginning_of_generated_help_line_number=$(grep -n "${lead_marker}"
<"${BREEZE_RST_FILE}" | sed 's/\(.*\):.*/\1/g')
+end_beginning_of_generated_help_line_number=$(grep -n "${tail_marker}"
<"${BREEZE_RST_FILE}" | sed 's/\(.*\):.*/\1/g')
+cat <(head -n "${beginning_of_generated_help_line_number}"
"${BREEZE_RST_FILE}") \
"${TMP_FILE}" \
- <(tail -n +"${END_GEN}" "${BREEZE_RST_FILE}") \
+ <(tail -n +"${end_beginning_of_generated_help_line_number}"
"${BREEZE_RST_FILE}") \
>"${TMP_OUTPUT}"
mv "${TMP_OUTPUT}" "${BREEZE_RST_FILE}"
diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
index e5235e4..3aa606b 100755
--- a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
+++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
@@ -18,13 +18,18 @@
set -euo pipefail
PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+readonly PRE_COMMIT_DIR
+
AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
+readonly AIRFLOW_SOURCES
cd "${AIRFLOW_SOURCES}" || exit 1
+
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
PYTHONPATH="$(pwd)"
export PYTHONPATH
+
find airflow/providers -name '*.py' -print0 | \
xargs -0 python3 tests/build_provider_packages_dependencies.py \
--provider-dependencies-file "airflow/providers/dependencies.json" \
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
index 6e8b0a7..dbf82b7 100755
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -22,20 +22,22 @@ export SKIP_CHECK_REMOTE_IMAGE="true"
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
TMP_OUTPUT=$(mktemp)
+readonly TMP_OUTPUT
# Remove temp file if it's hanging around
traps::add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM
LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml"
+readonly LOCAL_YML_FILE
-LEAD=' # START automatically generated volumes from LOCAL_MOUNTS in
_local_mounts.sh'
-TAIL=' # END automatically generated volumes from LOCAL_MOUNTS in
_local_mounts.sh'
+lead_marker=' # START automatically generated volumes from LOCAL_MOUNTS
in _local_mounts.sh'
+tail_marker=' # END automatically generated volumes from LOCAL_MOUNTS in
_local_mounts.sh'
local_mounts::generate_local_mounts_list " - ../../../"
-sed "/$LEAD/q" "${LOCAL_YML_FILE}" > "${TMP_OUTPUT}"
+sed "/$lead_marker/q" "${LOCAL_YML_FILE}" > "${TMP_OUTPUT}"
printf '%s\n' "${LOCAL_MOUNTS[@]}" >> "${TMP_OUTPUT}"
-sed -ne "/$TAIL/,\$ p" "${LOCAL_YML_FILE}" >> "${TMP_OUTPUT}"
+sed -ne "/$tail_marker/,\$ p" "${LOCAL_YML_FILE}" >> "${TMP_OUTPUT}"
mv "${TMP_OUTPUT}" "${LOCAL_YML_FILE}"
diff --git a/scripts/ci/pre_commit/pre_commit_mermaid.sh
b/scripts/ci/pre_commit/pre_commit_mermaid.sh
index 4485cce..ebef846 100755
--- a/scripts/ci/pre_commit/pre_commit_mermaid.sh
+++ b/scripts/ci/pre_commit/pre_commit_mermaid.sh
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
-export NO_TERMINAL_OUTPUT_FROM_SCRIPTS="true"
+export PRINT_INFO_FROM_SCRIPTS="false"
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -28,29 +28,38 @@ if ! command -v npm; then
exit 0
fi
-tmp_file="${CACHE_TMP_FILE_DIR}/tmp.mermaid"
+TMP_FILE="${CACHE_TMP_FILE_DIR}/tmp.mermaid"
+readonly TMP_FILE
+
cd "${AIRFLOW_SOURCES}"
MERMAID_INSTALLATION_DIR="${AIRFLOW_SOURCES}/.build/mermaid/"
+readonly MERMAID_INSTALLATION_DIR
+
MERMAID_CONFIG_FILE="${MERMAID_INSTALLATION_DIR}/mermaid-config.json"
+readonly MERMAID_CONFIG_FILE
+
MERMAID_CLI="${MERMAID_INSTALLATION_DIR}/node_modules/.bin/mmdc"
+readonly MERMAID_CLI
+
export NODE_VIRTUAL_ENV="${MERMAID_INSTALLATION_DIR}"
+readonly NODE_VIRTUAL_ENV
if [[ -f "${MERMAID_CLI}" ]]; then
- MERMAID_INSTALLED="true"
+ mermaid_installed="true"
else
- MERMAID_INSTALLED="false"
+ mermaid_installed="false"
fi
# shellcheck disable=SC2064
-traps::add_trap "rm -rf '${tmp_file}'" EXIT HUP INT TERM
+traps::add_trap "rm -rf '${TMP_FILE}'" EXIT HUP INT TERM
for file in "${@}"
do
basename_file=${AIRFLOW_SOURCES}/"$(dirname "${file}")/$(basename
"${file}" .mermaid)"
md5sum_file="${basename_file}.md5"
if ! diff "${md5sum_file}" <(md5sum "${file}"); then
- if [[ ${MERMAID_INSTALLED} != "true" ]]; then
+ if [[ ${mermaid_installed} != "true" ]]; then
echo "Installing mermaid"
mkdir -p "${MERMAID_INSTALLATION_DIR}/node_modules"
pushd "${MERMAID_INSTALLATION_DIR}"
@@ -61,7 +70,7 @@ do
"themeCSS": ".label foreignObject { overflow: visible; }"
}
EOF
- MERMAID_INSTALLED="true"
+ mermaid_installed="true"
popd
fi
echo "Running generation for ${file}"
@@ -70,11 +79,11 @@ EOF
# unfortunately mermaid does not handle well multiline comments and we
need licence comment
# Stripping them manually :(. Multiline comments are coming in the
future
# https://github.com/mermaid-js/mermaid/issues/1249
- grep -v "^%%" <"${file}" > "${tmp_file}"
+ grep -v "^%%" <"${file}" > "${TMP_FILE}"
mkdir -p "${MERMAID_INSTALLATION_DIR}"
"${MERMAID_CLI}" \
- -i "${tmp_file}" \
+ -i "${TMP_FILE}" \
-w 2048 \
-o "${basename_file}.png" \
-c "${MERMAID_CONFIG_FILE}"
diff --git a/scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh
b/scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh
index ede294c..e741f9e 100755
--- a/scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh
+++ b/scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh
@@ -16,26 +16,36 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
+export PRINT_INFO_FROM_SCRIPTS="false"
+readonly PRINT_INFO_FROM_SCRIPTS
+
PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+readonly PRE_COMMIT_DIR
+
AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
+readonly AIRFLOW_SOURCES
+
cd "${AIRFLOW_SOURCES}" || exit 1
-export PRINT_INFO_FROM_SCRIPTS="false"
TMP_FILE=$(mktemp)
+readonly TMP_FILE
+
TMP_OUTPUT=$(mktemp)
+readonly TMP_OUTPUT
find "licenses" -type f -exec echo " " {} \; | LC_ALL=C sort >>"${TMP_FILE}"
SETUP_CFG_FILE="${AIRFLOW_SOURCES}/setup.cfg"
+readonly SETUP_CFG_FILE
-LEAD='^# Start of licenses generated automatically$'
-TAIL='^# End of licences generated automatically$'
+lead_marker='^# Start of licenses generated automatically$'
+tail_marker='^# End of licences generated automatically$'
-BEGIN_GEN=$(grep -n "${LEAD}" <"${SETUP_CFG_FILE}" | sed 's/\(.*\):.*/\1/g')
-END_GEN=$(grep -n "${TAIL}" <"${SETUP_CFG_FILE}" | sed 's/\(.*\):.*/\1/g')
-cat <(head -n "${BEGIN_GEN}" "${SETUP_CFG_FILE}") \
+beginning_of_generated_help_line_number=$(grep -n "${lead_marker}"
<"${SETUP_CFG_FILE}" | sed 's/\(.*\):.*/\1/g')
+end_of_generated_help_line_number=$(grep -n "${tail_marker}"
<"${SETUP_CFG_FILE}" | sed 's/\(.*\):.*/\1/g')
+cat <(head -n "${beginning_of_generated_help_line_number}"
"${SETUP_CFG_FILE}") \
"${TMP_FILE}" \
- <(tail -n +"${END_GEN}" "${SETUP_CFG_FILE}") \
+ <(tail -n +"${end_of_generated_help_line_number}" "${SETUP_CFG_FILE}") \
>"${TMP_OUTPUT}"
mv "${TMP_OUTPUT}" "${SETUP_CFG_FILE}"
