This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 84c2379d6dcc28b86eda9854b9c70835bb2b1180 Author: Omair Khan <[email protected]> AuthorDate: Tue Sep 29 15:29:06 2020 +0530 in_container bats pre-commit hook and updated bats-tests hook (#11179) (cherry picked from commit 68e0eb6976ecc8a31092e68a33b9687ea3e9f868) --- .dockerignore | 1 + .pre-commit-config.yaml | 9 ++++++- BREEZE.rst | 6 ++--- Dockerfile.ci | 31 ++++++++++++++++++++++ STATIC_CODE_CHECKS.rst | 8 ++++++ breeze-complete | 1 + .../pre_commit_in_container_bats_test.sh} | 23 ++++++++-------- scripts/ci/static_checks/bats_tests.sh | 19 ++++++++++--- .../{bats_tests.sh => in_container_bats_tests.sh} | 24 ++++++++++++----- .../test_in_container.bats} | 14 +++++++--- tests/bats/test_empty_test.bats | 1 + 11 files changed, 109 insertions(+), 28 deletions(-) diff --git a/.dockerignore b/.dockerignore index 7d29561..c914d56 100644 --- a/.dockerignore +++ b/.dockerignore @@ -38,6 +38,7 @@ # Add those folders to the context so that they are available in the CI container !scripts/in_container !scripts/docker +!scripts/ci/dockerfiles/bats # Add tests and kubernetes_tests to context. !tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e7cc4d..541c480 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -308,6 +308,12 @@ repos: pass_filenames: false require_serial: true additional_dependencies: ['pyyaml'] + - id: bats-in-container-tests + name: Run in container bats tests + language: system + entry: "./scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh" + files: ^tests/bats/in_container/.*.bats$|^scripts/in_container/.*sh + pass_filenames: false - id: mypy name: Run mypy language: system @@ -325,7 +331,8 @@ repos: name: Run BATS bash tests for changed bash files language: system entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh" - files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|.bats$ + files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$ + exclude: ^tests/bats/in_container/.*bats$|^scripts/in_container/.*sh$ pass_filenames: false - id: pre-commit-descriptions name: Check if pre-commits are described diff --git a/BREEZE.rst b/BREEZE.rst index 9cf824b..86e8e88 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1732,9 +1732,9 @@ This is the current syntax for `./breeze <./breeze>`_: Run selected static checks for currently changed files. You should specify static check that you would like to run or 'all' to run all checks. One of: - all airflow-config-yaml base-operator bats-tests black build check-apache-license - check-builtin-literals check-executables-have-shebangs check-hooks-apply - check-integrations check-merge-conflict check-xml debug-statements + all airflow-config-yaml base-operator bats-tests bats-in-container-tests black build + check-apache-license check-builtin-literals check-executables-have-shebangs + check-hooks-apply check-integrations check-merge-conflict check-xml debug-statements detect-private-key doctoc dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 forbid-tabs helm-lint incorrect-use-of-LoggingMixin insert-license language-matters lint-dockerfile lint-openapi mixed-line-ending mypy diff --git a/Dockerfile.ci b/Dockerfile.ci index 37c405d..8ea42d3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -164,6 +164,35 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \ ARG PIP_DEPENDENCIES_EPOCH_NUMBER="4" ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER} +# Install BATS and its dependencies for "in container" tests +ARG BATS_VERSION="0.4.0" +ARG BATS_SUPPORT_VERSION="0.3.0" +ARG BATS_ASSERT_VERSION="2.0.0" +ARG BATS_FILE_VERSION="0.2.0" + +RUN curl -sSL https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz -o /tmp/bats.tgz \ + && tar -zxf /tmp/bats.tgz -C /tmp \ + && /bin/bash /tmp/bats-core-${BATS_VERSION}/install.sh /opt/bats && rm -rf + +RUN mkdir -p /opt/bats/lib/bats-support \ + && curl -sSL https://github.com/bats-core/bats-support/archive/v${BATS_SUPPORT_VERSION}.tar.gz -o /tmp/bats-support.tgz \ + && tar -zxf /tmp/bats-support.tgz -C /opt/bats/lib/bats-support --strip 1 && rm -rf /tmp/* + +RUN mkdir -p /opt/bats/lib/bats-assert \ + && curl -sSL https://github.com/bats-core/bats-assert/archive/v${BATS_ASSERT_VERSION}.tar.gz -o /tmp/bats-assert.tgz \ + && tar -zxf /tmp/bats-assert.tgz -C /opt/bats/lib/bats-assert --strip 1 && rm -rf /tmp/* + +RUN mkdir -p /opt/bats/lib/bats-file \ + && curl -sSL https://github.com/bats-core/bats-file/archive/v${BATS_FILE_VERSION}.tar.gz -o /tmp/bats-file.tgz \ + && tar -zxf /tmp/bats-file.tgz -C /opt/bats/lib/bats-file --strip 1 && rm -rf /tmp/* + +RUN echo "export PATH=/opt/bats/bin:${PATH}" >> /root/.bashrc + +# Additional scripts for managing BATS addons +COPY scripts/ci/dockerfiles/bats/load.bash /opt/bats/lib/ +RUN chmod a+x /opt/bats/lib/load.bash + + # Optimizing installation of Cassandra driver # Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes ARG CASS_DRIVER_NO_CYTHON="1" @@ -266,6 +295,8 @@ RUN chmod a+x /entrypoint # add it with ! in .dockerignore next to the airflow, test etc. directories there COPY . ${AIRFLOW_SOURCES}/ + + # Install autocomplete for airflow RUN register-python-argcomplete airflow >> ~/.bashrc diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index a8ce69a..b16ac06 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -54,6 +54,14 @@ require Breeze Docker images to be installed locally: ----------------------------------- ---------------------------------------------------------------- ------------ ``build`` Builds image for mypy, flake8. * ----------------------------------- ---------------------------------------------------------------- ------------ +``bats-in-container-tests`` Run in Breeze container bats tests * +----------------------------------- ---------------------------------------------------------------- ------------ +``black`` Runs Black (the uncompromising Python code formatter) +----------------------------------- ---------------------------------------------------------------- ------------ +``build`` Builds image for mypy, pylint, flake8. * +----------------------------------- ---------------------------------------------------------------- ------------ +``build-providers-dependencies`` Regenerates the json file with cross-provider dependencies +----------------------------------- ---------------------------------------------------------------- ------------ ``check-apache-license`` Checks compatibility with Apache License requirements. ----------------------------------- ---------------------------------------------------------------- ------------ ``check-builtin-literals`` Require literal syntax when initializing Python builtin types diff --git a/breeze-complete b/breeze-complete index 7ac46c8..0ec7bab 100644 --- a/breeze-complete +++ b/breeze-complete @@ -68,6 +68,7 @@ all airflow-config-yaml base-operator bats-tests +bats-in-container-tests black build check-apache-license diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh similarity index 64% copy from scripts/ci/static_checks/bats_tests.sh copy to scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh index 2066278..b699a54 100755 --- a/scripts/ci/static_checks/bats_tests.sh +++ b/scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh @@ -15,15 +15,16 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -function run_bats_tests() { - if [[ "${#@}" == "0" ]]; then - docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow:bats-2020.09.05-1.2.1 --tap -r /airflow/tests/bats - else - docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow:bats-2020.09.05-1.2.1 --tap "${@}" - fi -} +export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} +export REMEMBER_LAST_ANSWER="true" +export PRINT_INFO_FROM_SCRIPTS="false" +export SKIP_CHECK_REMOTE_IMAGE="true" -run_bats_tests "$@" +if [[ $# -eq 0 ]]; then + params=("tests/bats/in_container/") +else + params=("${@}") +fi + +# shellcheck source=scripts/ci/static_checks/in_container_bats_tests.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/in_container_bats_tests.sh" "${params[@]}" diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/bats_tests.sh index 2066278..72330e5 100755 --- a/scripts/ci/static_checks/bats_tests.sh +++ b/scripts/ci/static_checks/bats_tests.sh @@ -16,13 +16,26 @@ # specific language governing permissions and limitations # under the License. # shellcheck source=scripts/ci/libraries/_script_init.sh + +####################################################################################################### +# Runs tests for bash scripts in a docker container. This script is the entrypoint for the bats-tests +# pre-commit hook where it runs all the bats tests (exluding in container tests). +######################################################################################################## function run_bats_tests() { - if [[ "${#@}" == "0" ]]; then + local bats_scripts=() + for script in "$@" + do + if [[ $script =~ \bats$ ]]; + then + bats_scripts+=( "$script" ) + fi + done + if [[ "${#@}" == "0" || "${#bats_scripts[@]}" != "${#@}" ]]; then docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow:bats-2020.09.05-1.2.1 --tap -r /airflow/tests/bats + apache/airflow:bats-2020.09.05-1.2.1 --tap /airflow/tests/bats/ else docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow:bats-2020.09.05-1.2.1 --tap "${@}" + apache/airflow:bats-2020.09.05-1.2.1 --tap "${bats_scripts[@]}" fi } diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/in_container_bats_tests.sh old mode 100755 new mode 100644 similarity index 60% copy from scripts/ci/static_checks/bats_tests.sh copy to scripts/ci/static_checks/in_container_bats_tests.sh index 2066278..a7c0121 --- a/scripts/ci/static_checks/bats_tests.sh +++ b/scripts/ci/static_checks/in_container_bats_tests.sh @@ -16,14 +16,26 @@ # specific language governing permissions and limitations # under the License. # shellcheck source=scripts/ci/libraries/_script_init.sh -function run_bats_tests() { +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +function run_in_container_bats_tests() { if [[ "${#@}" == "0" ]]; then - docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow:bats-2020.09.05-1.2.1 --tap -r /airflow/tests/bats + docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint "/opt/bats/bin/bats" \ + "-v" "$(pwd):/airflow" \ + "${AIRFLOW_CI_IMAGE}" \ + --tap "tests/bats/in_container/" else - docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow:bats-2020.09.05-1.2.1 --tap "${@}" + docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint "/opt/bats/bin/bats" \ + "-v" "$(pwd):/airflow" \ + "${AIRFLOW_CI_IMAGE}" \ + --tap "${@}" fi } -run_bats_tests "$@" +build_images::prepare_ci_build + +build_images::rebuild_ci_image_if_needed + +run_in_container_bats_tests "$@" diff --git a/tests/bats/test_empty_test.bats b/tests/bats/in_container/test_in_container.bats similarity index 73% copy from tests/bats/test_empty_test.bats copy to tests/bats/in_container/test_in_container.bats index 2a4f680..fe2e0c3 100644 --- a/tests/bats/test_empty_test.bats +++ b/tests/bats/in_container/test_in_container.bats @@ -1,5 +1,6 @@ #!/usr/bin/env bats + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,10 +17,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# shellcheck disable=SC1091 +source "/opt/bats/lib/load.bash" -@test "empty test" { - load bats_utils +setup() { +# shellcheck source=scripts/in_container/_in_container_utils.sh + source "${AIRFLOW_SOURCES}/scripts/in_container/_in_container_utils.sh" +} - run pwd - assert_success +@test "test in_container" { + run assert_in_container + assert [ $status -eq 0 ] } diff --git a/tests/bats/test_empty_test.bats b/tests/bats/test_empty_test.bats index 2a4f680..2830014 100644 --- a/tests/bats/test_empty_test.bats +++ b/tests/bats/test_empty_test.bats @@ -17,6 +17,7 @@ # specific language governing permissions and limitations # under the License. + @test "empty test" { load bats_utils
