This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 68e0eb6 in_container bats pre-commit hook and updated bats-tests hook
(#11179)
68e0eb6 is described below
commit 68e0eb6976ecc8a31092e68a33b9687ea3e9f868
Author: Omair Khan <[email protected]>
AuthorDate: Tue Sep 29 15:29:06 2020 +0530
in_container bats pre-commit hook and updated bats-tests hook (#11179)
---
.dockerignore | 1 +
.pre-commit-config.yaml | 9 ++++++-
BREEZE.rst | 24 ++++++++---------
Dockerfile.ci | 31 ++++++++++++++++++++++
STATIC_CODE_CHECKS.rst | 2 ++
breeze-complete | 1 +
.../pre_commit_in_container_bats_test.sh} | 23 ++++++++--------
scripts/ci/static_checks/bats_tests.sh | 19 ++++++++++---
.../{bats_tests.sh => in_container_bats_tests.sh} | 24 ++++++++++++-----
.../test_in_container.bats} | 14 +++++++---
tests/bats/test_empty_test.bats | 1 +
11 files changed, 112 insertions(+), 37 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index e390df9..412339f 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -38,6 +38,7 @@
# Add those folders to the context so that they are available in the CI
container
!scripts/in_container
!scripts/docker
+!scripts/ci/dockerfiles/bats
# Add backport packages to the context
!backport_packages
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index cc28653..3c35c2a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -381,6 +381,12 @@ repos:
pass_filenames: false
require_serial: true
additional_dependencies: ['pyyaml']
+ - id: bats-in-container-tests
+ name: Run in container bats tests
+ language: system
+ entry: "./scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh"
+ files: ^tests/bats/in_container/.*.bats$|^scripts/in_container/.*sh
+ pass_filenames: false
- id: mypy
name: Run mypy
language: system
@@ -406,7 +412,8 @@ repos:
name: Run BATS bash tests for changed bash files
language: system
entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
- files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|.bats$
+ files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$
+ exclude: ^tests/bats/in_container/.*bats$|^scripts/in_container/.*sh$
pass_filenames: false
- id: stylelint
name: stylelint
diff --git a/BREEZE.rst b/BREEZE.rst
index 396d5b6..8a19830 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1857,18 +1857,18 @@ This is the current syntax for `./breeze <./breeze>`_:
Run selected static checks for currently changed files. You should
specify static check that
you would like to run or 'all' to run all checks. One of:
- all all-but-pylint airflow-config-yaml base-operator
bats-tests black build
- build-providers-dependencies check-apache-license
check-builtin-literals
- check-executables-have-shebangs check-hooks-apply
check-integrations
- check-merge-conflict check-xml consistent-pylint
daysago-import-check
- debug-statements detect-private-key doctoc
dont-use-safe-filter end-of-file-fixer
- fix-encoding-pragma flake8 forbid-tabs helm-lint
incorrect-use-of-LoggingMixin
- insert-license isort language-matters lint-dockerfile
lint-openapi mermaid
- mixed-line-ending mypy no-relative-imports
pre-commit-descriptions
- provide-create-sessions pydevd pydocstyle pylint pylint-tests
python-no-log-warn
- restrict-start_date rst-backticks setup-order shellcheck
sort-in-the-wild stylelint
- trailing-whitespace update-breeze-file update-extras
update-local-yml-file
- update-setup-cfg-file yamllint
+ all all-but-pylint airflow-config-yaml base-operator
bats-tests
+ bats-in-container-tests black build
build-providers-dependencies
+ check-apache-license check-builtin-literals
check-executables-have-shebangs
+ check-hooks-apply check-integrations check-merge-conflict
check-xml
+ consistent-pylint daysago-import-check debug-statements
detect-private-key doctoc
+ dont-use-safe-filter end-of-file-fixer fix-encoding-pragma
flake8 forbid-tabs
+ helm-lint incorrect-use-of-LoggingMixin insert-license isort
language-matters
+ lint-dockerfile lint-openapi mermaid mixed-line-ending mypy
no-relative-imports
+ pre-commit-descriptions provide-create-sessions pydevd
pydocstyle pylint
+ pylint-tests python-no-log-warn restrict-start_date
rst-backticks setup-order
+ shellcheck sort-in-the-wild stylelint trailing-whitespace
update-breeze-file
+ update-extras update-local-yml-file update-setup-cfg-file
yamllint
You can pass extra arguments including options to to the pre-commit
framework as
<EXTRA_ARGS> passed after --. For example:
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 21e61e2..7c270f1 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -164,6 +164,35 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \
ARG PIP_DEPENDENCIES_EPOCH_NUMBER="3"
ENV PIP_DEPENDENCIES_EPOCH_NUMBER=${PIP_DEPENDENCIES_EPOCH_NUMBER}
+# Install BATS and its dependencies for "in container" tests
+ARG BATS_VERSION="0.4.0"
+ARG BATS_SUPPORT_VERSION="0.3.0"
+ARG BATS_ASSERT_VERSION="2.0.0"
+ARG BATS_FILE_VERSION="0.2.0"
+
+RUN curl -sSL
https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz -o
/tmp/bats.tgz \
+ && tar -zxf /tmp/bats.tgz -C /tmp \
+ && /bin/bash /tmp/bats-core-${BATS_VERSION}/install.sh /opt/bats && rm -rf
+
+RUN mkdir -p /opt/bats/lib/bats-support \
+ && curl -sSL
https://github.com/bats-core/bats-support/archive/v${BATS_SUPPORT_VERSION}.tar.gz
-o /tmp/bats-support.tgz \
+ && tar -zxf /tmp/bats-support.tgz -C /opt/bats/lib/bats-support --strip 1
&& rm -rf /tmp/*
+
+RUN mkdir -p /opt/bats/lib/bats-assert \
+ && curl -sSL
https://github.com/bats-core/bats-assert/archive/v${BATS_ASSERT_VERSION}.tar.gz
-o /tmp/bats-assert.tgz \
+ && tar -zxf /tmp/bats-assert.tgz -C /opt/bats/lib/bats-assert --strip 1 &&
rm -rf /tmp/*
+
+RUN mkdir -p /opt/bats/lib/bats-file \
+ && curl -sSL
https://github.com/bats-core/bats-file/archive/v${BATS_FILE_VERSION}.tar.gz -o
/tmp/bats-file.tgz \
+ && tar -zxf /tmp/bats-file.tgz -C /opt/bats/lib/bats-file --strip 1 && rm
-rf /tmp/*
+
+RUN echo "export PATH=/opt/bats/bin:${PATH}" >> /root/.bashrc
+
+# Additional scripts for managing BATS addons
+COPY scripts/ci/dockerfiles/bats/load.bash /opt/bats/lib/
+RUN chmod a+x /opt/bats/lib/load.bash
+
+
# Optimizing installation of Cassandra driver
# Speeds up building the image - cassandra driver without CYTHON saves around
10 minutes
ARG CASS_DRIVER_NO_CYTHON="1"
@@ -264,6 +293,8 @@ RUN chmod a+x /entrypoint
# add it with ! in .dockerignore next to the airflow, test etc. directories
there
COPY . ${AIRFLOW_SOURCES}/
+
+
# Install autocomplete for airflow
RUN register-python-argcomplete airflow >> ~/.bashrc
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 55f7000..f19cf7a 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -52,6 +52,8 @@ require Breeze Docker images to be installed locally:
-----------------------------------
---------------------------------------------------------------- ------------
``bats-tests`` Runs BATS bash unit tests
-----------------------------------
---------------------------------------------------------------- ------------
+``bats-in-container-tests`` Run in Breeze container bats tests
*
+-----------------------------------
---------------------------------------------------------------- ------------
``black`` Runs Black (the uncompromising Python
code formatter)
-----------------------------------
---------------------------------------------------------------- ------------
``build`` Builds image for mypy, pylint, flake8.
*
diff --git a/breeze-complete b/breeze-complete
index 29a5604..0591a0f 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -69,6 +69,7 @@ all-but-pylint
airflow-config-yaml
base-operator
bats-tests
+bats-in-container-tests
black
build
build-providers-dependencies
diff --git a/scripts/ci/static_checks/bats_tests.sh
b/scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh
similarity index 64%
copy from scripts/ci/static_checks/bats_tests.sh
copy to scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh
index 2066278..b699a54 100755
--- a/scripts/ci/static_checks/bats_tests.sh
+++ b/scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh
@@ -15,15 +15,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-function run_bats_tests() {
- if [[ "${#@}" == "0" ]]; then
- docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow:bats-2020.09.05-1.2.1 --tap -r /airflow/tests/bats
- else
- docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow:bats-2020.09.05-1.2.1 --tap "${@}"
- fi
-}
+export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"}
+export REMEMBER_LAST_ANSWER="true"
+export PRINT_INFO_FROM_SCRIPTS="false"
+export SKIP_CHECK_REMOTE_IMAGE="true"
-run_bats_tests "$@"
+if [[ $# -eq 0 ]]; then
+ params=("tests/bats/in_container/")
+else
+ params=("${@}")
+fi
+
+# shellcheck source=scripts/ci/static_checks/in_container_bats_tests.sh
+. "$( dirname "${BASH_SOURCE[0]}"
)/../static_checks/in_container_bats_tests.sh" "${params[@]}"
diff --git a/scripts/ci/static_checks/bats_tests.sh
b/scripts/ci/static_checks/bats_tests.sh
index 2066278..72330e5 100755
--- a/scripts/ci/static_checks/bats_tests.sh
+++ b/scripts/ci/static_checks/bats_tests.sh
@@ -16,13 +16,26 @@
# specific language governing permissions and limitations
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
+
+#######################################################################################################
+# Runs tests for bash scripts in a docker container. This script is the
entrypoint for the bats-tests
+# pre-commit hook where it runs all the bats tests (exluding in container
tests).
+########################################################################################################
function run_bats_tests() {
- if [[ "${#@}" == "0" ]]; then
+ local bats_scripts=()
+ for script in "$@"
+ do
+ if [[ $script =~ \bats$ ]];
+ then
+ bats_scripts+=( "$script" )
+ fi
+ done
+ if [[ "${#@}" == "0" || "${#bats_scripts[@]}" != "${#@}" ]]; then
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow:bats-2020.09.05-1.2.1 --tap -r /airflow/tests/bats
+ apache/airflow:bats-2020.09.05-1.2.1 --tap /airflow/tests/bats/
else
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow:bats-2020.09.05-1.2.1 --tap "${@}"
+ apache/airflow:bats-2020.09.05-1.2.1 --tap "${bats_scripts[@]}"
fi
}
diff --git a/scripts/ci/static_checks/bats_tests.sh
b/scripts/ci/static_checks/in_container_bats_tests.sh
old mode 100755
new mode 100644
similarity index 60%
copy from scripts/ci/static_checks/bats_tests.sh
copy to scripts/ci/static_checks/in_container_bats_tests.sh
index 2066278..a7c0121
--- a/scripts/ci/static_checks/bats_tests.sh
+++ b/scripts/ci/static_checks/in_container_bats_tests.sh
@@ -16,14 +16,26 @@
# specific language governing permissions and limitations
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
-function run_bats_tests() {
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+function run_in_container_bats_tests() {
if [[ "${#@}" == "0" ]]; then
- docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow:bats-2020.09.05-1.2.1 --tap -r /airflow/tests/bats
+ docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ --entrypoint "/opt/bats/bin/bats" \
+ "-v" "$(pwd):/airflow" \
+ "${AIRFLOW_CI_IMAGE}" \
+ --tap "tests/bats/in_container/"
else
- docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow:bats-2020.09.05-1.2.1 --tap "${@}"
+ docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ --entrypoint "/opt/bats/bin/bats" \
+ "-v" "$(pwd):/airflow" \
+ "${AIRFLOW_CI_IMAGE}" \
+ --tap "${@}"
fi
}
-run_bats_tests "$@"
+build_images::prepare_ci_build
+
+build_images::rebuild_ci_image_if_needed
+
+run_in_container_bats_tests "$@"
diff --git a/tests/bats/test_empty_test.bats
b/tests/bats/in_container/test_in_container.bats
similarity index 73%
copy from tests/bats/test_empty_test.bats
copy to tests/bats/in_container/test_in_container.bats
index 2a4f680..fe2e0c3 100644
--- a/tests/bats/test_empty_test.bats
+++ b/tests/bats/in_container/test_in_container.bats
@@ -1,5 +1,6 @@
#!/usr/bin/env bats
+
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,10 +17,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+# shellcheck disable=SC1091
+source "/opt/bats/lib/load.bash"
-@test "empty test" {
- load bats_utils
+setup() {
+# shellcheck source=scripts/in_container/_in_container_utils.sh
+ source "${AIRFLOW_SOURCES}/scripts/in_container/_in_container_utils.sh"
+}
- run pwd
- assert_success
+@test "test in_container" {
+ run assert_in_container
+ assert [ $status -eq 0 ]
}
diff --git a/tests/bats/test_empty_test.bats b/tests/bats/test_empty_test.bats
index 2a4f680..2830014 100644
--- a/tests/bats/test_empty_test.bats
+++ b/tests/bats/test_empty_test.bats
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+
@test "empty test" {
load bats_utils
