This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 977da8d Separate out coverage files for different executors (#16689)
977da8d is described below
commit 977da8d54b483a8c226f1a457900c617b1841145
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Jun 28 20:02:05 2021 +0200
Separate out coverage files for different executors (#16689)
* Separate out coverage files for different executors
The coverage files and virtualenvs were not separated out
for different executors - running them in parallell could cause
the files override each other and mysterious failures of
the K8S tests.
This PR separates out the coverage files.
* Update TESTING.rst
---
TESTING.rst | 6 ++++++
scripts/ci/kubernetes/ci_run_helm_upgrade.sh | 7 +++++--
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 6 ++++--
.../ci_setup_cluster_and_run_kubernetes_tests_single_job.sh | 8 ++++++++
.../ci_upgrade_cluster_with_different_executors_single_job.sh | 8 ++++++++
scripts/ci/libraries/_initialization.sh | 1 -
scripts/ci/libraries/_parallel.sh | 8 +++++---
7 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/TESTING.rst b/TESTING.rst
index 1011c66..60d0926 100644
--- a/TESTING.rst
+++ b/TESTING.rst
@@ -625,6 +625,8 @@ Running Kubernetes tests via shell:
.. code-block:: bash
+ export EXECUTOR="KubernetesExecutor" ## can be also CeleryExecutor or
CeleryKubernetesExecutor
+
./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
- runs all kubernetes tests
./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh TEST [TEST ...]
- runs selected kubernetes tests (from kubernetes_tests folder)
@@ -657,6 +659,10 @@ in the host.
You can enter the shell via those scripts
+.. code-block:: bash
+
+ export EXECUTOR="KubernetesExecutor" ## can be also CeleryExecutor or
CeleryKubernetesExecutor
+
./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive]
- Activates virtual environment ready to run tests and drops you in
./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [--help]
- Prints this help message
diff --git a/scripts/ci/kubernetes/ci_run_helm_upgrade.sh
b/scripts/ci/kubernetes/ci_run_helm_upgrade.sh
index 4a35136..1e9b94c 100755
--- a/scripts/ci/kubernetes/ci_run_helm_upgrade.sh
+++ b/scripts/ci/kubernetes/ci_run_helm_upgrade.sh
@@ -18,10 +18,13 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+EXECUTOR=KubernetesExecutor
+export EXECUTOR
+
# We started with KubernetesExecutor. Let's run tests first
"$( dirname "${BASH_SOURCE[0]}" )/ci_run_kubernetes_tests.sh"
-for mode in CeleryExecutor KubernetesExecutor
+for EXECUTOR in CeleryExecutor KubernetesExecutor
do
- kind::upgrade_airflow_with_helm "${mode}"
+ kind::upgrade_airflow_with_helm "${EXECUTOR}"
"$( dirname "${BASH_SOURCE[0]}" )/ci_run_kubernetes_tests.sh"
done
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index 487b16b..6a03fb39 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -18,6 +18,8 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+: "${EXECUTOR:?You must set EXECUTOR to one of 'KubernetesExecutor',
'CeleryExecutor', 'CeleryKubernetesExecutor' }"
+
kind::make_sure_kubernetes_tools_are_installed
kind::get_kind_cluster_name
@@ -62,7 +64,7 @@ function parse_tests_to_run() {
"--durations=100"
"--cov=airflow/"
"--cov-config=.coveragerc"
- "--cov-report=xml:files/coverage=${KIND_CLUSTER_NAME}.xml"
+
"--cov-report=xml:files/coverage-${KIND_CLUSTER_NAME}-${HOST_PYTHON_VERSION}-${EXECUTOR}.xml"
"--color=yes"
"--maxfail=50"
"--pythonwarnings=ignore::DeprecationWarning"
@@ -76,7 +78,7 @@ function create_virtualenv() {
HOST_PYTHON_VERSION=$(python3 -c 'import sys;
print(f"{sys.version_info[0]}.{sys.version_info[1]}")')
readonly HOST_PYTHON_VERSION
- local
virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv/${KIND_CLUSTER_NAME}_host_python_${HOST_PYTHON_VERSION}"
+ local
virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv/${KIND_CLUSTER_NAME}_host_python_${HOST_PYTHON_VERSION}_${EXECUTOR}"
mkdir -pv "${BUILD_CACHE_DIR}/.kubernetes_venv/"
if [[ ! -d ${virtualenv_path} ]]; then
diff --git
a/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
index 914e426..839c1ca 100755
---
a/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
+++
b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
@@ -30,6 +30,13 @@ fi
export PYTHON_MAJOR_MINOR_VERSION=$1
shift
+if [[ $1 == "" ]]; then
+ >&2 echo "Requires executor mode as third parameter"
+ exit 1
+fi
+export EXECUTOR=$1
+shift
+
# Requires PARALLEL_JOB_STATUS
@@ -41,6 +48,7 @@ fi
echo
echo "KUBERNETES_VERSION: ${KUBERNETES_VERSION}"
echo "PYTHON_MAJOR_MINOR_VERSION: ${PYTHON_MAJOR_MINOR_VERSION}"
+echo "EXECUTOR: ${EXECUTOR}"
echo
# shellcheck source=scripts/ci/libraries/_script_init.sh
diff --git
a/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_single_job.sh
b/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_single_job.sh
index 8090fe8..4e411be 100755
---
a/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_single_job.sh
+++
b/scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_single_job.sh
@@ -30,6 +30,13 @@ fi
export PYTHON_MAJOR_MINOR_VERSION=$1
shift
+if [[ $1 == "" ]]; then
+ >&2 echo "Requires executor mode as third parameter"
+ exit 1
+fi
+export EXECUTOR=$1
+shift
+
# Requires PARALLEL_JOB_STATUS
@@ -41,6 +48,7 @@ fi
echo
echo "KUBERNETES_VERSION: ${KUBERNETES_VERSION}"
echo "PYTHON_MAJOR_MINOR_VERSION: ${PYTHON_MAJOR_MINOR_VERSION}"
+echo "EXECUTOR: ${EXECUTOR}"
echo
# shellcheck source=scripts/ci/libraries/_script_init.sh
diff --git a/scripts/ci/libraries/_initialization.sh
b/scripts/ci/libraries/_initialization.sh
index 6fd5d21..24fce97 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -814,7 +814,6 @@ function initialization::make_constants_read_only() {
readonly KIND_VERSION
readonly HELM_VERSION
readonly KUBECTL_VERSION
- readonly EXECUTOR
readonly POSTGRES_VERSION
readonly MYSQL_VERSION
diff --git a/scripts/ci/libraries/_parallel.sh
b/scripts/ci/libraries/_parallel.sh
index 2c3dcfb..8deeb07 100644
--- a/scripts/ci/libraries/_parallel.sh
+++ b/scripts/ci/libraries/_parallel.sh
@@ -274,7 +274,8 @@ function parallel::get_maximum_parallel_k8s_jobs() {
function parallel::run_single_helm_test() {
local kubernetes_version=$1
local python_version=$2
- local single_job_filename=$3
+ local executor=$3
+ local single_job_filename=$4
local job="Cluster-${kubernetes_version}-python-${python_version}"
mkdir -p "${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}"
@@ -283,7 +284,7 @@ function parallel::run_single_helm_test() {
echo "Starting helm tests for kubernetes version ${kubernetes_version},
python version: ${python_version}"
parallel --ungroup --bg --semaphore --semaphorename "${SEMAPHORE_NAME}" \
--jobs "${MAX_PARALLEL_K8S_JOBS}" "${single_job_filename}" \
- "${kubernetes_version}" "${python_version}" >"${JOB_LOG}" 2>&1
+ "${kubernetes_version}" "${python_version}" "${executor}"
>"${JOB_LOG}" 2>&1
}
function parallel::run_helm_tests_in_parallel() {
@@ -305,7 +306,8 @@ function parallel::run_helm_tests_in_parallel() {
export FORWARDED_PORT_NUMBER
API_SERVER_PORT=$((19090 + index))
export API_SERVER_PORT
- parallel::run_single_helm_test "${kubernetes_version}"
"${python_version}" "${single_job_filename}" "${@}"
+ # shellcheck disable=SC2153
+ parallel::run_single_helm_test "${kubernetes_version}"
"${python_version}" "${EXECUTOR}" "${single_job_filename}" "${@}"
done
set +e
parallel --semaphore --semaphorename "${SEMAPHORE_NAME}" --wait
