This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 41d9e0359e HDDS-7645. Kubernetes check should fail fast if cluster
cannot start (#5028)
41d9e0359e is described below
commit 41d9e0359e630b2eefdaa8016cb4921a3337d986
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Jul 7 10:26:52 2023 +0200
HDDS-7645. Kubernetes check should fail fast if cluster cannot start (#5028)
---
.../src/main/k8s/examples/getting-started/test.sh | 16 ++-----
.../dist/src/main/k8s/examples/minikube/test.sh | 16 ++-----
.../dist/src/main/k8s/examples/ozone-dev/test.sh | 16 ++-----
.../dist/src/main/k8s/examples/ozone/test.sh | 16 ++-----
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 54 +++++++++++++++-------
5 files changed, 50 insertions(+), 68 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/test.sh
b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/test.sh
index 7d6bdfb981..23d8ad795d 100755
--- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/test.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/test.sh
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -eu -o pipefail
+
export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 &&
pwd )"
cd "$K8S_DIR"
@@ -22,18 +24,6 @@ cd "$K8S_DIR"
# shellcheck source=/dev/null
source "../testlib.sh"
-rm -rf result
-
-regenerate_resources
-
-start_k8s_env
+pre_run_setup
execute_robot_test scm-0 smoketest/basic/basic.robot
-
-combine_reports
-
-get_logs
-
-stop_k8s_env
-
-revert_resources
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/test.sh
b/hadoop-ozone/dist/src/main/k8s/examples/minikube/test.sh
index 7d6bdfb981..23d8ad795d 100755
--- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/test.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/test.sh
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -eu -o pipefail
+
export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 &&
pwd )"
cd "$K8S_DIR"
@@ -22,18 +24,6 @@ cd "$K8S_DIR"
# shellcheck source=/dev/null
source "../testlib.sh"
-rm -rf result
-
-regenerate_resources
-
-start_k8s_env
+pre_run_setup
execute_robot_test scm-0 smoketest/basic/basic.robot
-
-combine_reports
-
-get_logs
-
-stop_k8s_env
-
-revert_resources
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/test.sh
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/test.sh
index 7d6bdfb981..23d8ad795d 100755
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/test.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/test.sh
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -eu -o pipefail
+
export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 &&
pwd )"
cd "$K8S_DIR"
@@ -22,18 +24,6 @@ cd "$K8S_DIR"
# shellcheck source=/dev/null
source "../testlib.sh"
-rm -rf result
-
-regenerate_resources
-
-start_k8s_env
+pre_run_setup
execute_robot_test scm-0 smoketest/basic/basic.robot
-
-combine_reports
-
-get_logs
-
-stop_k8s_env
-
-revert_resources
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh
index fd5b0331d0..d1006a3fc8 100755
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -eu -o pipefail
+
export K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 &&
pwd )"
cd "$K8S_DIR"
@@ -22,11 +24,7 @@ cd "$K8S_DIR"
# shellcheck source=/dev/null
source "../testlib.sh"
-rm -rf result
-
-regenerate_resources
-
-start_k8s_env
+pre_run_setup
export SCM=scm-0
@@ -41,11 +39,3 @@ wait_for_startup
execute_robot_test ${SCM} -v PREFIX:pre smoketest/freon/validate.robot
execute_robot_test ${SCM} -v PREFIX:post smoketest/freon/generate.robot
execute_robot_test ${SCM} -v PREFIX:post smoketest/freon/validate.robot
-
-combine_reports
-
-get_logs
-
-stop_k8s_env
-
-revert_resources
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
b/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
index 5c777b7e75..ab10784ac5 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
+++ b/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh
@@ -16,15 +16,16 @@
# limitations under the License.
retry() {
- n=0
- until [ $n -ge 100 ]
+ local -i n=0
+ local -i attempts=${RETRY_ATTEMPTS:-100}
+ until [ $n -ge $attempts ]
do
"$@" && break
n=$[$n+1]
echo "$n '$@' is failed..."
sleep ${RETRY_SLEEP:-3}
done
- if [ $n -eq 100 ]; then
+ if [ $n -eq $attempts ]; then
return 255
fi
}
@@ -47,15 +48,13 @@ wait_for_startup(){
}
all_pods_are_running() {
- RUNNING_COUNT=$(kubectl get pod --field-selector status.phase=Running | wc
-l)
- ALL_COUNT=$(kubectl get pod | wc -l)
- RUNNING_COUNT=$((RUNNING_COUNT - 1))
- ALL_COUNT=$((ALL_COUNT - 1))
- if [ "$RUNNING_COUNT" -lt "3" ]; then
- echo "$RUNNING_COUNT pods are running. Waiting for more."
+ local -i running=$(kubectl get pod --field-selector status.phase=Running |
grep -v 'STATUS' | wc -l)
+ local -i all=$(kubectl get pod | grep -v 'STATUS' | wc -l)
+ if [ "$running" -lt "3" ]; then
+ echo "$running pods are running. Waiting for more."
return 1
- elif [ "$RUNNING_COUNT" -ne "$ALL_COUNT" ]; then
- echo "$RUNNING_COUNT pods are running out from the $ALL_COUNT"
+ elif [ "$running" -ne "$all" ]; then
+ echo "$running / $all pods are running"
return 2
else
STARTED=true
@@ -63,7 +62,15 @@ all_pods_are_running() {
fi
}
-start_k8s_env() {
+pre_run_setup() {
+ rm -fr logs result
+ regenerate_resources
+ reset_k8s_env
+ start_k8s_env
+ wait_for_startup
+}
+
+reset_k8s_env() {
print_phase "Deleting existing k8s resources"
#reset environment
kubectl delete statefulset --all
@@ -74,13 +81,25 @@ start_k8s_env() {
kubectl delete pod --all
kubectl delete pvc --all
kubectl delete pv --all
+}
+start_k8s_env() {
print_phase "Applying k8s resources from $(basename $(pwd))"
kubectl apply -k .
- wait_for_startup
+ trap post_run EXIT HUP INT TERM
+}
+
+post_run() {
+ set +e
+ combine_reports
+ get_logs
+ stop_k8s_env
+ revert_resources
+ set -e
}
get_logs() {
+ print_phase "Collecting container logs"
mkdir -p logs
for pod in $(kubectl get pods -o custom-columns=NAME:.metadata.name | tail
-n +2); do
for initContainer in $(kubectl get pod -o
jsonpath='{.spec.initContainers[*].name}' "${pod}"); do
@@ -91,7 +110,8 @@ get_logs() {
}
stop_k8s_env() {
- if [ ! "$KEEP_RUNNING" ]; then
+ if [ "${KEEP_RUNNING:-false}" != "true" ]; then
+ print_phase "Deleting k8s resources"
kubectl delete -k .
fi
}
@@ -145,8 +165,10 @@ execute_robot_test() {
}
combine_reports() {
- rm result/output.xml || true
- rebot -d result --nostatusrc -o output.xml -N $(basename "$(pwd)")
result/*.xml
+ if [[ -d result ]]; then
+ rm -f result/output.xml
+ rebot -d result --nostatusrc -o output.xml -N $(basename "$(pwd)")
result/*.xml
+ fi
}
print_phase() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
