This is an automated email from the ASF dual-hosted git repository. dgrove pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git
The following commit(s) were added to refs/heads/master by this push: new 3c4a1ba Testing improvements (#332) 3c4a1ba is described below commit 3c4a1bab9d632667ac72253d95a8adf72daefe6e Author: David Grove <dgrove-...@users.noreply.github.com> AuthorDate: Sat Dec 8 12:57:10 2018 -0500 Testing improvements (#332) * Support for running :tests:testSystemBasic as part of `helm test`. * Enable running tests:testSystemBasic for DockerContainerFactory configurations (ping test still fails with KubernetesContainerFactory because need to fix upstream to drop CAP_NET_ADMIN on user action pod). * Change error handling strategy in myTask.sh to being mostly explicit to get better control and allow retries of commands that might be vulnerable to CouchDB's eventual consistiency. * Remove obsolete script for starting minikube in TravisCI * Push openwhisk core git tag forward 2 commits to pick up fix for org.apache package rename in tests:testSystemBasic. * Separate chart deployment and testing into separate scripts because we need to use travis_wait when running helm tests, but we want to be able to eagerly see the output of the deploy step to enable early aborts of the test run when it gets hung. --- .travis.yml | 12 +- .../configMapFiles/tests/packageChecker/myTask.sh | 1 + .../configMapFiles/tests/smoketest/myTask.sh | 15 ++- .../configMapFiles/tests/systemTest/myTask.sh | 17 +++ .../templates/tests/package-checker-pod.yaml | 2 +- helm/openwhisk/templates/tests/smoketest-pod.yaml | 2 +- helm/openwhisk/templates/tests/systemtest-cm.yaml | 13 +++ .../{smoketest-pod.yaml => systemtest-pod.yaml} | 14 ++- helm/openwhisk/values.yaml | 4 +- tools/travis/{build-helm.sh => deploy-chart.sh} | 29 +---- tools/travis/{deploy.sh => publish-images.sh} | 0 tools/travis/run-tests.sh | 14 +++ tools/travis/start-minikube.sh | 129 --------------------- 13 files changed, 81 insertions(+), 171 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7d0944..976a1bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,16 +6,15 @@ group: edge dist: xenial sudo: required -jdk: openjdk8 env: global: - secure: d7CuMXbhT83W2x78qiLwgogX1+3aPicd1PlTwwNNDN6QSkImbxareyKThnsqlHIiNj3o5l5DBuiYjy7wrF/xD1g8BQMmTwm99DRx5q3CI3Im3VCi/ZK8SaNjuOy24d7cf5k2tB/87Gk7zmKsMDYm+fpCl+GpgUmIEeIwthiAxuXSDWZ8eQPIptmxj56DeFRNouvXG+dEUtBfWiwN27UPxNKExCixFnegmdtffLbz6hhst7BHr5Ry9acbycre98PCwWZcu9lxFs+SJ1kvnzX2iue4otmDkF1WkJjxaOFPJVs/D3YItg+neLCSxjwBskPed+Fct8bOjcM/uVROJPNIq5icBmaPX2isH0lvtxOeVw/dmioWYXYPN9ygBOe4eO/vtPllN0bcAUo5xl9jXev8ciAozYrYpHVh9Fplfd81rcYTeYzALmRJBdoiWoc3KQGzwGc9sB1ffmy+KWgG9T0zbnS4fALSR4PS [...] - secure: CJtnU94HTDqd4A6uvhFl8IpnmU+wTdlzb8bPBFUl/lI/VKXiRrYpgJdKUro5xEoxFKuqMprLhbyf66niyWLTIeogjUAEu/h/o2dBVeGgSGKoqC0hQgqvnxKFeGlzFJ0XuEs3vbStJGRnQszGsfnnDrscJtR0x9X+1w4aBKI7iPyyuFtVkDD1UsmBbSi+M8FTeq7G7A0reMDaey7uog3CFCpIMl4geshcohQEcKEGbnXQZoLPFpb7cBOE83VXBJ7Y7Dgf/U4keiLovvnuJThGKZm/SVV2KlELmBmtmbx3rMT6Vb5k9ChSdRWapromNnnzmJBIQ5Scc2mwV3A93/SMha1F3IlYpDKs5djfTw8jZfVnuiou7HhTaRjHkmmcwP12/k30gLe2kw0Vezg1TCY4zgtOpcmCxc8RHEy0ceA74rKvRi8LbexTCwX+iAMQFn/pSrh/OqAq/50JbLyczcoO1zXWS38txUQN [...] matrix: - - TRAVIS_KUBE_VERSION=1.10 OW_RUN_SYSTEM_TESTS=no OW_CONTAINER_FACTORY=docker - - TRAVIS_KUBE_VERSION=1.10 OW_RUN_SYSTEM_TESTS=no OW_CONTAINER_FACTORY=kubernetes - - TRAVIS_KUBE_VERSION=1.11 OW_RUN_SYSTEM_TESTS=no OW_CONTAINER_FACTORY=docker + - TRAVIS_KUBE_VERSION=1.10 OW_INCLUDE_SYSTEM_TESTS=true OW_CONTAINER_FACTORY=docker + - TRAVIS_KUBE_VERSION=1.10 OW_INCLUDE_SYSTEM_TESTS=false OW_CONTAINER_FACTORY=kubernetes + - TRAVIS_KUBE_VERSION=1.11 OW_INCLUDE_SYSTEM_TESTS=true OW_CONTAINER_FACTORY=docker services: - docker @@ -33,13 +32,14 @@ before_install: - ./tools/travis/start-kubeadm-dind.sh script: - - ./tools/travis/build-helm.sh + - ./tools/travis/deploy-chart.sh + - travis_wait 20 ./tools/travis/run-tests.sh - ./tools/travis/collect-logs.sh - ./tools/travis/box-upload.py "logs" "deploy-kube-$TRAVIS_BUILD_ID-$TRAVIS_BRANCH-$TRAVIS_JOB_NUMBER.tar.gz" deploy: - provider: script - script: ./tools/travis/deploy.sh + script: ./tools/travis/publish-images.sh on: branch: master repo: apache/incubator-openwhisk-deploy-kube diff --git a/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh b/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh index 240e534..da083f8 100644 --- a/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh +++ b/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh @@ -5,6 +5,7 @@ # installed and is accessible using the guest auth. packageListingCheck() { + echo "Looking for package $1" if [ -z "$1" ]; then echo "Error, package listing check called without a package name" exit 1 diff --git a/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh b/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh index d1c9f1c..d802bcb 100644 --- a/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh +++ b/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh @@ -12,10 +12,13 @@ function main() { return {body: 'Hello world'} } EOL -wsk -i action create hello /tmp/hello.js --web true + +echo "Creating action" +wsk -i action create hello /tmp/hello.js --web true || (echo "FAILED! Could not create a hello action!"; exit 1) # first list actions and expect to see hello # Try several times to accommodate eventual consistency of CouchDB +echo "Listing action" ACTION_LIST_PASSED=false ACTION_LIST_ATTEMPTS=0 until $ACTION_LIST_PASSED; do @@ -35,6 +38,7 @@ until $ACTION_LIST_PASSED; do done # next invoke the new hello world action via the CLI +echo "Inoking action via CLI" RESULT=$(wsk -i action invoke --blocking hello | grep "\"status\": \"success\"") if [ -z "$RESULT" ]; then echo "FAILED! Could not invoke hello action via CLI" @@ -42,6 +46,7 @@ if [ -z "$RESULT" ]; then fi # now run it as a web action +echo "Invoking as web action" HELLO_URL=$(wsk -i action get hello --url | grep "https://") RESULT=$(wget --no-check-certificate -qO- $HELLO_URL | grep 'Hello world') if [ -z "$RESULT" ]; then @@ -50,7 +55,9 @@ if [ -z "$RESULT" ]; then fi # now define it as an api and invoke it that way -wsk -i api create /demo /hello get hello +echo "Registering as an api" +wsk -i api create /demo /hello get hello || (echo "FAILED: unable to create API"; exit 1) +echo "Invoking action via the api" API_URL=$(wsk -i api list | grep hello | awk '{print $4}') RESULT=$(wget --no-check-certificate -qO- "$API_URL" | grep 'Hello world') if [ -z "$RESULT" ]; then @@ -59,8 +66,8 @@ if [ -z "$RESULT" ]; then fi # now delete the resouces so the test could be run again -wsk -i api delete /demo -wsk -i action delete hello +wsk -i api delete /demo || (echo "FAILED! failed to delete API"; exit 1) +wsk -i action delete hello || (echo "FAILED! failed to delete action"; exit 1) echo "PASSED! Created Hello action and invoked via cli, web and apigateway" diff --git a/helm/openwhisk/configMapFiles/tests/systemTest/myTask.sh b/helm/openwhisk/configMapFiles/tests/systemTest/myTask.sh new file mode 100644 index 0000000..39b4751 --- /dev/null +++ b/helm/openwhisk/configMapFiles/tests/systemTest/myTask.sh @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements; and to You under the Apache License, Version 2.0. + +set -ex + +# Clone openwhisk repo at specified tag to get test suite +git clone -q https://github.com/apache/incubator-openwhisk openwhisk +cd /openwhisk +git checkout $OW_GIT_TAG_OPENWHISK + +# compile test suite +./gradlew --console=plain compileTestsScala + +# run tests:testSystemBasic +./gradlew --console=plain :tests:testSystemBasic -Dwhisk.auth="$WSK_AUTH" -Dwhisk.server=$WSK_API_HOST_URL -Dopenwhisk.home=/openwhisk + +echo "PASSED! Successfully executed tests:testSystemBasic" diff --git a/helm/openwhisk/templates/tests/package-checker-pod.yaml b/helm/openwhisk/templates/tests/package-checker-pod.yaml index 72f0764..7e68d33 100644 --- a/helm/openwhisk/templates/tests/package-checker-pod.yaml +++ b/helm/openwhisk/templates/tests/package-checker-pod.yaml @@ -21,7 +21,7 @@ spec: - name: package-checker image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}" imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }} - command: ["/bin/bash", "-c", "set -e; . /task/myTask.sh"] + command: ["/bin/bash", "/task/myTask.sh"] volumeMounts: - name: task-dir mountPath: "/task/myTask.sh" diff --git a/helm/openwhisk/templates/tests/smoketest-pod.yaml b/helm/openwhisk/templates/tests/smoketest-pod.yaml index c2780e0..d33dcd3 100644 --- a/helm/openwhisk/templates/tests/smoketest-pod.yaml +++ b/helm/openwhisk/templates/tests/smoketest-pod.yaml @@ -21,7 +21,7 @@ spec: - name: smoketest image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}" imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }} - command: ["/bin/bash", "-c", "set -e; . /task/myTask.sh"] + command: ["/bin/bash", "/task/myTask.sh"] volumeMounts: - name: task-dir mountPath: "/task/myTask.sh" diff --git a/helm/openwhisk/templates/tests/systemtest-cm.yaml b/helm/openwhisk/templates/tests/systemtest-cm.yaml new file mode 100644 index 0000000..e0edf63 --- /dev/null +++ b/helm/openwhisk/templates/tests/systemtest-cm.yaml @@ -0,0 +1,13 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements; and to You under the Apache License, Version 2.0. + +{{- if .Values.whisk.testing.includeSystemTests }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: openwhisk-tests-systemtest-cm + labels: +{{ include "openwhisk.label_boilerplate" . | indent 4 }} +data: +{{ (.Files.Glob "configMapFiles/tests/systemTest/myTask.sh").AsConfig | indent 2 }} +{{- end }} diff --git a/helm/openwhisk/templates/tests/smoketest-pod.yaml b/helm/openwhisk/templates/tests/systemtest-pod.yaml similarity index 72% copy from helm/openwhisk/templates/tests/smoketest-pod.yaml copy to helm/openwhisk/templates/tests/systemtest-pod.yaml index c2780e0..5127016 100644 --- a/helm/openwhisk/templates/tests/smoketest-pod.yaml +++ b/helm/openwhisk/templates/tests/systemtest-pod.yaml @@ -1,12 +1,13 @@ # Licensed to the Apache Software Foundation (ASF) under one or more contributor # license agreements; and to You under the Apache License, Version 2.0. +{{- if .Values.whisk.testing.includeSystemTests }} apiVersion: v1 kind: Pod metadata: - name: openwhisk-tests-smoketest + name: openwhisk-tests-systemtest labels: - name: openwhisk-tests-smoketest + name: openwhisk-tests-systemtest ow-testpod: "true" {{ include "openwhisk.label_boilerplate" . | indent 4 }} annotations: @@ -16,12 +17,12 @@ spec: volumes: - name: task-dir configMap: - name: openwhisk-tests-smoketest-cm + name: openwhisk-tests-systemtest-cm containers: - - name: smoketest + - name: systemtest image: "{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}" imagePullPolicy: {{ .Values.utility.imagePullPolicy | quote }} - command: ["/bin/bash", "-c", "set -e; . /task/myTask.sh"] + command: ["/bin/bash", "/task/myTask.sh"] volumeMounts: - name: task-dir mountPath: "/task/myTask.sh" @@ -37,3 +38,6 @@ spec: configMapKeyRef: name: whisk.config key: whisk_api_host_url + - name: "OW_GIT_TAG_OPENWHISK" + value: {{ .Values.whisk.versions.openwhisk.gitTag | quote }} +{{- end }} diff --git a/helm/openwhisk/values.yaml b/helm/openwhisk/values.yaml index 9a2d8ab..d6af394 100644 --- a/helm/openwhisk/values.yaml +++ b/helm/openwhisk/values.yaml @@ -59,11 +59,13 @@ whisk: containerPool: userMemory: "2048m" runtimes: "runtimes.json" + testing: + includeSystemTests: false versions: openwhisk: buildDate: "2018-11-26-16:11:11Z" buildNo: "20181126a" - gitTag: "7bd49718232d0381b53794f9353c014a1abfef15" + gitTag: "83de20ef2b7ef383a5708b59f36ba38e4cc279a8" openwhiskCli: tag: "8da6e3b" openwhiskCatalog: diff --git a/tools/travis/build-helm.sh b/tools/travis/deploy-chart.sh similarity index 85% rename from tools/travis/build-helm.sh rename to tools/travis/deploy-chart.sh index da3ec82..5f4e88a 100755 --- a/tools/travis/build-helm.sh +++ b/tools/travis/deploy-chart.sh @@ -151,6 +151,9 @@ ROOTDIR="$SCRIPTDIR/../../" # Default to docker container factory if not specified OW_CONTAINER_FACTORY=${OW_CONTAINER_FACTORY:="docker"} +# Default to not including system tests in helm test suite +OW_INCLUDE_SYSTEM_TESTS=${$OW_INCLUDE_SYSTEM_TESTS:="false"} + # Default timeout limit to 60 steps TIMEOUT_STEP_LIMIT=${TIMEOUT_STEP_LIMIT:=60} @@ -181,6 +184,8 @@ whisk: apiHostName: $WSK_HOST apiHostPort: $WSK_PORT runtimes: "runtimes-minimal-travis.json" + testing: + includeSystemTests: $OW_INCLUDE_SYSTEM_TESTS # TODO: instead document how to enable dynamic volume provisioning for dind k8s: @@ -229,27 +234,3 @@ deploymentHealthCheck "alarmprovider" deploymentHealthCheck "cloudantprovider" deploymentHealthCheck "kafkaprovider" - -### -# Now run the tests provided in the Chart to sanity check the deployment -### -if helm test ow4travis; then - echo "PASSED! Deployment verification tests passed." -else - echo "FAILED: Deployment verification tests failed." - kubectl logs -n openwhisk -low-testpod=true - exit 1 -fi - - -### -# Finally, clone the main openwhisk repo to get the test suite and run tests:testSystemBasic -# TODO: The following tests have issues under the KubernetesContainerFactory -# 1. WskActionTest "not be able to use 'ping' in an action" -- fails because user actions are full fledged pods with unrestricted network -# 2. Tests that read activation logs in retry loops fail; perhaps because log extraction is so slow -### -if [ "$OW_RUN_SYSTEM_TESTS" == "yes" ]; then - (git clone https://github.com/apache/incubator-openwhisk openwhisk && cd openwhisk && \ - TERM=dumb ./gradlew install && \ - TERM=dumb ./gradlew :tests:testSystemBasic -Dwhisk.auth="$WSK_AUTH" -Dwhisk.server=https://$WSK_HOST:$WSK_PORT -Dopenwhisk.home=`pwd`) || exit 1 -fi diff --git a/tools/travis/deploy.sh b/tools/travis/publish-images.sh similarity index 100% rename from tools/travis/deploy.sh rename to tools/travis/publish-images.sh diff --git a/tools/travis/run-tests.sh b/tools/travis/run-tests.sh new file mode 100755 index 0000000..8aa1fba --- /dev/null +++ b/tools/travis/run-tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements; and to You under the Apache License, Version 2.0. + +### +# Now run the tests provided in the Chart to verify the deployment +### +if helm test ow4travis --timeout 1200; then + echo "PASSED! Deployment verification tests passed." +else + echo "FAILED: Deployment verification tests failed." + kubectl logs -n openwhisk -low-testpod=true + exit 1 +fi diff --git a/tools/travis/start-minikube.sh b/tools/travis/start-minikube.sh deleted file mode 100755 index 7571adb..0000000 --- a/tools/travis/start-minikube.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more contributor -# license agreements; and to You under the Apache License, Version 2.0. - -# NOTE: This script is not currently being used. -# Will remove after a couple of weeks experience with -# kubeadm-dind-cluster on TravisCI - -# This script assumes Docker is already installed -set -x - -#check_or_build_nsenter derived from https://github.com/bitnami/kubernetes-travis/blob/master/scripts/cluster-up-minikube.sh (Apache 2.0 License) -check_or_build_nsenter() { - which nsenter >/dev/null && return 0 - echo "INFO: Building 'nsenter' ..." -cat <<-EOF | docker run -i --rm -v "$(pwd):/build" ubuntu:14.04 >& nsenter.build.log - apt-get update - apt-get install -qy git bison build-essential autopoint libtool automake autoconf gettext pkg-config - git clone --depth 1 git://git.kernel.org/pub/scm/utils/util-linux/util-linux.git /tmp/util-linux - cd /tmp/util-linux - ./autogen.sh - ./configure --without-python --disable-all-programs --enable-nsenter - make nsenter - cp -pfv nsenter /build -EOF - if [ ! -f ./nsenter ]; then - echo "ERROR: nsenter build failed, log:" - cat nsenter.build.log - return 1 - fi - echo "INFO: nsenter build OK, installing ..." - sudo install -v nsenter /usr/local/bin -} - -# Download and install misc packages and utilities -pushd /tmp - # helm on minikube's vm-driver=none requires nsenter, which is not present by default on ubuntu 14.04 - check_or_build_nsenter - - # Need socat for helm to forward connections to tiller on ubuntu 16.04 - sudo apt update - sudo apt install -y socat - - # download and install the wsk cli - wget -q https://github.com/apache/incubator-openwhisk-cli/releases/download/latest/OpenWhisk_CLI-latest-linux-amd64.tgz - tar xzf OpenWhisk_CLI-latest-linux-amd64.tgz - sudo cp wsk /usr/local/bin/wsk - - # Download and install kubectl and minikube following the recipe in the minikube - # project README.md for using minikube for Linux Continuous Integration with VM Support - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$TRAVIS_KUBE_VERSION/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin - curl -Lo minikube https://storage.googleapis.com/minikube/releases/$TRAVIS_MINIKUBE_VERSION/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin - - # Download and install helm - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh && chmod +x get_helm.sh && ./get_helm.sh -popd - -# set docker0 to promiscuous mode -sudo ip link set docker0 promisc on - -export MINIKUBE_WANTUPDATENOTIFICATION=false -export MINIKUBE_WANTREPORTERRORPROMPT=false -export MINIKUBE_HOME=$HOME -export CHANGE_MINIKUBE_NONE_USER=true -mkdir $HOME/.kube || true -touch $HOME/.kube/config - -export KUBECONFIG=$HOME/.kube/config - -if [ "$TRAVIS_MINIKUBE_VERSION" == "v0.25.2" ]; then - # bootstrap with localkube -- restricts choice of kube version - sudo -E /usr/local/bin/minikube start --vm-driver=none --bootstrapper=localkube --extra-config=apiserver.Authorization.Mode=RBAC --kubernetes-version=$TRAVIS_KUBE_VERSION -else - # bootstrap with kubeadm (preferred). - sudo -E /usr/local/bin/minikube start --vm-driver=none --bootstrapper=kubeadm --extra-config=apiserver.authorization-mode=RBAC --kubernetes-version=$TRAVIS_KUBE_VERSION -fi - -# Wait until we have a ready node in minikube -TIMEOUT=0 -TIMEOUT_COUNT=60 -until [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do - if [ -n "$(/usr/local/bin/kubectl get nodes | grep Ready)" ]; then - break - fi - - echo "minikube is not up yet" - let TIMEOUT=TIMEOUT+1 - sleep 5 -done - -if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then - echo "Failed to start minikube" - exit 1 -fi - -minikube update-context - -echo "minikube is deployed and reachable" -/usr/local/bin/kubectl describe nodes - -# Pods running in kube-system namespace should have cluster-admin role -kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default - -# Install tiller into the minikube cluster -/usr/local/bin/helm init --service-account default - -# Wait for tiller to be ready -TIMEOUT=0 -TIMEOUT_COUNT=60 -until [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do - TILLER_STATUS=$(/usr/local/bin/kubectl -n kube-system get pods -o wide | grep tiller-deploy | awk '{print $3}') - TILLER_READY_COUNT=$(/usr/local/bin/kubectl -n kube-system get pods -o wide | grep tiller-deploy | awk '{print $2}') - if [[ "$TILLER_STATUS" == "Running" ]] && [[ "$TILLER_READY_COUNT" == "1/1" ]]; then - break - fi - echo "Waiting for tiller to be ready" - /usr/local/bin/kubectl -n kube-system get pods -o wide - let TIMEOUT=TIMEOUT+1 - sleep 5 -done - -if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then - echo "Failed to install tiller" - - # Dump lowlevel logs to help diagnose failure to start tiller - minikube logs - kubectl -n kube-system describe pods - exit 1 -fi