This is an automated email from the ASF dual-hosted git repository. houshengbo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git
The following commit(s) were added to refs/heads/master by this push: new bf0f1c0 Deploy CouchDB manually (#46) bf0f1c0 is described below commit bf0f1c040dc92ecffca194ea3d7655863d75a4a7 Author: Dan Lavine <dlav...@us.ibm.com> AuthorDate: Thu Aug 10 20:38:54 2017 +0000 Deploy CouchDB manually (#46) * Deploy CouchDB manually * This gets rid of the need for the OpenWhisk configuration image * Update Docker images so there is no more configuration image * Remove specific Instructions for Minikube since everything is deployed via yaml. The only properteis that should now change is an additional environment variable in the invoker.yml * Update CI * Cleanup generated Docker files * Moved Docker configuration to the Nginx folder. * fix typos * compine couchdb and couchdb-setup * fix typos in travis * Fix auth_whisk_system environment variable --- .travis/build.sh | 59 +++--- .travis/setup.sh | 2 +- Dockerfile | 42 ----- README.md | 198 +++------------------ .../environments/kube/files/db-service.yml | 15 -- ansible-kube/environments/kube/files/db.yml | 24 --- .../environments/kube/files/nginx-service.yml | 22 --- ansible-kube/environments/kube/files/nginx.yml | 52 ------ ansible-kube/environments/kube/group_vars/all | 59 ------ ansible-kube/environments/kube/hosts | 26 --- ansible-kube/openwhisk.yml | 6 - ansible-kube/roles/couchdb/tasks/deploy.yml | 37 ---- ansible-kube/roles/couchdb/tasks/main.yml | 6 - ansible-kube/roles/routemgmt/tasks/deploy.yml | 6 - ansible-kube/tasks/initdb.yml | 31 ---- ansible-kube/tasks/installOpenwhiskCatalog.yml | 34 ---- configure/cleanup.sh | 3 - configure/configure.sh | 47 ----- configure/configure_whisk.yml | 21 --- docker/build.sh | 76 -------- kubernetes/controller/controller.yml | 22 +-- kubernetes/couchdb/README.md | 49 +++++ kubernetes/couchdb/couchdb.yml | 51 ++++++ kubernetes/couchdb/docker/Dockerfile | 14 ++ kubernetes/couchdb/docker/init.sh | 93 ++++++++++ kubernetes/invoker/README.md | 13 ++ kubernetes/invoker/invoker.yml | 10 +- kubernetes/kafka/README.md | 21 +++ kubernetes/nginx/README.md | 17 ++ .../nginx => kubernetes/nginx/docker}/Dockerfile | 0 kubernetes/nginx/docker/build.sh | 45 +++++ minikube/README.md | 41 ----- minikube/ci/build.sh | 84 --------- minikube/ci/setup.sh | 38 ---- minikube/deploy_minikube.sh | 61 ------- 35 files changed, 366 insertions(+), 959 deletions(-) diff --git a/.travis/build.sh b/.travis/build.sh index cfc0ab0..6eca786 100755 --- a/.travis/build.sh +++ b/.travis/build.sh @@ -7,41 +7,33 @@ ROOTDIR="$SCRIPTDIR/../" cd $ROOTDIR -# setup the appropriate configuration image -sed -ie "s/whisk_config:v1.5.6/whisk_config:$TRAVIS_KUBE_VERSION/g" configure/configure_whisk.yml - kubectl apply -f configure/openwhisk_kube_namespace.yml -kubectl apply -f configure/configure_whisk.yml - -sleep 5 -CONFIGURE_POD=$(kubectl get pods --all-namespaces -o wide | grep configure | awk '{print $2}') - -PASSED=false -TIMEOUT=0 -until $PASSED || [ $TIMEOUT -eq 25 ]; do - KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get jobs | grep configure-openwhisk | awk '{print $3}') - if [ $KUBE_DEPLOY_STATUS -eq 1 ]; then - PASSED=true - break - fi +couchdbHealthCheck () { + # wait for the pod to be created before getting the job name + sleep 5 + POD_NAME=$(kubectl -n openwhisk get pods -o wide --show-all | grep "couchdb" | awk '{print $1}') - kubectl get pods --all-namespaces -o wide --show-all + PASSED=false + TIMEOUT=0 + until [ $TIMEOUT -eq 25 ]; do + if [ -n "$(kubectl -n openwhisk logs $POD_NAME | grep "Apache CouchDB has started on http://0.0.0.0:5984")" ]; then + break + fi - let TIMEOUT=TIMEOUT+1 - sleep 30 -done + let TIMEOUT=TIMEOUT+1 + sleep 30 + done -if [ "$PASSED" = false ]; then - kubectl -n openwhisk logs $CONFIGURE_POD - kubectl get jobs --all-namespaces -o wide --show-all - kubectl get pods --all-namespaces -o wide --show-all + if [ $TIMEOUT -eq 25 ]; then + echo "Failed to finish deploying CouchDB" - echo "The job to configure OpenWhisk did not finish with an exit code of 1" - exit 1 -fi + kubectl -n openwhisk logs $POD_NAME + exit 1 + fi -echo "The job to configure OpenWhisk finished successfully" + echo "CouchDB is up and running" +} deploymentHealthCheck () { if [ -z "$1" ]; then @@ -106,6 +98,13 @@ statefulsetHealthCheck () { } +# setup couchdb +pushd kubernetes/couchdb + kubectl apply -f couchdb.yml + + couchdbHealthCheck +popd + # setup zookeeper pushd kubernetes/zookeeper kubectl apply -f zookeeper.yml @@ -191,10 +190,10 @@ if [ -z "$RESULT" ]; then echo "FAILED! Could not invoked custom action" echo " ----------------------------- controller logs ---------------------------" - kubectl -n openwhisk logs $(kubectl get pods --all-namespaces -o wide | grep controller | awk '{print $2}') + kubectl -n openwhisk logs controller-0 echo " ----------------------------- invoker logs ---------------------------" - kubectl -n openwhisk logs $(kubectl get pods --all-namespaces -o wide | grep invoker | awk '{print $2}') + kubectl -n openwhisk logs invoker-0 exit 1 fi diff --git a/.travis/setup.sh b/.travis/setup.sh index 8f19f39..6fdbc42 100755 --- a/.travis/setup.sh +++ b/.travis/setup.sh @@ -57,7 +57,7 @@ echo "Kubernetes is deployed and reachable" # a command that is run in the background. Technically Kubernetes could be # up and running, but those files might not exist yet as the previous command # could create them after Kube starts successfully. -sleep 1 +sleep 5 sudo chown -R $USER:$USER $HOME/.kube diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4dca29e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM ubuntu:trusty -ENV DEBIAN_FRONTEND noninteractive -ENV UCF_FORCE_CONFFNEW YES -RUN ucf --purge /boot/grub/menu.lst -ARG KUBE_VERSION - -# install openwhisk -RUN apt-get -y update && \ - apt-get -y upgrade && \ - apt-get install -y \ - git \ - curl \ - wget \ - apt-transport-https \ - ca-certificates \ - python-pip \ - python-dev \ - libffi-dev \ - libssl-dev \ - libxml2-dev \ - libxslt1-dev \ - libjpeg8-dev \ - zlib1g-dev \ - vim - -# clone OpenWhisk and install dependencies -# Note that we are not running the install all script since we do not care about Docker. -RUN git clone https://github.com/openwhisk/openwhisk && \ - /openwhisk/tools/ubuntu-setup/misc.sh && \ - /openwhisk/tools/ubuntu-setup/pip.sh && \ - /openwhisk/tools/ubuntu-setup/java8.sh && \ - /openwhisk/tools/ubuntu-setup/scala.sh && \ - /openwhisk/tools/ubuntu-setup/ansible.sh - -# Change this to https://github.com/openwhisk/openwhisk-devtools when committing to master -COPY ansible-kube /incubator-openwhisk-deploy-kube/ansible-kube -COPY configure /incubator-openwhisk-deploy-kube/configure - -# install kube dependencies -RUN wget https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubectl && \ - chmod +x kubectl && \ - mv kubectl /usr/local/bin/kubectl diff --git a/README.md b/README.md index c50a96a..4908cef 100644 --- a/README.md +++ b/README.md @@ -3,22 +3,7 @@ [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) [![Build Status](https://travis-ci.org/apache/incubator-openwhisk-deploy-kube.svg?branch=master)](https://travis-ci.org/apache/incubator-openwhisk-deploy-kube) -This repository can be used to deploy OpenWhisk to a Kubernetes cluster. -To accomplish this, we have created a Kubernetes job responsible for -deploying OpenWhisk from inside of Kubernetes. This job runs through -the OpenWhisk Ansible playbooks with some modifications to "Kube-ify" -specific actions. The reason for this approach is to try and streamline -a one size fits all way of deploying OpenWhisk. - -Currently, the OpenWhisk deployment is going to be a static set of -Kube yaml files. It should be easy to use the tools from this -repo to build your own OpenWhisk deployment job, allowing you to -set up your own configurations if need be. - -The scripts and Docker images should be able to: - -1. Build the Docker image used for deploying OpenWhisk. -2. Uses a Kubernetes job to deploy OpenWhisk. +This repo can be used to deploy OpenWhisk to a Kubernetes cluster. # Table of Contents @@ -26,17 +11,14 @@ The scripts and Docker images should be able to: * [Setting up Kuberentes](#setting-up-kubernetes) * [Configuring OpenWhisk](#configure-openwhisk) * [Cleanup](#cleanup) -* [Troubleshooting](#troubleshooting) -* [Manually Building Custom Docker Files](#manually-building-custom-docker-files) -* [Limitations and Enhancements](#limitations-and-enhancements) +* [Limitations](#limitations) * [Issues](#issues) # Requirements A number of requirements must be met for OpenWhisk to deploy on Kubernetes. **Kubernetes** -* Kubernetes version 1.5.6 and 1.6.2 - - https://github.com/kubernetes/kubernetes +* [Kubernetes](https://github.com/kubernetes/kubernetes) version 1.5+ * Kubernetes has [KubeDNS](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/) deployed * (Optional) Kubernetes Pods can receive public addresses. This will be required if you wish to reach Nginx from outside @@ -55,49 +37,23 @@ ways to setup a Dev/Test environmnet depending on your host OS. To mitigate that VirtualBox with a Ubuntu VM. For the full instructions on setting an environment up, take a look at these [instructions](/docs/setting_up_kube/README.md). -We also have experimental support for -[Minikube](https://github.com/kubernetes/minikube), see the -[Minikube-specific install instructions](/minikube/README.md) for more details. - # Configure OpenWhisk -To configure OpenWhisk on Kubernetes, you will need to target a Kubernetes -environment. If you do not have one up and running, then you can look -at the [Setting up Kubernetes](#setting-up-kubernetes) section. Once you -are successfully up, running, and targetd, you will then need to create a -namespace called `openwhisk`. To do this, you can just run the following command. - -``` -kubectl apply -f configure/openwhisk_kube_namespace.yml -``` - -From here, you should just need to run the Kubernetes job to -setup the OpenWhisk environment. The only caveat is that -the default image is used to deploy to kube v1.5.6. -Take a look -[here](https://github.com/apache/incubator-openwhisk-deploy-kube/blob/master/configure/configure_whisk.yml#L19) -if you wish to change to kube v1.6.2 by replacing `v1.5.2` to `v1.6.2`. +To deploy OpenWhisk on Kubernetes, you first need to setup CouchDB. +In this repo, there is a CouchDB instance that can be used to standup +a small OpenWhisk test environment. We would not support using this +deployment strategy for production environments, but to test OpenWhisk +on Kube check out the CouchDB README. -**NOTE** Unfortunately Kube does not have backward compatibility -requirements between the cli and Kube api server. However, -the v1.5.6 image will probably work with any Kube v1.5+ -and the v1.6.2 image will probably work with any Kube v1.6+. -If the configuration image does return compatibility -issues then try [building a custom image](#manually-building-custom-docker-files). +* [CouchDB](kubernetes/couchdb/README.md) -``` -kubectl apply -f configure/configure_whisk.yml -``` - -To see what is happening during the deployment process, you can view -the logs from the configuration Pod creted by the previous command. - -``` -kubectl -n openwhisk logs configure-openwhisk-XXXXX -``` +For a production level CouchDB instance, take a look at the main +OpenWhisk [Docs](https://github.com/apache/incubator-openwhisk/blob/master/tools/db/README.md) +for CouchDB. -Once the configuration job successfully finishes, you will need +Once CouchDB has been successfully deployed, you will need manually deploy the rest of the OpenWhisk components. + * [Zookeeper](kubernetes/zookeeper/README.md) * [Kafka](kubernetes/kafka/README.md) * [Controller](kubernetes/controller/README.md) @@ -105,7 +61,9 @@ manually deploy the rest of the OpenWhisk components. * [Nginx](kubernetes/nginx/README.md) From here, you will now need to get the publicly available address -of Nginx. +of Nginx. If you are using the default Nginx image with a NodePort +Service, then you can obtain the public IP using the following guide: + 1. Obtain the IP address of the Kubernetes nodes. ``` @@ -151,108 +109,14 @@ replace the `[nginx_ip]` bellow. # Cleanup -At some point there might be a need to cleanup the Kubernetes -environment. For this, we want to delete all the OpenWhisk deployments, -services, jobs and whatever else might be there. For this, you can run the following script: +At some point there might be a need to cleanup the Kubernetes environment. +For this, we want to delete all the OpenWhisk deployments, services, jobs +and whatever else might be there. For this, you can run the following script: ``` -./kube_environment/cleanup.sh +./configure/cleanup.sh ``` -# Troubleshooting -## Kafka - -When inspecting kafka logs of various components and they are not able to -send/receive message then Kafka is the usual problem. If everything is deployed -on a single machine, then you might need to allow Kube Pods to communicate with -themselves over a Kube Service. Setting a network to promiscous mode can be the -solution will enable network traffic to route in a loop back to itself. E.g: - -``` -ip link set docker0 promisc on -``` - -## Kube RBAC - -When deploying the configuration pod, if it fails with a -`error validating data: the server does not allow access to the requested resource;` -error then you probably do not have permissions to create Pods from a Pod running -in the Kube cluster. You will need to create a ClusterRoleBinding with proper -security settings. For information about the role bindings, -take a look at the info [here](https://kubernetes.io/docs/admin/authorization/rbac/). - -# Manually Building Custom Docker Files - -There are two images that are required when deploying OpenWhisk on Kube, -Nginx and the OpenWhisk configuration image. Right now the the configuration -images built will work with a Kube version 1.5.6 and 1.6.2. To build the -configuration image with a custom Kube version you can edit the build script -[here](https://github.com/apache/incubator-openwhisk-deploy-kube/blob/kube-1.6/docker/build.sh#L87-L88) - -To build these images, there is a helper script that installs all -required dependencies and the Docker images themselves. For example, -one the required dependencies is the wsk cli and to build it you will need -to download the [OpenWhisk repo](https://github.com/openwhisk/openwhisk) -and setup your invironment to build the docker images via gradle. That -setup can be found [here](https://github.com/apache/incubator-openwhisk#native-development). - -**Important** -To build custom docker images, you will need to be on a Linux machine. -During the `wsk` cli build process it mounts a number of files from the -host machine. Because of this, Golang determines that the `wsk` build -architecture should be for macOS, but of course this is the wrong version -when running later. It needs to be built for the Linux architecture. - -To use the script, it takes in 2 arguments: -1. (Required) The first argument is the Docker account to push the built images - to. For Nginx, it will tag the image as `account_name/whisk_nginx:latest` - and the OpenWhisk configuration image will be tagged - `account_name/whisk_config:dev-v1.5.6` and `account_name/whisk_config:dev-v1.6.2`. - - NOTE: **log into Docker** before running the script or it will - fail to properly upload the docker images. - -2. (Optional) The second argument is the location of where the - repo is installed locally. By default it assumes that this repo exists at - `$HOME/workspace/openwhisk`. - -If you plan on building your own images and would like to change from `danlavine's`, -then make sure to update the -[configure_whisk.yml](configure/configure_whisk.yml) and -[nginx](ansible-kube/environments/kube/files/nginx.yml) with your images. - -To run the script, use the command: - -``` -docker/build <Docker username> <(optional) openwhisk dir> -``` - -# Editing the Openwhisk Kube Deployment -## Kubernetes Deployments and Services - -The current Kube Deployment and Services files that define the OpenWhisk -cluster can be found [here](ansible-kube/environments/kube/files). Only one -instance of each OpenWhisk process is created, but if you would like -to increase that number, then this would be the place to do it. Simply edit -the appropriate file and -[Manually Build Custom Docker Files](#manually-building-custom-docker-files) - -# Development -## Debugging OpenWhisk on Kubernetes Configuration Pod - -When in the process of creating a new deployment, it is nice to -run things by hand to see what is going on inside the container and -not have it be removed as soon as it finishes or fails. For this, -you can change the command of [configure_whisk.yml](configure/configure_whisk.yml) -to `command: [ "tail", "-f", "/dev/null" ]`. Then just run the -original command from inside the Pod's container. - -# Limitations and Enhancements -## Limitations - -During the deployment process, OpenWhisk needs to generate a CA-cert -for Nginx and currently it has a static dns entry. Because of this, you -will need to connect to OpenWhisk using the insecure mode (e.g. `wsk -i`). -There is future work to make this CA-cert configurable. +# Limitations A couple of components for OpenWhisk on Kube deployment strategy requires custom built Docker images. One such component is Nginx and currently resides at @@ -260,22 +124,6 @@ built Docker images. One such component is Nginx and currently resides at is currently and open [issue](https://github.com/openwhisk/openwhisk/issues/2152) to make a public image and once it is resolved, then we can switch to the public image. -The second Docker image this deployment strategy relies on is the OpenWhisk -configuration image. For now, it is hosted at -[danlavine/whisk_config](https://hub.docker.com/r/danlavine/whisk_config/), -but ideally an official images can be built an maintained at some point. -If you would like to build your own deployment image, see -[Manually Build Custom Docker Files](#manually-building-custom-docker-files) - -Lastly, since OpenWhisk is configured/deployed via a Kubernetes Pod it requires -the correct kubectl version to be built into `danlavine/whisk_config`. For now, -there is only a version for Kube 1.5, and one can be built for 1.6, but there -is no CI to test it against at the moment. - -**Minikube (experimental)** -We also have experimental support for -* [Minikube](https://github.com/kubernetes/minikube), see the -* [Minikube-specific install instructions](/minikube/README.md) for more details. **Bad Kube versions** * Kube 1.6.3 has an issue with volume mount subpaths. See @@ -284,8 +132,6 @@ We also have experimental support for ## Enhancements -* Enable the configuration job to run any number of times. This way it updates an already running - OpenWhisk deployment on all subsequent runs * Use a public Edge Docker image once this [issue](https://github.com/apache/incubator-openwhisk/issues/2152) is resolved diff --git a/ansible-kube/environments/kube/files/db-service.yml b/ansible-kube/environments/kube/files/db-service.yml deleted file mode 100644 index ee1ed05..0000000 --- a/ansible-kube/environments/kube/files/db-service.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: couchdb - namespace: openwhisk - labels: - name: couchdb -spec: - selector: - name: couchdb - ports: - - port: 5984 - targetPort: 5984 - name: couchdb diff --git a/ansible-kube/environments/kube/files/db.yml b/ansible-kube/environments/kube/files/db.yml deleted file mode 100644 index 78de1af..0000000 --- a/ansible-kube/environments/kube/files/db.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: couchdb - namespace: openwhisk - labels: - name: couchdb -spec: - replicas: 1 - template: - metadata: - labels: - name: couchdb - spec: - restartPolicy: Always - - containers: - - name: couchdb - imagePullPolicy: IfNotPresent - image: couchdb:1.6 - ports: - - name: couchdb - containerPort: 5984 diff --git a/ansible-kube/environments/kube/files/nginx-service.yml b/ansible-kube/environments/kube/files/nginx-service.yml deleted file mode 100644 index c1758d3..0000000 --- a/ansible-kube/environments/kube/files/nginx-service.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: nginx - namespace: openwhisk - labels: - name: nginx -spec: - type: NodePort - selector: - name: nginx - ports: - - port: 80 - targetPort: 80 - name: http - - port: 443 - targetPort: 443 - name: https-api - - port: 8443 - targetPort: 8443 - name: https-admin diff --git a/ansible-kube/environments/kube/files/nginx.yml b/ansible-kube/environments/kube/files/nginx.yml deleted file mode 100644 index d9780b2..0000000 --- a/ansible-kube/environments/kube/files/nginx.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: nginx - namespace: openwhisk - labels: - name: nginx -spec: - replicas: 1 - template: - metadata: - labels: - name: nginx - spec: - restartPolicy: Always - volumes: - - name: nginx-certs - configMap: - name: nginx - items: - - key: "openwhisk-cert.pem" - path: openwhisk-cert.pem - - key: "openwhisk-key.pem" - path: openwhisk-key.pem - - name: nginx-conf - configMap: - name: nginx - items: - - key: "nginx.conf" - path: nginx.conf - - name: logs - emptyDir: {} - containers: - - name: nginx - imagePullPolicy: Always - image: danlavine/whisk_nginx - ports: - - name: http - containerPort: 80 - - name: http-api - containerPort: 443 - - name: https-admin - containerPort: 8443 - volumeMounts: - - name: nginx-conf - mountPath: "/etc/nginx/nginx.conf" - subPath: "nginx.conf" - - name: nginx-certs - mountPath: "/etc/nginx/certs" - - name: logs - mountPath: "/logs" diff --git a/ansible-kube/environments/kube/group_vars/all b/ansible-kube/environments/kube/group_vars/all deleted file mode 100644 index bc7b59e..0000000 --- a/ansible-kube/environments/kube/group_vars/all +++ /dev/null @@ -1,59 +0,0 @@ ---- -# general properties -kube_pod_dir: "{{ playbook_dir }}/environments/kube/files" -whisk_version_name: kube -config_root_dir: /tmp -whisk_logs_dir: /tmp/wsklogs - -# docker properties -docker_dns: "" -docker_registry: "" -docker_image_prefix: "openwhisk" - -# CouchDB properties -db_host: couchdb.openwhisk -db_provider: CouchDB -db_port: 5984 -db_protocol: http -db_username: couch_user -db_password: couch_password -db_auth: "subjects" -db_prefix: "openwhisk_kube_" -db_split_actions_and_activations: true - -# apigw db credentials minimum read/write -db_apigw_username: "couch_user" -db_apigw_password: "couch_password" -db_apigw: "openwhisk_kube_gwapis" -apigw_initdb: true - -# API GW connection configuration -apigw_auth_user: "" -apigw_auth_pwd: "" -apigw_host: "nginx.openwhisk" -apigw_host_v2: "nginx.openwhisk" -whisk_api_host_name: "nginx.openwhisk" - - -# consul properties -consul_host: consul.openwhisk -consul_conf_dir: /tmp/consul - -# nginx properties -nginx_conf_dir: /tmp/nginx -cli_nginx_dir: "/tmp/nginx/cli/go/download" -nginx_host: nginx.openwhisk - -# controller properties -controller_host: controller.openwhisk - -# kafka properties -kafka_host: kafka.openwhisk -zookeeper_host: zookeeper.openwhisk - -# invoker properties -invoker_port: 8080 - -# registry -registry_conf_dir: /tmp/registry -registry_storage_dir: "/" diff --git a/ansible-kube/environments/kube/hosts b/ansible-kube/environments/kube/hosts deleted file mode 100644 index a3e1d30..0000000 --- a/ansible-kube/environments/kube/hosts +++ /dev/null @@ -1,26 +0,0 @@ -; the first parameter in a host is the inventory_hostname which has to be -; either an ip -; or a resolvable hostname - -; used for local actions only -ansible ansible_connection=local -[edge] -127.0.0.1 ansible_connection=local - -[controllers] -127.0.0.1 ansible_connection=local - -[kafka] -127.0.0.1 ansible_connection=local - -[consul_servers] -127.0.0.1 ansible_connection=local - -[db] -127.0.0.1 ansible_connection=local - -[invokers] -127.0.0.1 ansible_connection=local - -[registry] -127.0.0.1 ansible_connection=local diff --git a/ansible-kube/openwhisk.yml b/ansible-kube/openwhisk.yml deleted file mode 100644 index 760c28e..0000000 --- a/ansible-kube/openwhisk.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# This playbook deploys an Openwhisk stack. -# It assumes you have already set up your database with the respective db provider playbook (currently cloudant.yml or couchdb.yml) -# It assumes that wipe.yml have being deployed at least once - -#- include: routemgmt.yml diff --git a/ansible-kube/roles/couchdb/tasks/deploy.yml b/ansible-kube/roles/couchdb/tasks/deploy.yml deleted file mode 100644 index c26c084..0000000 --- a/ansible-kube/roles/couchdb/tasks/deploy.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -# This role will run a CouchDB server on the db group - -- name: check if db credentials are valid for CouchDB - fail: msg="The db provider in your {{ inventory_dir }}/group_vars/all is {{ db_provider }}, it has to be CouchDB, pls double check" - when: db_provider != "CouchDB" - -- name: create db pod - shell: "kubectl apply -f {{kube_pod_dir}}/db.yml" - -- name: wait until the CouchDB in this host is up and running - wait_for: - delay: 2 - host: "{{ db_host }}" - port: "{{ db_port }}" - timeout: 60 - -- name: create admin user - uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/_config/admins/{{ db_username }}" - method: PUT - body: > - "{{ db_password }}" - body_format: json - status_code: 200 - -- name: disable reduce limit on views - uri: - url: "{{ db_protocol }}://{{ db_host }}:{{ db_port }}/_config/query_server_config/reduce_limit" - method: PUT - body: > - "false" - body_format: json - status_code: 200 - user: "{{ db_username }}" - password: "{{ db_password }}" - force_basic_auth: yes diff --git a/ansible-kube/roles/couchdb/tasks/main.yml b/ansible-kube/roles/couchdb/tasks/main.yml deleted file mode 100644 index 5169f94..0000000 --- a/ansible-kube/roles/couchdb/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# This role will deploy a database server. Use the role if you want to use CouchCB locally. -# In deploy mode it will start the CouchDB container. - -- include: deploy.yml - when: mode == "deploy" diff --git a/ansible-kube/roles/routemgmt/tasks/deploy.yml b/ansible-kube/roles/routemgmt/tasks/deploy.yml deleted file mode 100644 index d9858cf..0000000 --- a/ansible-kube/roles/routemgmt/tasks/deploy.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# Install the API Gateway route management actions. -- name: install route management actions - shell: ./installRouteMgmt.sh {{ catalog_auth_key }} {{ nginx_host }} {{ catalog_namespace }} {{ cli.path }} chdir="{{ openwhisk_home }}/ansible/roles/routemgmt/files" - environment: - OPENWHISK_HOME: "{{ openwhisk_home }}" diff --git a/ansible-kube/tasks/initdb.yml b/ansible-kube/tasks/initdb.yml deleted file mode 100644 index f20356d..0000000 --- a/ansible-kube/tasks/initdb.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# This task will initialize the immortal DBs in the database account. -# This step is usually done only once per deployment. - -- include: db/recreateDb.yml - vars: - dbName: "{{ db.whisk.auth }}" - forceRecreation: False - -- include: db/recreateDoc.yml - vars: - dbName: "{{ db.whisk.auth }}" - doc: "{{ lookup('file', '{{ item }}') }}" - with_items: - - "{{ openwhisk_home }}/ansible/files/auth_index.json" - - "{{ openwhisk_home }}/ansible/files/filter_design_document.json" - -- name: create necessary "auth" keys - include: db/recreateDoc.yml - vars: - key: "{{ lookup('file', 'files/auth.{{ item }}') }}" - dbName: "{{ db.whisk.auth }}" - doc: > - { - "_id": "{{ item }}", - "subject": "{{ item }}", - "uuid": "{{ key.split(":")[0] }}", - "key": "{{ key.split(":")[1] }}", - "namespaces": [] - } - with_items: "{{ db.authkeys }}" diff --git a/ansible-kube/tasks/installOpenwhiskCatalog.yml b/ansible-kube/tasks/installOpenwhiskCatalog.yml deleted file mode 100644 index 56b53f8..0000000 --- a/ansible-kube/tasks/installOpenwhiskCatalog.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# This task will install the standard actions and packages available in openwhisk-catalog repos. - -- set_fact: - catalog_location={{ item.value.location }} - catalog_repo_url={{ item.value.url }} - api_host={{ nginx_host }} - version="HEAD" - repo_update="yes" - -- set_fact: - version={{ item.value.version }} - when: item.value.version is defined - -- set_fact: - repo_update={{ item.value.repo_update }} - when: item.value.repo_update is defined - -- name: "ensure catalog_location directory exists" - file: - path: "{{ catalog_location }}" - state: directory - -- name: download the catalog repository to the catalog location if necessary - git: - repo: "{{ catalog_repo_url }}" - dest: "{{ catalog_location }}" - update: "{{ repo_update }}" - version: "{{ version }}" - -- name: install the catalog from the catalog location - shell: ./installCatalog.sh {{ catalog_auth_key }} {{ api_host }} {{ catalog_namespace }} {{ cli.path }} chdir="{{ catalog_location }}/packages" - environment: - OPENWHISK_HOME: "{{ openwhisk_home }}" diff --git a/configure/cleanup.sh b/configure/cleanup.sh index 8bbccde..5cc6ef3 100755 --- a/configure/cleanup.sh +++ b/configure/cleanup.sh @@ -4,9 +4,6 @@ set -x -# delete OpenWhisk configure job -kubectl -n openwhisk delete job configure-openwhisk - # delete deployments kubectl -n openwhisk delete deployment couchdb kubectl -n openwhisk delete deployment zookeeper diff --git a/configure/configure.sh b/configure/configure.sh deleted file mode 100755 index a305b05..0000000 --- a/configure/configure.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# this script is used to deploy OpenWhisk from a pod already running in -# kubernetes. -# -# Note: This pod assumes that there is an openwhisk namespace and the pod -# running this script has been created in that namespace. - -deployCouchDB() { - COUCH_DEPLOYED=$(kubectl -n openwhisk get pods --show-all | grep couchdb | grep "1/1") - - if [ -z "$COUCH_DEPLOYED" ]; then - return 0; - else - return 1; - fi -} - -set -ex - -# copy the ansible playbooks and tools to this repo -cp -R /openwhisk/ansible/ /incubator-openwhisk-deploy-kube/ansible -cp -R /openwhisk/tools/ /incubator-openwhisk-deploy-kube/tools -cp -R /openwhisk/bin/ /incubator-openwhisk-deploy-kube/bin - -mkdir -p /incubator-openwhisk-deploy-kube/core -cp -R /openwhisk/core/routemgmt /incubator-openwhisk-deploy-kube/core/routemgmt - -# overwrite the default openwhisk ansible with the kube ones. -cp -R /incubator-openwhisk-deploy-kube/ansible-kube/. /incubator-openwhisk-deploy-kube/ansible/ - -# start kubectl in proxy mode so we can talk to the Kube Api server -kubectl proxy -p 8001 & - -pushd /incubator-openwhisk-deploy-kube/ansible - ansible-playbook -i environments/kube setup.yml - - # Create all of the necessary services - kubectl apply -f environments/kube/files/db-service.yml - - if deployCouchDB; then - # Create and configure the CouchDB deployment - ansible-playbook -i environments/kube couchdb.yml - ansible-playbook -i environments/kube initdb.yml - ansible-playbook -i environments/kube wipe.yml - fi -popd diff --git a/configure/configure_whisk.yml b/configure/configure_whisk.yml deleted file mode 100644 index 4306f95..0000000 --- a/configure/configure_whisk.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: configure-openwhisk - namespace: openwhisk - labels: - name: configure-openwhisk -spec: - completions: 1 - template: - metadata: - labels: - name: config - spec: - restartPolicy: Never - containers: - - name: configure-openwhisk - image: danlavine/whisk_config:v1.5.6 - imagePullPolicy: Always - command: [ "/incubator-openwhisk-deploy-kube/configure/configure.sh" ] diff --git a/docker/build.sh b/docker/build.sh deleted file mode 100755 index 22a7297..0000000 --- a/docker/build.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -# This script can be used to build the custom docker images required -# for deploying openwhisk on Kubernetes. - -set -ex - -if [ -z "$1" ]; then -cat <<- EndOfMessage - First argument should be location of which docker repo to push all - of the built OpenWhisk docker images. This way, Kubernetes can pull - any images it needs to. -EndOfMessage - -exit 1 -fi - - -OPENWHISK_DIR="" -if [ -z "$2" ]; then -cat <<- EndOfMessage - Second argument should be location of the OpenWhisk repo on the local - file system. By default, it is assumed to be at $HOME/workspace/openwhisk. -EndOfMessage - OPENWHISK_DIR=$HOME/workspace/openwhisk -else - OPENWHISK_DIR="$2" -fi - - -SOURCE="${BASH_SOURCE[0]}" -SCRIPTDIR="$( dirname "$SOURCE" )" - -# build nginx -pushd $SCRIPTDIR/nginx - mkdir -p blackbox - pushd blackbox - # copy docker sdk to dockerSkeleton in scratch space - cp $OPENWHISK_DIR/sdk/docker/buildAndPush.sh . - cp $OPENWHISK_DIR/sdk/docker/Dockerfile . - cp $OPENWHISK_DIR/sdk/docker/example.c . - cp $OPENWHISK_DIR/sdk/docker/README.md . - - # rename base image in Dockerfile - sed -i "s|FROM dockerskeleton|FROM openwhisk/dockerskeleton|g" Dockerfile - - # fix file permissions - chmod 0755 buildAndPush.sh - - # build blackbox container artifact - tar -czf ../blackbox-0.1.0.tar.gz . - popd - - NGINX_IMAGE=$(docker build . | grep "Successfully built" | awk '{print $3}') - docker tag $NGINX_IMAGE "$1"/whisk_nginx - docker push "$1"/whisk_nginx - - # cleanup - rm blackbox-0.1.0.tar.gz - rm -rf blackbox -popd - -BuildKubeConfigureImage () { - pushd $SCRIPTDIR/.. - - WHISK_DEPLOY_IMAGE=$(docker build --build-arg KUBE_VERSION="$2" . | grep "Successfully built" | awk '{print $3}') - docker tag $WHISK_DEPLOY_IMAGE "$1"/whisk_config:"$2"-dev - docker push "$1"/whisk_config:"$2"-dev - - popd -} - -# build the OpenWhisk configure image -BuildKubeConfigureImage "$1" "v1.5.6" -BuildKubeConfigureImage "$1" "v1.6.2" - diff --git a/kubernetes/controller/controller.yml b/kubernetes/controller/controller.yml index f91ef34..496879d 100644 --- a/kubernetes/controller/controller.yml +++ b/kubernetes/controller/controller.yml @@ -38,7 +38,7 @@ spec: # Update this image to the publix OpenWhisk Image once this PR is merged. # https://github.com/apache/incubator-openwhisk/pull/2452 image: danlavine/whisk_controller - command: ["/bin/bash", "-c", "/controller/bin/controller `hostname | cut -d'-' -f2`"] + command: ["/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /controller/bin/controller `hostname | cut -d'-' -f2`"] ports: - name: controller containerPort: 8080 @@ -53,16 +53,6 @@ spec: env: - name: "PORT" value: "8080" - - name: "COMPONENT_NAME" - value: "controller" - - name: "KAFKA_NUMPARTITIONS" - value: "2" - - name: "SERVICE_CHECK_HTTP" - value: "/ping" - - name: "SERVICE_CHECK_TIMEOUT" - value: "2s" - - name: "SERVICE_CHECK_INTERVAL" - value: "15s" # Loadbalancer options - name: "LOADBALANCER_INVOKERBUSYTHRESHOLD" @@ -105,9 +95,9 @@ spec: # properties for DB connection - name: "DB_USERNAME" - value: "couch_user" + value: "whisk_admin" - name: "DB_PASSWORD" - value: "couch_password" + value: "some_passw0rd" - name: "DB_PORT" value: "5984" - name: "DB_PROTOCOL" @@ -117,8 +107,8 @@ spec: - name: "DB_PROVIDER" value: "CouchDB" - name: "DB_WHISK_ACTIVATIONS" - value: "openwhisk_kube_activations" + value: "test_activations" - name: "DB_WHISK_ACTIONS" - value: "openwhisk_kube_whisks" + value: "test_whisks" - name: "DB_WHISK_AUTHS" - value: "openwhisk_kube_subjects" + value: "test_subjects" diff --git a/kubernetes/couchdb/README.md b/kubernetes/couchdb/README.md new file mode 100644 index 0000000..642a76d --- /dev/null +++ b/kubernetes/couchdb/README.md @@ -0,0 +1,49 @@ +CouchDB +----- + +# Deploying + +To deploy CouchDB, you first need to create the CouchDB +Pod. This can be done by running: + +``` +kubectl apply -f couchdb.yml +``` + +This pod goes through the process of pulling the OpenWhisk +repo and running through some of the ansible playbooks for +configuring CouchDB. + +**NOTE** the pod will say running as soon as the start command +runs, but it does not actually mean that the DB is ready to use. +This is because it might not yet be configured. To check if the +DB has been setup, you can look at the Pod logs with + +``` + export COUCH_DB_POD=$(kubectl -n openwhisk get pods -o wide --show-all | grep "couchdb" | awk '{print $1}') + kubectl -n openwhisk logs $COUCH_DB_POD +``` + +In the logs, you should see the line: + +``` +Apache CouchDB has started on http://0.0.0.0:5984 +``` + +This indicates that the CouchDB instancs is up and running. + +# Configuring CouchDB +## Usernames and Passwords + +To configure custom usernames and passwords, you can edit +the CouchDB [setup pod](https://github.com/apache/incubator-openwhisk-deploy-kube/blob/master/kubernetes/couchdb/couchdb.yml#L46-L49). + +**NOTE** If the CouchDB username and password properties +are updated, then you will need to update the Controller +and Invoker yamls with updated username and password. + +## Persistance + +To create a persistant CouchDB instance, you will need +to create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +for the [couchdb.yml](couchdb.yml). diff --git a/kubernetes/couchdb/couchdb.yml b/kubernetes/couchdb/couchdb.yml new file mode 100644 index 0000000..6fd3117 --- /dev/null +++ b/kubernetes/couchdb/couchdb.yml @@ -0,0 +1,51 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: couchdb + namespace: openwhisk + labels: + name: couchdb +spec: + selector: + name: couchdb + ports: + - port: 5984 + targetPort: 5984 + name: couchdb + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: couchdb + namespace: openwhisk + labels: + name: couchdb +spec: + replicas: 1 + template: + metadata: + labels: + name: couchdb + spec: + restartPolicy: Always + containers: + - name: couchdb + imagePullPolicy: IfNotPresent + image: danlavine/whisk_couchdb + command: ["/init.sh"] + ports: + - name: couchdb + containerPort: 5984 + env: + - name: "DB_HOST" + value: "couchdb.openwhisk" + - name: "DB_PREFIX" + value: "test_" + - name: "DB_USERNAME" + value: "whisk_admin" + - name: "DB_PASSWORD" + value: "some_passw0rd" + - name: "DB_PORT" + value: "5984" diff --git a/kubernetes/couchdb/docker/Dockerfile b/kubernetes/couchdb/docker/Dockerfile new file mode 100644 index 0000000..0011ef0 --- /dev/null +++ b/kubernetes/couchdb/docker/Dockerfile @@ -0,0 +1,14 @@ +FROM couchdb:1.6 + +RUN apt-get -y update && apt-get -y install \ + git \ + curl \ + sudo \ + python-dev \ + libffi-dev \ + libssl-dev + +COPY init.sh /init.sh +RUN chmod +X /init.sh + +CMD ["/init.sh"] diff --git a/kubernetes/couchdb/docker/init.sh b/kubernetes/couchdb/docker/init.sh new file mode 100755 index 0000000..af73049 --- /dev/null +++ b/kubernetes/couchdb/docker/init.sh @@ -0,0 +1,93 @@ +#!/bin/bash +set -ex + +# Always clone the latest version of OpenWhisk +git clone https://github.com/apache/incubator-openwhisk /openwhisk + +pushd /openwhisk + # Install ansible requirements + ./tools/ubuntu-setup/pip.sh + + # upgrade cffi for ansible error on Debian Jesse + pip install --upgrade cffi + sudo pip install markupsafe + sudo pip install ansible==2.3.0.0 + + + # if auth guest overwrite file + if [ -n "$AUTH_GUEST" ]; then + echo "$AUTH_GUEST" > /openwhisk/ansible/files/auth.guest + fi + + # if auth whisk system overwrite file + if [ -n "$AUTH_WHISK_SYSTEM" ]; then + echo "$AUTH_WHISK_SYSTEM" > /openwhisk/ansible/files/auth.whisk.system + fi + + # start couchdb with a background process + couchdb -b + + # wait for couchdb to be up and running + TIMEOUT=0 + echo "wait for CouchDB to be up and running" + until [ $TIMEOUT -eq 25 ]; do + echo "waiting for CouchDB to be available" + + if [ -n $(/etc/init.d/couchdb status | grep 'running') ]; then + echo "CouchDB is up and running" + break + fi + + sleep 0.2 + let TIMEOUT=TIMEOUT+1 + done + + if [ $TIMEOUT -eq 25 ]; then + echo "failed to setup CouchDB" + exit 1 + fi + + + # setup and initialize DB + pushd ansible + ansible-playbook -i environments/local setup.yml \ + -e db_host=$DB_HOST \ + -e db_prefix=$DB_PREFIX \ + -e db_username=$DB_USERNAME \ + -e db_password=$DB_PASSWORD \ + -e db_port=$DB_PORT \ + -e openwhisk_home=/openwhisk + popd + + # create the admin user + curl -X PUT http://$DB_HOST:$DB_PORT/_config/admins/$DB_USERNAME -d "\"$DB_PASSWORD\"" + + # disable reduce limits on views + curl -X PUT http://$DB_USERNAME:$DB_PASSWORD@$DB_HOST:$DB_PORT/_config/query-server_config/reduce_limit -d '"false"' + + pushd ansible + # initialize the DB + ansible-playbook -i environments/local initdb.yml \ + -e db_host=$DB_HOST \ + -e db_prefix=$DB_PREFIX \ + -e db_username=$DB_USERNAME \ + -e db_password=$DB_PASSWORD \ + -e db_port=$DB_PORT \ + -e openwhisk_home=/openwhisk + + # wipe the DB + ansible-playbook -i environments/local wipe.yml \ + -e db_host=$DB_HOST \ + -e db_prefix=$DB_PREFIX \ + -e db_username=$DB_USERNAME \ + -e db_password=$DB_PASSWORD \ + -e db_port=$DB_PORT \ + -e openwhisk_home=/openwhisk + popd + + # stop the CouchDB background process + couchdb -d +popd + +# start couchdb that has been setup +tini -s -- couchdb diff --git a/kubernetes/invoker/README.md b/kubernetes/invoker/README.md index 69308df..15d6d6e 100644 --- a/kubernetes/invoker/README.md +++ b/kubernetes/invoker/README.md @@ -40,6 +40,19 @@ To deploy an Invoker to specific Kube nodes, you will need to edit the file with Kubernetes [NodeSelectors](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/). # Troubleshooting +## Deploying to Minikube + +When deploying the Invoker to [Minikube](https://kubernetes.io/docs/getting-started-guides/minikube/) +you might need to edit the Invoker's Docker Api Version. +This is because Minikube uses Docker version 1.11.x. +To do this, you will need to add the following properties +to the invoker.yml file. + +``` +env: + - name: "DOCKER_API_VERSION" + value: "1.23" +``` ## Kubernetes Host Linux Versions Unfortunitaly when Deploying OpenWhisk on Kubernetes it currently mounts some diff --git a/kubernetes/invoker/invoker.yml b/kubernetes/invoker/invoker.yml index fc7f79c..fd20754 100644 --- a/kubernetes/invoker/invoker.yml +++ b/kubernetes/invoker/invoker.yml @@ -99,9 +99,9 @@ spec: # properties for DB connection - name: "DB_USERNAME" - value: "couch_user" + value: "whisk_admin" - name: "DB_PASSWORD" - value: "couch_password" + value: "some_passw0rd" - name: "DB_PORT" value: "5984" - name: "DB_PROTOCOL" @@ -111,11 +111,11 @@ spec: - name: "DB_PROVIDER" value: "CouchDB" - name: "DB_WHISK_ACTIVATIONS" - value: "openwhisk_kube_activations" + value: "test_activations" - name: "DB_WHISK_ACTIONS" - value: "openwhisk_kube_whisks" + value: "test_whisks" - name: "DB_WHISK_AUTHS" - value: "openwhisk_kube_subjects" + value: "test_subjects" ports: - name: invoker containerPort: 8080 diff --git a/kubernetes/kafka/README.md b/kubernetes/kafka/README.md index 954058e..cf91aaf 100644 --- a/kubernetes/kafka/README.md +++ b/kubernetes/kafka/README.md @@ -38,3 +38,24 @@ Kafka, Controller and Nginx deployments. * Controller: See the Controller [README](https://github.com/apache/incubator-openwhisk-deploy-kube/blob/master/kubernetes/controller/README.md) * Nginx: See the Nginx [README](https://github.com/apache/incubator-openwhisk-deploy-kube/blob/master/kubernetes/nginx/README.md#increase-controller-count) + +# Troubleshooting +## Networking errors + +When inspecting kafka logs of various components and they are not able to +send/receive message then Kafka is the usual problem. There are issues +when Kube Pods cannot communicate with themselves over a Kube Service. +Setting a network to promiscous mode can be the solution will enable network +traffic to route in a loop back to itself. E.g: + +``` +ip link set docker0 promisc on +``` + +**NOTE** The `docker0` network in the example above is the Pod network. +If you were using a CNI, then you would need to upgrade the CNI netowrk. + +These fixes are of course only temporary fixes that can be used +when developing OpenWhisk on Kube. To deploy Kubernetes without the +need for for setting the network up with this manual fix, you need +to setup the Kubelet with `--hairpin-mode`. diff --git a/kubernetes/nginx/README.md b/kubernetes/nginx/README.md index a425927..ff0dae4 100644 --- a/kubernetes/nginx/README.md +++ b/kubernetes/nginx/README.md @@ -102,4 +102,21 @@ file has routes for Controller [StatefulSet][StatefulSet] addresses. Specifically [these lines](https://github.com/apache/incubator-openwhisk-deploy-kube/tree/master/kubernetes/nginx/nginx.conf#L15-L20). will need to be updated with a list of all available routes. +# Create Nginx Docker Image + +To build the Nginx docker image for Kubernetes on OpenWhisk, +you will need to run the build script [build.sh](docker/build.sh). +This script requires one parameter, which is the repo to bush +the Docker image to. + +E.G +``` +docker/builds.sh <danlavine> +``` + +This script goes through and donwload the OpenWhisk reop under the +tmp directory, builds the Blackbox image and copies it into the +Docker image. Then, each of the published WSK CLIs are download into +the Docker image so that users are able to download them as usual. + [StatefulSet]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ diff --git a/docker/nginx/Dockerfile b/kubernetes/nginx/docker/Dockerfile similarity index 100% rename from docker/nginx/Dockerfile rename to kubernetes/nginx/docker/Dockerfile diff --git a/kubernetes/nginx/docker/build.sh b/kubernetes/nginx/docker/build.sh new file mode 100755 index 0000000..8e3f07d --- /dev/null +++ b/kubernetes/nginx/docker/build.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# This script can be used to build Nginx +# used by OpenWhisk on Kubernetes. + +set -ex + +if [ -z "$1" ]; then +cat <<- EndOfMessage + First argument should be location of which docker repo to push all + of the built OpenWhisk docker images. This way, Kubernetes can pull + any images it needs to. +EndOfMessage + +exit 1 +fi + +git clone https://github.com/apache/incubator-openwhisk /tmp/openwhisk + +# build nginx +mkdir -p blackbox +pushd blackbox + # copy docker sdk to dockerSkeleton in scratch space + cp /tmp/openwhisk/sdk/docker/buildAndPush.sh . + cp /tmp/openwhisk/sdk/docker/Dockerfile . + cp /tmp/openwhisk/sdk/docker/example.c . + cp /tmp/openwhisk/sdk/docker/README.md . + + # rename base image in Dockerfile + sed -i "s|FROM dockerskeleton|FROM openwhisk/dockerskeleton|g" Dockerfile + + # fix file permissions + chmod 0755 buildAndPush.sh + + # build blackbox container artifact + tar -czf ../blackbox-0.1.0.tar.gz . +popd + +NGINX_IMAGE=$(docker build . --no-cache | grep "Successfully built" | awk '{print $3}') +docker tag $NGINX_IMAGE "$1"/whisk_nginx +docker push "$1"/whisk_nginx + +# cleanup +rm blackbox-0.1.0.tar.gz +rm -rf blackbox diff --git a/minikube/README.md b/minikube/README.md deleted file mode 100644 index d15d10f..0000000 --- a/minikube/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Deploying OpenWhisk to Minikube - -**Note:** this is currently experimental, and the integration tests -don't yet run against Minikube on CI. Minikube currently uses an old -version of Docker (v1.11.x), so may not be fully compatible with -OpenWhisk (see [Requirements](../README.md#requirements) for more -info). These steps have only been manually tested with Minikube -v0.19.1. - - -# Installation - -1. Download and set up [Minikube](https://github.com/kubernetes/minikube) -2. Start minikube with `minikube start` -3. run `./deploy_minikube.sh`. This will configure Minikube for - OpenWhisk, run the standard Kubernetes OpenWhisk deployment, then - reconfigure the Invoker so it can properly talk to the Docker - daemon used by Minikube - -Once you've started the deploy script, you can follow along on the -progress of the configuration task as noted in the -[top-level README](../README.md#configure-openwhisk). - - -# The details - -There are currently two changes required in order for OpenWhisk to -work on Minikube, both of which are performed by the deploy script: - -1. The `docker0` interface inside Minikube needs to be put into - promiscuous mode to allow Kafka to talk to ZooKeeper -2. The Docker client used by the Invoker is newer than the Docker - daemon provided by Minikube, so we have to set an environment - variable (`DOCKER_API_VERSION`) in the environment for the - Invoker's container (via the StatefulSet template) to force it to - use an older version of the API protocol (1.23). We then need to - delete the existing Invoker pod that was started by the initial - deploy to force a new one to be created with the new environment. - - - diff --git a/minikube/ci/build.sh b/minikube/ci/build.sh deleted file mode 100755 index 0dbb5ef..0000000 --- a/minikube/ci/build.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -set -ex - -SCRIPTDIR=$(cd $(dirname "$0") && pwd) -ROOTDIR="$SCRIPTDIR/../" - -cd $ROOTDIR - -# TODO: need official repo -# build openwhisk images -# This way everything that is tested will use the latest openwhisk builds - -./deploy_minikube.sh - -sleep 5 - -CONFIGURE_POD=$(kubectl get pods --all-namespaces -o wide | grep configure | awk '{print $2}') - -PASSED=false -TIMEOUT=0 -until $PASSED || [ $TIMEOUT -eq 25 ]; do - KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get jobs | grep configure-openwhisk | awk '{print $3}') - if [ $KUBE_DEPLOY_STATUS -eq 1 ]; then - PASSED=true - break - fi - - kubectl get pods --all-namespaces -o wide --show-all - - let TIMEOUT=TIMEOUT+1 - sleep 30 -done - -if [ "$PASSED" = false ]; then - kubectl -n openwhisk logs $CONFIGURE_POD - kubectl get jobs --all-namespaces -o wide --show-all - kubectl get pods --all-namespaces -o wide --show-all - - echo "The job to configure OpenWhisk did not finish with an exit code of 1" - exit 1 -fi - -echo "The job to configure OpenWhisk finished successfully" - -# Don't try and perform wsk actions the second it finishes deploying. -# The CI ocassionaly fails if you perform actions to quickly. -sleep 30 - -AUTH_SECRET=$(kubectl -n openwhisk get secret openwhisk-auth-tokens -o yaml | grep 'auth_whisk_system:' | awk '{print $2}' | base64 --decode) -WSK_PORT=$(kubectl -n openwhisk describe service nginx | grep https-api | grep NodePort| awk '{print $3}' | cut -d'/' -f1) -WSK_HOST=$(minikube ip) -WSK_URL="https://$WSK_HOST:$WSK_PORT" - -# download the wsk cli from nginx -wget --no-check-certificate $WSK_URL/cli/go/download/linux/amd64/wsk -chmod +x wsk - -# setup the wsk cli -./wsk property set --auth $AUTH_SECRET --apihost $WSK_URL - -# create wsk action -cat > hello.js << EOL -function main() { - return {payload: 'Hello world'}; -} -EOL - -./wsk -i action create hello hello.js - - -sleep 5 - -# run the new hello world action -RESULT=$(./wsk -i action invoke --blocking hello | grep "\"status\": \"success\"") - -if [ -z "$RESULT" ]; then - echo "FAILED! Could not invoked custom action" - exit 1 -fi - -echo "PASSED! Deployed openwhisk and invoked custom action" - -# push the images to an official repo diff --git a/minikube/ci/setup.sh b/minikube/ci/setup.sh deleted file mode 100755 index c1888ef..0000000 --- a/minikube/ci/setup.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Installs and starts minikube - -set -x - -# download kubectl -wget https://storage.googleapis.com/kubernetes-release/release/v1.5.4/bin/linux/amd64/kubectl -chmod +x kubectl -sudo mv kubectl /usr/local/bin/kubectl - -# download minikube -wget https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 -chmod +x minikube-linux-amd64 -sudo mv minikube-linux-amd64 /usr/local/bin/minikube -minikube version - -# assumes virtualbox, override with --vm-driver virtualbox|kvm|xhyve|vmwarefusion -minikube start - -# Wait until minikube is up and running -TIMEOUT=0 -TIMEOUT_COUNT=40 -until $(minikube status &> /dev/null) || [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do - echo "Minikube is not up yet" - let TIMEOUT=TIMEOUT+1 - sleep 20 -done - -if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then - echo "Minikube is not up and running" - exit 1 -fi - -echo "Minikube is ready" - - - diff --git a/minikube/deploy_minikube.sh b/minikube/deploy_minikube.sh deleted file mode 100755 index e27f429..0000000 --- a/minikube/deploy_minikube.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -DIR=$( cd "$( dirname "$0" )" && pwd ) - -echo "Enabling promisc mode in minikube" -minikube ssh sudo ip link set docker0 promisc on - -echo "Launching configure job" -kubectl apply -f $DIR/../configure/openwhisk_kube_namespace.yml -kubectl apply -f $DIR/../configure/configure_whisk.yml - - -printf "Waiting for invoker StatefulSet to exist" -TIMEOUT=0 -TIMEOUT_COUNT=100 -until $(kubectl -n openwhisk get statefulset invoker &> /dev/null) || [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do - printf "." - let TIMEOUT=TIMEOUT+1 - sleep 5 -done -echo - -if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then - echo "Gave up waiting for the invoker StatefulSet" - exit 1 -fi - -echo "Patching the invoker StatefulSet to downgrade the docker API version" -kubectl -n openwhisk patch statefulset invoker --type=json -p '[{"op": "add", "path": "/spec/template/spec/containers/0/env/-", "value": {"name": "DOCKER_API_VERSION", "value": "1.23"}}]' - -echo "Deleting existing invoker pod so it will get recreated with new env" -kubectl -n openwhisk delete po/invoker-0 - -CONFIGURE_POD=$(kubectl get pods --all-namespaces -o wide | grep configure | awk '{print $2}') - -printf "Waiting for the configure job to complete" -PASSED=false -TIMEOUT=0 -TIMEOUT_COUNT=25 -until $PASSED || [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do - KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get jobs | grep configure-openwhisk | awk '{print $3}') - if [ $KUBE_DEPLOY_STATUS -eq 1 ]; then - PASSED=true - break - fi - printf "." - let TIMEOUT=TIMEOUT+1 - sleep 30 -done -echo - -if [ "$PASSED" = false ]; then - echo "The job to configure OpenWhisk did not finish successfully" - echo "Execute \`kubectl -n openwhisk logs $CONFIGURE_POD\` to see the output from the configure job" - exit 1 -fi - -port=$(kubectl -n openwhisk describe service nginx | grep https-api | grep NodePort | awk '{print $3}' | cut -d'/' -f1) -url="https://$(minikube ip):$port" - -echo "OpenWhisk should now be available at $url" -- To stop receiving notification emails like this one, please contact ['"commits@openwhisk.apache.org" <commits@openwhisk.apache.org>'].