rabbah closed pull request #145: Use YAML files deploy kafka package to OpenWhisk on K8s URL: https://github.com/apache/incubator-openwhisk-deploy-kube/pull/145
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/docker/kafkapkg-installer/Dockerfile b/docker/kafkapkg-installer/Dockerfile new file mode 100644 index 0000000..b1960db --- /dev/null +++ b/docker/kafkapkg-installer/Dockerfile @@ -0,0 +1,11 @@ +from ubuntu:latest + +RUN apt-get -y update && apt-get -y install \ + git \ + wget \ + zip + +COPY init.sh /init.sh +RUN chmod +x /init.sh + +CMD ["/init.sh"] diff --git a/docker/kafkapkg-installer/init.sh b/docker/kafkapkg-installer/init.sh new file mode 100644 index 0000000..2c30a7c --- /dev/null +++ b/docker/kafkapkg-installer/init.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -ex + +git clone https://github.com/apache/incubator-openwhisk-package-kafka.git + +# TODO: installxxxCatalog.sh wants OPENWHISK_HOME set, but doesn't actually need +# it for anything. Fix upstream and then remove this. +export OPENWHISK_HOME=/openwhisk +export DB_URL=http://$DB_USERNAME:$DB_PASSWORD@$DB_HOST:$DB_PORT +mkdir -p $OPENWHISK_HOME/bin + +# Download and install openwhisk cli +pushd $OPENWHISK_HOME/bin + wget -q https://github.com/apache/incubator-openwhisk-cli/releases/download/latest/OpenWhisk_CLI-latest-linux-amd64.tgz + tar xzf OpenWhisk_CLI-latest-linux-amd64.tgz +popd + +pushd /incubator-openwhisk-package-kafka + ./installKafka.sh $AUTH $APIHOST $DB_URL $DB_PREFIX $APIHOST + ./installCatalog.sh $AUTH $APIHOST $DB_URL $DB_PREFIX $APIHOST +popd + +echo "successfully setup kafka package" + diff --git a/kubernetes/package-kafka/README.md b/kubernetes/package-kafka/README.md new file mode 100644 index 0000000..792e551 --- /dev/null +++ b/kubernetes/package-kafka/README.md @@ -0,0 +1,43 @@ +# Deploy kafka package to Apache OpenWhisk + +This project is to deploy kafka package to local Apache OpenWhisk on a K8s using YAML file. + +## Prerequisite +Edit package-kafka.env as needed to set the appropriate values for your deployment, then create the configmap packages.kafkaprovider: +``` +kubectl -n openwhisk create cm packages.kafkaprovider --from-literal=kafkapkg_db_prefix=mq +``` + +The deployment also has dependencies to secret `whisk.auth` and `db.auth`, and configmap `whisk.ingress`. Make sure you have these settings before you start the deployment. + +## Step 1. Install kafka provider +``` +kubectl apply -f kafkaprovider.yml +``` + +## Step 2. Install messaging package to your local Apache OpenWhisk +``` +kubectl apply -f kafkapkginstaller.yml +``` + +## Verify your Kafka package +Get the description of your Kafka package by: +``` +wsk package get /whisk.system/messaging --summary -i +``` +Create a kafka package binding: +``` +wsk package bind /whisk.system/messaging myKafkaPkg -p brokers "[\"kafka_host1:9093\", \"kafka_host2:9093\"]" -i +``` +Create a trigger: +``` +wsk trigger create MyKafkaTrigger -f myKafkaPkg/kafkaFeed -p topic in-topic -i +``` +Send a message to kafka topic by invoking the action `kafkaProduce`: +``` +wsk action invoke myKafkaPkg/kafkaProduce -p topic in-topic -p value "this is a message" -i +``` +Check activation log to see `MyKafkaTrigger` is triggered when a new message is sent. +``` +wsk activation poll -i +``` diff --git a/kubernetes/package-kafka/kafkapkginstaller.yml b/kubernetes/package-kafka/kafkapkginstaller.yml new file mode 100644 index 0000000..6b10dc4 --- /dev/null +++ b/kubernetes/package-kafka/kafkapkginstaller.yml @@ -0,0 +1,48 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kafkapkginstaller + namespace: openwhisk + labels: + name: kafkapkginstaller +spec: + template: + metadata: + labels: + name: kafkapkginstaller + spec: + restartPolicy: Never + containers: + - name: kafkapkginstaller + imagePullPolicy: IfNotPresent + image: openwhisk/kube-kafkapkginstaller + env: + - name: "APIHOST" + valueFrom: + configMapKeyRef: + name: whisk.ingress + key: api_host + - name: "DB_HOST" + value: "$(COUCHDB_SERVICE_HOST)" + - name: "DB_PORT" + value: "$(COUCHDB_SERVICE_PORT_COUCHDB)" + - name: "DB_USERNAME" + valueFrom: + secretKeyRef: + name: db.auth + key: db_username + - name: "DB_PASSWORD" + valueFrom: + secretKeyRef: + name: db.auth + key: db_password + - name: "DB_PREFIX" + valueFrom: + configMapKeyRef: + name: packages.kafkaprovider + key: kafkapkg_db_prefix + - name: "AUTH" + valueFrom: + secretKeyRef: + name: whisk.auth + key: system diff --git a/kubernetes/package-kafka/kafkaprovider.yml b/kubernetes/package-kafka/kafkaprovider.yml new file mode 100644 index 0000000..db9c51a --- /dev/null +++ b/kubernetes/package-kafka/kafkaprovider.yml @@ -0,0 +1,54 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kafkaprovider + namespace: openwhisk + labels: + name: kafkaprovider +spec: + replicas: 1 + template: + metadata: + labels: + name: kafkaprovider + access: db + spec: + restartPolicy: Always + containers: + - name: kafkaprovider + imagePullPolicy: IfNotPresent + image: openwhisk/kafkaprovider + ports: + - name: kafkaprovider + containerPort: 8080 + env: + - name: "DB_URL" + value: "http://$(COUCHDB_SERVICE_HOST):$(COUCHDB_SERVICE_PORT_COUCHDB)" + - name: "DB_USER" + valueFrom: + secretKeyRef: + name: db.auth + key: db_username + - name: "DB_PASS" + valueFrom: + secretKeyRef: + name: db.auth + key: db_password + - name: "DB_PREFIX" + valueFrom: + configMapKeyRef: + name: packages.kafkaprovider + key: kafkapkg_db_prefix + - name: "LOCAL_DEV" + value: "true" + - name: "ROUTER_HOST" + valueFrom: + configMapKeyRef: + name: whisk.ingress + key: api_host + - name: "ENDPOINT_AUTH" + valueFrom: + secretKeyRef: + name: whisk.auth + key: system diff --git a/tools/travis/build.sh b/tools/travis/build.sh index 35cb378..5b2dd7b 100755 --- a/tools/travis/build.sh +++ b/tools/travis/build.sh @@ -250,6 +250,15 @@ pushd kubernetes/openwhisk-catalog jobHealthCheck "install-catalog" popd +# install package-kafka +echo "Installing kafka package" +pushd kubernetes/package-kafka + kubectl -n openwhisk create cm packages.kafkaprovider --from-literal=kafkapkg_db_prefix=mq + kubectl apply -f kafkaprovider.yml + kubectl apply -f kafkapkginstaller.yml + jobHealthCheck "kafkapkginstaller" +popd + # list packages and actions now installed in /whisk.system wsk -i --auth `cat kubernetes/cluster-setup/auth.whisk.system` package list wsk -i --auth `cat kubernetes/cluster-setup/auth.whisk.system` action list diff --git a/tools/travis/collect-logs.sh b/tools/travis/collect-logs.sh index 0798c5f..020626f 100755 --- a/tools/travis/collect-logs.sh +++ b/tools/travis/collect-logs.sh @@ -20,8 +20,10 @@ kubectl -n openwhisk logs controller-1 >& logs/controller-1.log kubectl -n openwhisk logs -lname=invoker -c docker-pull-runtimes >& logs/invoker-docker-pull.log kubectl -n openwhisk logs -lname=invoker -c invoker >& logs/invoker-invoker.log kubectl -n openwhisk logs -lname=nginx >& logs/nginx.log +kubectl -n openwhisk logs -lname=kafkaprovider >& logs/kafkaprovider.log kubectl -n openwhisk logs jobs/install-routemgmt >& logs/routemgmt.log kubectl -n openwhisk logs jobs/install-catalog >& logs/catalog.log +kubectl -n openwhisk logs jobs/kafkapkginstaller >& logs/kafkapkginstaller.log kubectl get pods --all-namespaces -o wide --show-all >& logs/all-pods.txt # System level logs from minikube diff --git a/tools/travis/deploy.sh b/tools/travis/deploy.sh index 11d75a0..7b2606a 100755 --- a/tools/travis/deploy.sh +++ b/tools/travis/deploy.sh @@ -18,3 +18,6 @@ echo "Publishing kube-openwhisk-catalog image" echo "Publishing kube-routemgmt image" ./tools/travis/publish.sh openwhisk kube-routemgmt latest docker/routemgmt + +echo "Publishing kube-kafkapkginstaller image" +./tools/travis/publish.sh openwhisk kube-kafkapkginstaller latest docker/kafapkg-installer ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
