This is an automated email from the ASF dual-hosted git repository.
zike pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 05844a5be Add Pulsar's Messaging Layer to a Helm Deployment (#1820)
05844a5be is described below
commit 05844a5bead2a328047cae2220b01d8b0b11256d
Author: zhenyu luo <[email protected]>
AuthorDate: Sat Aug 12 20:10:55 2023 +0800
Add Pulsar's Messaging Layer to a Helm Deployment (#1820)
* Add Pulsar's Messaging Layer to a Helm Deployment
* Add environment variables for backend deployment
---
installer/k8s/README.md | 14 +++++
.../k8s/templates/core/backend-deployment.yaml | 24 ++++++++-
.../pulsar/pulsar-deployment.yml} | 63 +++++++++++-----------
.../k8s/templates/external/pulsar/pulsar-pvc.yml | 44 +++++++++++++++
.../templates/external/pulsar/pulsar-service.yml | 29 ++++++++++
installer/k8s/values.yaml | 12 +++++
6 files changed, 152 insertions(+), 34 deletions(-)
diff --git a/installer/k8s/README.md b/installer/k8s/README.md
index 779b8174f..135236c44 100644
--- a/installer/k8s/README.md
+++ b/installer/k8s/README.md
@@ -237,6 +237,20 @@ rm -rf ${HOME}/streampipes-k8s
| external.zookeeper.persistence.claimName | Name of the ZooKeeper
PersistentVolumeClaim | "zookeeper-pvc" |
| external.zookeeper.persistence.pvName | Name of the ZooKeeper
PersistentVolume | "zookeeper-pv" |
+
+####Pulsar common parameters
+| Parameter Name | Description
| Value |
+|-------------------------------------------------|----------------------------------------------------------|------------------------------------------|
+| external.pulsar.appName | pulsar application name
| "pulsar" |
+| external.pulsar.version | pulsar version
| 3.0.0 |
+| external.pulsar.port | Port for the pulsar
service | 6650
|
+| external.pulsar.service.name | Name of the pulsar service
| "pulsar" |
+| external.pulsar.service.port | TargetPort of the pulsar
service | 6650 |
+| external.pulsar.persistence.storageClassName | Storage class name for
pulsar PVs | "hostpath" |
+| external.pulsar.persistence.storageSize | Size of the pulsar PV
| "1Gi" |
+| external.pulsar.persistence.claimName | Name of the pulsar
PersistentVolumeClaim | "pulsar-pvc"
|
+| external.pulsar.persistence.pvName | Name of the pulsar
PersistentVolume | "pulsar-pv"
|
+
## Bugs and Feature Requests
If you've found a bug or have a feature that you'd love to see in StreamPipes,
feel free to create an issue on
[GitHub](https://github.com/apache/streampipes/issues).
diff --git a/installer/k8s/templates/core/backend-deployment.yaml
b/installer/k8s/templates/core/backend-deployment.yaml
index 678828ef9..abb166c0c 100644
--- a/installer/k8s/templates/core/backend-deployment.yaml
+++ b/installer/k8s/templates/core/backend-deployment.yaml
@@ -46,7 +46,29 @@ spec:
imagePullPolicy: {{ .Values.pullPolicy }}
env:
- name: SP_PRIORITIZED_PROTOCOL
- value: {{ .Values.preferredBroker }}
+ {{- if eq .Values.preferredBroker "pulsar" }}
+ value: "pulsar"
+ - name: SP_PULSAR_URL
+ value: "pulsar://{{ .Values.external.pulsar.service.name }}:{{
.Values.external.pulsar.service.port }}"
+ {{- end }}
+ {{- if eq .Values.preferredBroker "kafka" }}
+ value: "kafka"
+ - name: SP_KAFKA_HOST
+ value: "{{ .Values.external.kafka.service.name }}"
+ - name: SP_KAFKA_PORT
+ value: "{{ .Values.external.kafka.service.port }}"
+ - name: SP_ZOOKEEPER_HOST
+ value: "{{ .Values.external.zookeeper.service.name }}"
+ - name: SP_ZOOKEEPER_PORT
+ value: "{{ .Values.external.zookeeper.service.port }}"
+ {{- end }}
+ {{- if eq .Values.preferredBroker "nats" }}
+ value: "nats"
+ - name: SP_NATS_HOST
+ value: "{{ .Values.external.nats.service.name }}"
+ - name: SP_NATS_PORT
+ value: "{{ .Values.external.nats.service.port }}"
+ {{- end }}
ports:
- containerPort: {{ .Values.streampipes.core.port }}
volumeMounts:
diff --git a/installer/k8s/templates/core/backend-deployment.yaml
b/installer/k8s/templates/external/pulsar/pulsar-deployment.yml
similarity index 52%
copy from installer/k8s/templates/core/backend-deployment.yaml
copy to installer/k8s/templates/external/pulsar/pulsar-deployment.yml
index 678828ef9..1cf581aba 100644
--- a/installer/k8s/templates/core/backend-deployment.yaml
+++ b/installer/k8s/templates/external/pulsar/pulsar-deployment.yml
@@ -13,63 +13,60 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+{{- if eq .Values.preferredBroker "pulsar" }}
apiVersion: apps/v1
kind: Deployment
metadata:
- name: {{ .Values.streampipes.core.appName }}
+ name: {{ .Values.external.pulsar.appName }}
spec:
selector:
matchLabels:
- app: {{ .Values.streampipes.core.appName }}
+ app: {{ .Values.external.pulsar.appName }}
replicas: 1
template:
metadata:
labels:
- app: {{ .Values.streampipes.core.appName }}
+ app: {{ .Values.external.pulsar.appName }}
spec:
- hostAliases:
- - ip: "127.0.0.1"
- hostnames:
- - "backend"
+ restartPolicy: {{ .Values.restartPolicy }}
+
volumes:
- - name: {{ .Values.streampipes.core.persistence.pvName }}
+ - name: {{ .Values.external.pulsar.persistence.pvName }}
persistentVolumeClaim:
- claimName: {{ .Values.streampipes.core.persistence.claimName }}
- restartPolicy: {{ .Values.restartPolicy }}
- initContainers:
- - name: init-wait-consul-couchdb
- image: alpine
- command: ["sh", "-c", "for i in $(seq 1 300); do nc -zvw1 {{
.Values.external.consul.service.name }} {{
.Values.external.consul.service.webPort }} && nc -zvw1 {{
.Values.external.couchdb.service.name }} {{
.Values.external.couchdb.service.port }} && exit 0 || sleep 3; done; exit 1"]
+ claimName: {{ .Values.external.pulsar.persistence.claimName }}
containers:
- - name: {{ .Values.streampipes.core.appName }}
- image: {{ .Values.streampipes.registry }}/backend:{{
.Values.streampipes.version }}
+ - name: {{ .Values.external.pulsar.appName }}
+ image: apachepulsar/pulsar:{{ .Values.external.pulsar.version }}
imagePullPolicy: {{ .Values.pullPolicy }}
- env:
- - name: SP_PRIORITIZED_PROTOCOL
- value: {{ .Values.preferredBroker }}
ports:
- - containerPort: {{ .Values.streampipes.core.port }}
+ - containerPort: {{ .Values.external.pulsar.port }}
+ env:
+ - name: advertisedListeners
+ value: "external:pulsar://127.0.0.1:{{
.Values.external.pulsar.port }}"
+ - name: PULSAR_MEM
+ value: "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m"
+ command: ["bash", "-c"]
+ args:
+ - "bin/pulsar standalone -nfw -nss"
volumeMounts:
- - mountPath: "/root/.streampipes"
- name: {{ .Values.streampipes.core.persistence.pvName }}
- readinessProbe:
- httpGet:
- path: /streampipes-backend/api/v2/setup/configured
- port: {{ .Values.streampipes.core.port }}
+ - mountPath: "/pulsar/data"
+ name: {{ .Values.external.pulsar.persistence.pvName }}
+ livenessProbe:
+ tcpSocket:
+ port: {{ .Values.external.pulsar.port }}
initialDelaySeconds: {{ .Values.initialDelaySeconds }}
periodSeconds: {{ .Values.periodSeconds }}
failureThreshold: {{ .Values.failureThreshold }}
- livenessProbe:
- httpGet:
- path: /streampipes-backend/api/v2/setup/configured
- port: {{ .Values.streampipes.core.port }}
+ readinessProbe:
+ tcpSocket:
+ port: {{ .Values.external.pulsar.port }}
initialDelaySeconds: {{ .Values.initialDelaySeconds }}
periodSeconds: {{ .Values.periodSeconds }}
failureThreshold: {{ .Values.failureThreshold }}
startupProbe:
- httpGet:
- path: /streampipes-backend/api/v2/setup/configured
- port: {{ .Values.streampipes.core.port }}
+ tcpSocket:
+ port: {{ .Values.external.pulsar.port }}
initialDelaySeconds: {{ .Values.initialDelaySeconds }}
periodSeconds: {{ .Values.periodSeconds }}
failureThreshold: {{ .Values.failureThreshold }}
+{{- end }}
\ No newline at end of file
diff --git a/installer/k8s/templates/external/pulsar/pulsar-pvc.yml
b/installer/k8s/templates/external/pulsar/pulsar-pvc.yml
new file mode 100644
index 000000000..001c63e4b
--- /dev/null
+++ b/installer/k8s/templates/external/pulsar/pulsar-pvc.yml
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if eq .Values.preferredBroker "pulsar" }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: {{ .Values.external.pulsar.persistence.pvName }}
+spec:
+ storageClassName: {{ .Values.external.pulsar.persistence.storageClassName }}
+ capacity:
+ storage: {{ .Values.external.pulsar.persistence.storageSize }}
+ accessModes:
+ - {{ .Values.persistentVolumeAccessModes }}
+ persistentVolumeReclaimPolicy: {{ .Values.persistentVolumeReclaimPolicy }}
+ hostPath:
+ path: {{ .Values.hostPath }}/pulsar
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ labels:
+ app: {{ .Values.external.pulsar.appName }}
+ name: {{ .Values.external.pulsar.persistence.claimName }}
+spec:
+ storageClassName: {{ .Values.external.pulsar.persistence.storageClassName }}
+ accessModes:
+ - {{ .Values.persistentVolumeAccessModes }}
+ resources:
+ requests:
+ storage: {{ .Values.external.pulsar.persistence.storageSize }}
+{{- end }}
\ No newline at end of file
diff --git a/installer/k8s/templates/external/pulsar/pulsar-service.yml
b/installer/k8s/templates/external/pulsar/pulsar-service.yml
new file mode 100644
index 000000000..6871aad6f
--- /dev/null
+++ b/installer/k8s/templates/external/pulsar/pulsar-service.yml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if eq .Values.preferredBroker "pulsar" }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Values.external.pulsar.service.name }}
+spec:
+ selector:
+ app: {{ .Values.external.pulsar.appName }}
+ ports:
+ - name: main
+ protocol: TCP
+ port: {{ .Values.external.pulsar.port }}
+ targetPort: {{ .Values.external.pulsar.port }}
+{{- end }}
\ No newline at end of file
diff --git a/installer/k8s/values.yaml b/installer/k8s/values.yaml
index 1cd3d3f42..4e6b3932b 100644
--- a/installer/k8s/values.yaml
+++ b/installer/k8s/values.yaml
@@ -162,6 +162,18 @@ external:
storageSize: "1Gi"
claimName: "zookeeper-pvc"
pvName: "zookeeper-pv"
+ pulsar:
+ appName: "pulsar"
+ version: 3.0.0
+ port: 6650
+ service:
+ name: "pulsar"
+ port: 6650
+ persistence:
+ storageClassName: "hostpath"
+ storageSize: "1Gi"
+ claimName: "pulsar-pvc"
+ pvName: "pulsar-pv"
