This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 714f46232 Add configuration hint for Kafka users. (#1945)
714f46232 is described below
commit 714f46232c8276d3f1531cebb292dce22dcee45f
Author: Stefan Obermeier <[email protected]>
AuthorDate: Mon Oct 2 09:54:16 2023 +0200
Add configuration hint for Kafka users. (#1945)
* Add configuration hint for Kafka users.
* Add configuration hints
---
installer/cli/deploy/standalone/kafka/docker-compose.dev.yml | 2 +-
installer/compose/docker-compose.full.yml | 6 +++---
installer/compose/docker-compose.yml | 6 +++---
installer/k8s/README.md | 6 ++++--
installer/k8s/templates/external/kafka/kafka-deployment.yaml | 6 +++---
installer/k8s/templates/external/kafka/kafka-service.yaml | 4 ++++
installer/k8s/values.yaml | 3 +++
.../3-getting-live-data-from-the-streampipes-data-stream.ipynb | 2 +-
8 files changed, 22 insertions(+), 13 deletions(-)
diff --git a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
index 0076dd354..e6ea59878 100644
--- a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
+++ b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
environment:
# see: https://github.com/confluentinc/schema-registry/issues/648
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094
#(local, i.e. Single-Node with IDE + Docker)
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 #
Replace localhost with your external address if Kafka should be reachable from
external systems.
KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
diff --git a/installer/compose/docker-compose.full.yml
b/installer/compose/docker-compose.full.yml
index 6044f569a..6a634f5bc 100644
--- a/installer/compose/docker-compose.full.yml
+++ b/installer/compose/docker-compose.full.yml
@@ -124,9 +124,9 @@ services:
depends_on:
- zookeeper
environment:
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
- KAFKA_LISTENERS: PLAINTEXT://:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 #
Replace localhost with your external address if Kafka should be reachable from
external systems.
+ KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
diff --git a/installer/compose/docker-compose.yml
b/installer/compose/docker-compose.yml
index 102fd87cc..30d3b4de0 100644
--- a/installer/compose/docker-compose.yml
+++ b/installer/compose/docker-compose.yml
@@ -96,9 +96,9 @@ services:
depends_on:
- zookeeper
environment:
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
- KAFKA_LISTENERS: PLAINTEXT://:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 #
Replace localhost with your external address if Kafka should be reachable from
external systems.
+ KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
diff --git a/installer/k8s/README.md b/installer/k8s/README.md
index c61f9889d..ac93dab55 100644
--- a/installer/k8s/README.md
+++ b/installer/k8s/README.md
@@ -55,7 +55,7 @@ helm install streampipes ./
# full template only recommend if you have sufficient resources
# helm install streampipes ./ --set deployment=full
```
-After a while, all containers should successfully started, indicated by the
`Running` status.
+After a while, all containers should successfully started, indicated by the
`Running` status.
```bash
kubectl get pods
NAME READY STATUS RESTARTS AGE
@@ -222,13 +222,15 @@ rm -rf ${HOME}/streampipes-k8s
| external.kafka.appName | Kafka application name
| "kafka" |
| external.kafka.version | Kafka version
| 2.2.0 |
| external.kafka.port | Port for the Kafka service
| 9092 |
+| external.kafka.external.hostname | Name which will be
advertised to external clients. Clients which use (default) port 9094 |
"localhost"
| external.kafka.service.name | Name of the Kafka service
| "kafka" |
| external.kafka.service.port | TargetPort of the Kafka
service | 9092 |
+| external.kafka.service.portOutside | Port for Kafka client
outside of the cluster | 9094 |
| external.kafka.persistence.storageClassName | Storage class name for
Kafka PVs | "hostpath" |
| external.kafka.persistence.storageSize | Size of the Kafka PV
| "1Gi" |
| external.kafka.persistence.claimName | Name of the Kafka
PersistentVolumeClaim | "kafka-pvc"
|
| external.kafka.persistence.pvName | Name of the Kafka
PersistentVolume | "kafka-pv"
|
-
+|
####Zookeeper common parameters
| Parameter Name | Description
| Value |
diff --git a/installer/k8s/templates/external/kafka/kafka-deployment.yaml
b/installer/k8s/templates/external/kafka/kafka-deployment.yaml
index a8db6331b..f5680ace4 100644
--- a/installer/k8s/templates/external/kafka/kafka-deployment.yaml
+++ b/installer/k8s/templates/external/kafka/kafka-deployment.yaml
@@ -54,13 +54,13 @@ spec:
- name: KAFKA_ZOOKEEPER_CONNECT
value: "zookeeper:{{ .Values.external.zookeeper.port }}"
- name: KAFKA_LISTENERS
- value: "PLAINTEXT://:{{ .Values.external.kafka.port }}"
+ value: "PLAINTEXT://:{{ .Values.external.kafka.port
}},OUTSIDE://:9094"
- name: KAFKA_ADVERTISED_LISTENERS
- value: "PLAINTEXT://kafka:{{ .Values.external.kafka.port }}"
+ value: "PLAINTEXT://kafka:{{ .Values.external.kafka.port
}},OUTSIDE://{{ .Values.external.kafka.external.hostname }}:9094"
- name: KAFKA_INTER_BROKER_LISTENER_NAME
value: "PLAINTEXT"
- name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
- value: "PLAINTEXT:PLAINTEXT"
+ value: "PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT"
- name: KAFKA_MESSAGE_MAX_BYTES
value: "5000012"
- name: KAFKA_FETCH_MESSAGE_MAX_BYTES
diff --git a/installer/k8s/templates/external/kafka/kafka-service.yaml
b/installer/k8s/templates/external/kafka/kafka-service.yaml
index acad6101c..df9ce91dd 100644
--- a/installer/k8s/templates/external/kafka/kafka-service.yaml
+++ b/installer/k8s/templates/external/kafka/kafka-service.yaml
@@ -28,4 +28,8 @@ spec:
protocol: TCP
port: {{ .Values.external.kafka.port }}
targetPort: {{ .Values.external.kafka.service.port }}
+ - name: outside
+ protocol: TCP
+ port: 9094
+ targetPort: {{ .Values.external.kafka.service.portOutside }}
{{- end }}
\ No newline at end of file
diff --git a/installer/k8s/values.yaml b/installer/k8s/values.yaml
index 63abf6cc8..6effc47d0 100644
--- a/installer/k8s/values.yaml
+++ b/installer/k8s/values.yaml
@@ -153,9 +153,12 @@ external:
appName: "kafka"
version: 2.2.0
port: 9092
+ external:
+ hostname: "localhost" # Replace localhost with your external address if
Kafka should be reachable from external systems.
service:
name: "kafka"
port: 9092
+ portOutside: 9094
persistence:
storageClassName: "hostpath"
storageSize: "1Gi"
diff --git
a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
index 6f20e4f90..91fbaba7c 100644
---
a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
+++
b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
@@ -56,7 +56,7 @@
"\n",
"# Use this if you work locally:\n",
"os.environ[\"BROKER-HOST\"] = \"localhost\" \n",
- "os.environ[\"KAFKA-PORT\"] = \"9094\" # When using Kafka as message
broker"
+ "os.environ[\"KAFKA-PORT\"] = \"9094\" # When using Kafka as message
broker. If Kafka is not running on localhost, KAFKA_ADVERTISED_LISTENERS should
be adjusted to the external address"
]
},
{