This is an automated email from the ASF dual-hosted git repository.

bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new 714f46232 Add configuration hint for Kafka users. (#1945)
714f46232 is described below

commit 714f46232c8276d3f1531cebb292dce22dcee45f
Author: Stefan Obermeier <[email protected]>
AuthorDate: Mon Oct 2 09:54:16 2023 +0200

    Add configuration hint for Kafka users. (#1945)
    
    * Add configuration hint for Kafka users.
    
    * Add configuration hints
---
 installer/cli/deploy/standalone/kafka/docker-compose.dev.yml        | 2 +-
 installer/compose/docker-compose.full.yml                           | 6 +++---
 installer/compose/docker-compose.yml                                | 6 +++---
 installer/k8s/README.md                                             | 6 ++++--
 installer/k8s/templates/external/kafka/kafka-deployment.yaml        | 6 +++---
 installer/k8s/templates/external/kafka/kafka-service.yaml           | 4 ++++
 installer/k8s/values.yaml                                           | 3 +++
 .../3-getting-live-data-from-the-streampipes-data-stream.ipynb      | 2 +-
 8 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml 
b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
index 0076dd354..e6ea59878 100644
--- a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
+++ b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
     environment:
       # see: https://github.com/confluentinc/schema-registry/issues/648
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 
PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 
#(local, i.e. Single-Node with IDE + Docker)
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # 
 Replace localhost with your external address if Kafka should be reachable from 
external systems.
       KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_ADVERTISED_HOST_NAME: kafka
diff --git a/installer/compose/docker-compose.full.yml 
b/installer/compose/docker-compose.full.yml
index 6044f569a..6a634f5bc 100644
--- a/installer/compose/docker-compose.full.yml
+++ b/installer/compose/docker-compose.full.yml
@@ -124,9 +124,9 @@ services:
     depends_on:
       - zookeeper
     environment:
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
-      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 
PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # 
 Replace localhost with your external address if Kafka should be reachable from 
external systems.
+      KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_ADVERTISED_HOST_NAME: kafka
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
diff --git a/installer/compose/docker-compose.yml 
b/installer/compose/docker-compose.yml
index 102fd87cc..30d3b4de0 100644
--- a/installer/compose/docker-compose.yml
+++ b/installer/compose/docker-compose.yml
@@ -96,9 +96,9 @@ services:
     depends_on:
       - zookeeper
     environment:
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
-      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 
PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 # 
 Replace localhost with your external address if Kafka should be reachable from 
external systems.
+      KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_ADVERTISED_HOST_NAME: kafka
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
diff --git a/installer/k8s/README.md b/installer/k8s/README.md
index c61f9889d..ac93dab55 100644
--- a/installer/k8s/README.md
+++ b/installer/k8s/README.md
@@ -55,7 +55,7 @@ helm install streampipes ./
 # full template only recommend if you have sufficient resources
 # helm install streampipes ./ --set deployment=full
 ```
-After a while, all containers should successfully started, indicated by the 
`Running` status. 
+After a while, all containers should successfully started, indicated by the 
`Running` status.
 ```bash
 kubectl get pods
 NAME                                           READY   STATUS    RESTARTS   AGE
@@ -222,13 +222,15 @@ rm -rf ${HOME}/streampipes-k8s
 | external.kafka.appName                          | Kafka application name     
                              | "kafka"                                  |
 | external.kafka.version                          | Kafka version              
                              | 2.2.0                                    |
 | external.kafka.port                             | Port for the Kafka service 
                              | 9092                                     |
+| external.kafka.external.hostname                | Name which will be 
advertised to external clients. Clients which use (default) port 9094       | 
"localhost"
 | external.kafka.service.name                     | Name of the Kafka service  
                              | "kafka"                                  |
 | external.kafka.service.port                     | TargetPort of the Kafka 
service                          | 9092                                     |
+| external.kafka.service.portOutside              | Port for Kafka client 
outside of the cluster | 9094                              |
 | external.kafka.persistence.storageClassName     | Storage class name for 
Kafka PVs                         | "hostpath"                               |
 | external.kafka.persistence.storageSize          | Size of the Kafka PV       
                              | "1Gi"                                    |
 | external.kafka.persistence.claimName            | Name of the Kafka 
PersistentVolumeClaim                  | "kafka-pvc"                            
  |
 | external.kafka.persistence.pvName               | Name of the Kafka 
PersistentVolume                       | "kafka-pv"                             
  |
-
+|
 
 ####Zookeeper common parameters
 | Parameter Name                                  | Description                
                              | Value                                    |
diff --git a/installer/k8s/templates/external/kafka/kafka-deployment.yaml 
b/installer/k8s/templates/external/kafka/kafka-deployment.yaml
index a8db6331b..f5680ace4 100644
--- a/installer/k8s/templates/external/kafka/kafka-deployment.yaml
+++ b/installer/k8s/templates/external/kafka/kafka-deployment.yaml
@@ -54,13 +54,13 @@ spec:
             - name: KAFKA_ZOOKEEPER_CONNECT
               value: "zookeeper:{{ .Values.external.zookeeper.port }}"
             - name: KAFKA_LISTENERS
-              value: "PLAINTEXT://:{{ .Values.external.kafka.port }}"
+              value: "PLAINTEXT://:{{ .Values.external.kafka.port 
}},OUTSIDE://:9094"
             - name: KAFKA_ADVERTISED_LISTENERS
-              value: "PLAINTEXT://kafka:{{ .Values.external.kafka.port }}"
+              value: "PLAINTEXT://kafka:{{ .Values.external.kafka.port 
}},OUTSIDE://{{ .Values.external.kafka.external.hostname }}:9094"
             - name: KAFKA_INTER_BROKER_LISTENER_NAME
               value: "PLAINTEXT"
             - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
-              value: "PLAINTEXT:PLAINTEXT"
+              value: "PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT"
             - name: KAFKA_MESSAGE_MAX_BYTES
               value: "5000012"
             - name: KAFKA_FETCH_MESSAGE_MAX_BYTES
diff --git a/installer/k8s/templates/external/kafka/kafka-service.yaml 
b/installer/k8s/templates/external/kafka/kafka-service.yaml
index acad6101c..df9ce91dd 100644
--- a/installer/k8s/templates/external/kafka/kafka-service.yaml
+++ b/installer/k8s/templates/external/kafka/kafka-service.yaml
@@ -28,4 +28,8 @@ spec:
       protocol: TCP
       port: {{ .Values.external.kafka.port }}
       targetPort: {{ .Values.external.kafka.service.port }}
+    - name: outside
+      protocol: TCP
+      port: 9094
+      targetPort: {{ .Values.external.kafka.service.portOutside }}
 {{- end }}
\ No newline at end of file
diff --git a/installer/k8s/values.yaml b/installer/k8s/values.yaml
index 63abf6cc8..6effc47d0 100644
--- a/installer/k8s/values.yaml
+++ b/installer/k8s/values.yaml
@@ -153,9 +153,12 @@ external:
     appName: "kafka"
     version: 2.2.0
     port: 9092
+    external:
+      hostname: "localhost" #  Replace localhost with your external address if 
Kafka should be reachable from external systems.
     service:
       name: "kafka"
       port: 9092
+      portOutside: 9094
     persistence:
       storageClassName: "hostpath"
       storageSize: "1Gi"
diff --git 
a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
 
b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
index 6f20e4f90..91fbaba7c 100644
--- 
a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
+++ 
b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
@@ -56,7 +56,7 @@
     "\n",
     "# Use this if you work locally:\n",
     "os.environ[\"BROKER-HOST\"] = \"localhost\"  \n",
-    "os.environ[\"KAFKA-PORT\"] = \"9094\" # When using Kafka as message 
broker"
+    "os.environ[\"KAFKA-PORT\"] = \"9094\" # When using Kafka as message 
broker. If Kafka is not running on localhost, KAFKA_ADVERTISED_LISTENERS should 
be adjusted to the external address"
    ]
   },
   {

Reply via email to