This is an automated email from the ASF dual-hosted git repository. riemer pushed a commit to branch migrate-kafka-3 in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 64637a927c42acbfd4f2de9713669890a29996e2 Author: Dominik Riemer <[email protected]> AuthorDate: Sun Aug 20 18:40:21 2023 +0200 Bump Kafka version to 3.x --- docker-compose.yml | 38 +++++------ .../deploy/standalone/kafka/docker-compose.dev.yml | 27 ++++---- .../cli/deploy/standalone/kafka/docker-compose.yml | 32 +++++----- installer/compose/docker-compose.yml | 51 ++++----------- installer/k8s/README.md | 14 ---- .../external/zookeeper/zookeeper-deployment.yaml | 74 ---------------------- .../external/zookeeper/zookeeper-pvc.yaml | 44 ------------- .../external/zookeeper/zookeeper-service.yaml | 29 --------- installer/k8s/values.yaml | 12 ---- .../pipeline-element-runtime-info.component.ts | 6 +- 10 files changed, 61 insertions(+), 266 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5570b5f3d..4ac2fed52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -113,30 +113,23 @@ services: spnet: kafka: - image: fogsyio/kafka:2.2.0 - hostname: kafka - depends_on: - - zookeeper + image: bitnami/kafka:3.5.1 environment: - # see: https://github.com/confluentinc/schema-registry/issues/648 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092 - KAFKA_LISTENERS: PLAINTEXT://:9092 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER volumes: - - kafka:/kafka - - /var/run/docker.sock:/var/run/docker.sock + - kafka3:/bitnami logging: *default-logging - networks: - spnet: - - zookeeper: - image: fogsyio/zookeeper:3.4.13 - logging: *default-logging - volumes: - - zookeeper:/opt/zookeeper-3.4.13 + restart: unless-stopped networks: spnet: @@ -160,11 +153,10 @@ services: spnet: volumes: - kafka: + kafka3: files: consul: couchdb: - zookeeper: influxdb: influxdb2: backend: diff --git a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml index 0076dd354..5d6433c21 100644 --- a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml +++ b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml @@ -17,17 +17,20 @@ version: "3.4" services: kafka: ports: + - "9093:9093" + - "9092:9092" - "9094:9094" - depends_on: - - zookeeper environment: - # see: https://github.com/confluentinc/schema-registry/issues/648 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 #(local, i.e. Single-Node with IDE + Docker) - KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_MESSAGE_MAX_BYTES: 5000012 - KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012 - KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000 + # KRaft settings + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 + # Listeners + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER diff --git a/installer/cli/deploy/standalone/kafka/docker-compose.yml b/installer/cli/deploy/standalone/kafka/docker-compose.yml index 9bac791c4..306663268 100644 --- a/installer/cli/deploy/standalone/kafka/docker-compose.yml +++ b/installer/cli/deploy/standalone/kafka/docker-compose.yml @@ -16,24 +16,24 @@ version: "3.4" services: kafka: - image: fogsyio/kafka:2.2.0 + image: bitnami/kafka:3.5.1 hostname: kafka - depends_on: - - zookeeper environment: - # see: https://github.com/confluentinc/schema-registry/issues/648 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092 - KAFKA_LISTENERS: PLAINTEXT://:9092 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_MESSAGE_MAX_BYTES: 5000012 - KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012 - KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000 + # KRaft settings + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 + # Listeners + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER volumes: - - kafka:/kafka - - /var/run/docker.sock:/var/run/docker.sock + - kafka3:/bitnami logging: driver: "json-file" options: @@ -43,7 +43,7 @@ services: spnet: volumes: - kafka: + kafka3: networks: spnet: diff --git a/installer/compose/docker-compose.yml b/installer/compose/docker-compose.yml index 102fd87cc..e4fe18a04 100644 --- a/installer/compose/docker-compose.yml +++ b/installer/compose/docker-compose.yml @@ -91,32 +91,21 @@ services: spnet: kafka: - image: fogsyio/kafka:2.2.0 - hostname: kafka - depends_on: - - zookeeper + image: bitnami/kafka:3.5.1 environment: - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092 - KAFKA_LISTENERS: PLAINTEXT://:9092 - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_MESSAGE_MAX_BYTES: 5000012 - KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012 - KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000 - volumes: - - kafka:/kafka - - /var/run/docker.sock:/var/run/docker.sock - logging: *default-logging - restart: unless-stopped - networks: - spnet: - - zookeeper: - image: fogsyio/zookeeper:3.4.13 + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012 + - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER volumes: - - zookeeper:/opt/zookeeper-3.4.13 + - kafka3:/bitnami logging: *default-logging restart: unless-stopped networks: @@ -154,24 +143,12 @@ services: networks: spnet: - sources-watertank-simulator: - image: "${SP_DOCKER_REGISTRY}/sources-watertank-simulator:${SP_VERSION}" - depends_on: - - consul - - kafka - logging: *default-logging - restart: unless-stopped - networks: - spnet: - - volumes: backend: consul: connect: couchdb: - kafka: - zookeeper: + kafka3: influxdb: influxdb2: files: diff --git a/installer/k8s/README.md b/installer/k8s/README.md index 135236c44..cb2f641e8 100644 --- a/installer/k8s/README.md +++ b/installer/k8s/README.md @@ -224,20 +224,6 @@ rm -rf ${HOME}/streampipes-k8s | external.kafka.persistence.pvName | Name of the Kafka PersistentVolume | "kafka-pv" | -####Zookeeper common parameters -| Parameter Name | Description | Value | -|-------------------------------------------------|----------------------------------------------------------|------------------------------------------| -| external.zookeeper.appName | ZooKeeper application name | "zookeeper" | -| external.zookeeper.version | ZooKeeper version | 3.4.13 | -| external.zookeeper.port | Port for the ZooKeeper service | 2181 | -| external.zookeeper.service.name | Name of the ZooKeeper service | "zookeeper" | -| external.zookeeper.service.port | TargetPort of the ZooKeeper service | 2181 | -| external.zookeeper.persistence.storageClassName | Storage class name for ZooKeeper PVs | "hostpath" | -| external.zookeeper.persistence.storageSize | Size of the ZooKeeper PV | "1Gi" | -| external.zookeeper.persistence.claimName | Name of the ZooKeeper PersistentVolumeClaim | "zookeeper-pvc" | -| external.zookeeper.persistence.pvName | Name of the ZooKeeper PersistentVolume | "zookeeper-pv" | - - ####Pulsar common parameters | Parameter Name | Description | Value | |-------------------------------------------------|----------------------------------------------------------|------------------------------------------| diff --git a/installer/k8s/templates/external/zookeeper/zookeeper-deployment.yaml b/installer/k8s/templates/external/zookeeper/zookeeper-deployment.yaml deleted file mode 100644 index 07314cdeb..000000000 --- a/installer/k8s/templates/external/zookeeper/zookeeper-deployment.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if eq .Values.preferredBroker "kafka" }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.external.zookeeper.appName }} -spec: - selector: - matchLabels: - app: {{ .Values.external.zookeeper.appName }} - replicas: 1 - template: - metadata: - labels: - app: {{ .Values.external.zookeeper.appName }} - spec: - restartPolicy: {{ .Values.restartPolicy }} - volumes: - - name: {{ .Values.external.zookeeper.persistence.pvName }} - persistentVolumeClaim: - claimName: {{ .Values.external.zookeeper.persistence.claimName }} - containers: - #TODO: wurstmeister/zookeeper:latest is running ZK 3.4.13. Once this - # changes, the mount path needs to be adapted - - name: {{ .Values.external.zookeeper.appName }} - image: fogsyio/zookeeper:{{ .Values.external.zookeeper.version }} - imagePullPolicy: {{ .Values.pullPolicy }} - ports: - - containerPort: {{ .Values.external.zookeeper.port }} - volumeMounts: - - mountPath: "/opt/zookeeper-{{ .Values.external.zookeeper.version }}/data" - name: {{ .Values.external.zookeeper.persistence.pvName }} - livenessProbe: - exec: - command: - - sh - - -c - - echo ruok | nc localhost {{ .Values.external.zookeeper.port }} - initialDelaySeconds: {{ .Values.initialDelaySeconds }} - periodSeconds: {{ .Values.periodSeconds }} - failureThreshold: {{ .Values.failureThreshold }} - readinessProbe: - exec: - command: - - sh - - -c - - echo ruok | nc localhost {{ .Values.external.zookeeper.port }} - initialDelaySeconds: {{ .Values.initialDelaySeconds }} - periodSeconds: {{ .Values.periodSeconds }} - failureThreshold: {{ .Values.failureThreshold }} - startupProbe: - exec: - command: - - sh - - -c - - echo ruok | nc localhost {{ .Values.external.zookeeper.port }} - initialDelaySeconds: {{ .Values.initialDelaySeconds }} - periodSeconds: {{ .Values.periodSeconds }} - failureThreshold: {{ .Values.failureThreshold }} -{{- end }} \ No newline at end of file diff --git a/installer/k8s/templates/external/zookeeper/zookeeper-pvc.yaml b/installer/k8s/templates/external/zookeeper/zookeeper-pvc.yaml deleted file mode 100644 index 465963d6e..000000000 --- a/installer/k8s/templates/external/zookeeper/zookeeper-pvc.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if eq .Values.preferredBroker "kafka" }} -apiVersion: v1 -kind: PersistentVolume -metadata: - name: {{ .Values.external.zookeeper.persistence.pvName }} -spec: - storageClassName: {{ .Values.external.zookeeper.persistence.storageClassName }} - capacity: - storage: {{ .Values.external.zookeeper.persistence.storageSize }} - accessModes: - - {{ .Values.persistentVolumeAccessModes }} - persistentVolumeReclaimPolicy: {{ .Values.persistentVolumeReclaimPolicy }} - hostPath: - path: {{ .Values.hostPath }}/zookeeper ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - app: {{ .Values.external.zookeeper.appName }} - name: {{ .Values.external.zookeeper.persistence.claimName }} -spec: - storageClassName: {{ .Values.external.zookeeper.persistence.storageClassName }} - accessModes: - - {{ .Values.persistentVolumeAccessModes }} - resources: - requests: - storage: {{ .Values.external.zookeeper.persistence.storageSize }} -{{- end }} \ No newline at end of file diff --git a/installer/k8s/templates/external/zookeeper/zookeeper-service.yaml b/installer/k8s/templates/external/zookeeper/zookeeper-service.yaml deleted file mode 100644 index 492d0558e..000000000 --- a/installer/k8s/templates/external/zookeeper/zookeeper-service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if eq .Values.preferredBroker "kafka" }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.external.zookeeper.service.name }} -spec: - selector: - app: {{ .Values.external.zookeeper.appName }} - ports: - - name: main - protocol: TCP - port: {{ .Values.external.zookeeper.port }} - targetPort: {{ .Values.external.zookeeper.port }} -{{- end }} \ No newline at end of file diff --git a/installer/k8s/values.yaml b/installer/k8s/values.yaml index 4e6b3932b..8e58afcfb 100644 --- a/installer/k8s/values.yaml +++ b/installer/k8s/values.yaml @@ -150,18 +150,6 @@ external: storageSize: "1Gi" claimName: "kafka-pvc" pvName: "kafka-pv" - zookeeper: - appName: "zookeeper" - version: 3.4.13 - port: 2181 - service: - name: "zookeeper" - port: 2181 - persistence: - storageClassName: "hostpath" - storageSize: "1Gi" - claimName: "zookeeper-pvc" - pvName: "zookeeper-pv" pulsar: appName: "pulsar" version: 3.0.0 diff --git a/ui/src/app/connect/components/runtime-info/pipeline-element-runtime-info.component.ts b/ui/src/app/connect/components/runtime-info/pipeline-element-runtime-info.component.ts index 01e07422b..c5fe37f06 100644 --- a/ui/src/app/connect/components/runtime-info/pipeline-element-runtime-info.component.ts +++ b/ui/src/app/connect/components/runtime-info/pipeline-element-runtime-info.component.ts @@ -28,7 +28,7 @@ import { RestService } from '../../services/rest.service'; templateUrl: './pipeline-element-runtime-info.component.html', styleUrls: ['./pipeline-element-runtime-info.component.scss'], }) -export class PipelineElementRuntimeInfoComponent implements OnInit, OnDestroy { +export class PipelineElementRuntimeInfoComponent implements OnDestroy { @Input() streamDescription: SpDataStream; @@ -40,10 +40,6 @@ export class PipelineElementRuntimeInfoComponent implements OnInit, OnDestroy { constructor(private restService: RestService) {} - ngOnInit(): void { - this.checkPollingStart(); - } - checkPollingStart() { if (this._pollingActive) { this.getLatestRuntimeInfo();
