This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch
3807-upgrade-kafka-to-version-410-and-switch-to-official-images
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/3807-upgrade-kafka-to-version-410-and-switch-to-official-images by
this push:
new f069b552d1 fix(#3807): Update kafka version to 4.1.0
f069b552d1 is described below
commit f069b552d16a2f551d5267187679455efcd18bcf
Author: Philipp Zehnder <[email protected]>
AuthorDate: Wed Oct 1 16:51:30 2025 +0200
fix(#3807): Update kafka version to 4.1.0
---
docker-compose.yml | 30 ++++++++++++---------
.../deploy/standalone/kafka/docker-compose.dev.yml | 28 ++++++++++---------
.../cli/deploy/standalone/kafka/docker-compose.yml | 31 ++++++++++++----------
installer/compose/docker-compose.full.yml | 30 ++++++++++++---------
installer/compose/docker-compose.yml | 30 ++++++++++++---------
.../kafka/config/ConsumerConfigFactory.java | 1 +
6 files changed, 84 insertions(+), 66 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 6135ce4f3e..a8659193b1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -81,22 +81,26 @@ services:
spnet:
kafka:
- image: bitnami/kafka:3.9.0
+ image: apache/kafka:4.1.0
hostname: kafka
environment:
- - KAFKA_CFG_NODE_ID=0
- - KAFKA_CFG_PROCESS_ROLES=controller,broker
- - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- -
KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- -
KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- -
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_NODE_ID=0
+ - KAFKA_PROCESS_ROLES=controller,broker
+ - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+ - KAFKA_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+ - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
+ -
KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
+ -
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+ - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
+ - KAFKA_FETCH_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1
+ - KAFKA_DEFAULT_REPLICATION_FACTOR=1
volumes:
- - kafka3:/bitnami
+ - kafka3:/var/lib/kafka/data
logging: *default-logging
networks:
spnet:
diff --git a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
index 8ac1f2e8ba..0f43f788dc 100644
--- a/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
+++ b/installer/cli/deploy/standalone/kafka/docker-compose.dev.yml
@@ -18,17 +18,19 @@ services:
ports:
- "9094:9094"
environment:
- # KRaft settings
- - KAFKA_CFG_NODE_ID=0
- - KAFKA_CFG_PROCESS_ROLES=controller,broker
- - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- # Listeners
- -
KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- -
KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- -
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_NODE_ID=0
+ - KAFKA_PROCESS_ROLES=controller,broker
+ - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+ - KAFKA_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+ - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
+ -
KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
+ -
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+ - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
+ - KAFKA_FETCH_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1
+ - KAFKA_DEFAULT_REPLICATION_FACTOR=1
diff --git a/installer/cli/deploy/standalone/kafka/docker-compose.yml
b/installer/cli/deploy/standalone/kafka/docker-compose.yml
index a4d689d2e1..f4a28f8c9c 100644
--- a/installer/cli/deploy/standalone/kafka/docker-compose.yml
+++ b/installer/cli/deploy/standalone/kafka/docker-compose.yml
@@ -15,23 +15,26 @@
services:
kafka:
- image: bitnami/kafka:3.9.0
+ image: apache/kafka:4.1.0
hostname: kafka
environment:
- - KAFKA_CFG_NODE_ID=0
- - KAFKA_CFG_PROCESS_ROLES=controller,broker
- - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- # Listeners
- - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
- - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
- -
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
- - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_NODE_ID=0
+ - KAFKA_PROCESS_ROLES=controller,broker
+ - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+ - KAFKA_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+ - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
+ -
KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
+ -
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+ - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
+ - KAFKA_FETCH_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1
+ - KAFKA_DEFAULT_REPLICATION_FACTOR=1
volumes:
- - kafka3:/bitnami
+ - kafka3:/var/lib/kafka/data
logging:
driver: "json-file"
options:
diff --git a/installer/compose/docker-compose.full.yml
b/installer/compose/docker-compose.full.yml
index 65a2e1b652..aaa3a5613e 100644
--- a/installer/compose/docker-compose.full.yml
+++ b/installer/compose/docker-compose.full.yml
@@ -64,22 +64,26 @@ services:
spnet:
kafka:
- image: bitnami/kafka:3.9.0
+ image: apache/kafka:4.1.0
hostname: kafka
environment:
- - KAFKA_CFG_NODE_ID=0
- - KAFKA_CFG_PROCESS_ROLES=controller,broker
- - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- -
KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- -
KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- -
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_NODE_ID=0
+ - KAFKA_PROCESS_ROLES=controller,broker
+ - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+ - KAFKA_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+ - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
+ -
KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
+ -
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+ - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
+ - KAFKA_FETCH_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1
+ - KAFKA_DEFAULT_REPLICATION_FACTOR=1
volumes:
- - kafka3:/bitnami
+ - kafka3:/var/lib/kafka/data
logging: *default-logging
restart: unless-stopped
networks:
diff --git a/installer/compose/docker-compose.yml
b/installer/compose/docker-compose.yml
index 338d138fcc..f0d00599db 100644
--- a/installer/compose/docker-compose.yml
+++ b/installer/compose/docker-compose.yml
@@ -64,22 +64,26 @@ services:
spnet:
kafka:
- image: bitnami/kafka:3.9.0
+ image: apache/kafka:4.1.0
hostname: kafka
environment:
- - KAFKA_CFG_NODE_ID=0
- - KAFKA_CFG_PROCESS_ROLES=controller,broker
- - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
- -
KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
- -
KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
- -
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
- - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- - KAFKA_CFG_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES=5000012
- - KAFKA_CFG_REPLICA_FETCH_MAX_BYTES=10000000
- - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_NODE_ID=0
+ - KAFKA_PROCESS_ROLES=controller,broker
+ - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+ - KAFKA_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+ - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,OUTSIDE://:9094
+ -
KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,OUTSIDE://localhost:9094
+ -
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+ - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ - KAFKA_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
+ - KAFKA_FETCH_MESSAGE_MAX_BYTES=5000012
+ - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS=1
+ - KAFKA_DEFAULT_REPLICATION_FACTOR=1
volumes:
- - kafka3:/bitnami
+ - kafka3:/var/lib/kafka/data
logging: *default-logging
restart: unless-stopped
networks:
diff --git
a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/ConsumerConfigFactory.java
b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/ConsumerConfigFactory.java
index 3ab75d03e6..4b16a52b2a 100644
---
a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/ConsumerConfigFactory.java
+++
b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/ConsumerConfigFactory.java
@@ -43,6 +43,7 @@ public class ConsumerConfigFactory extends
AbstractConfigFactory {
public Properties makeDefaultProperties() {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, getBrokerUrl());
+ props.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, "consumer");
props.put(ConsumerConfig.GROUP_ID_CONFIG,
getConfigOrDefault(protocol::getGroupId, UUID.randomUUID().toString()));
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,
ENABLE_AUTO_COMMIT_CONFIG_DEFAULT);
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,
