This is an automated email from the ASF dual-hosted git repository. trohrmann pushed a commit to branch release-3.2 in repository https://gitbox.apache.org/repos/asf/flink-statefun-playground.git
commit d1fa6dd069979831b13ae933a1ee7ff651363133 Author: Till Rohrmann <[email protected]> AuthorDate: Sun Feb 6 12:11:43 2022 +0100 [FLINK-25934] Replace Kafka image with Redpanda --- go/greeter/README.md | 14 +++++------- go/greeter/docker-compose.yml | 30 +++++++++++++------------- go/showcase/docker-compose.yml | 30 +++++++++++++------------- java/connected-components/README.md | 5 +---- java/connected-components/docker-compose.yml | 30 +++++++++++++------------- java/greeter/README.md | 5 +---- java/greeter/docker-compose.yml | 32 +++++++++++++--------------- java/shopping-cart/README.md | 2 +- java/shopping-cart/docker-compose.yml | 30 +++++++++++++------------- javascript/greeter/README.md | 14 +++++------- javascript/greeter/docker-compose.yml | 30 +++++++++++++------------- javascript/showcase/docker-compose.yml | 30 +++++++++++++------------- python/greeter/README.md | 14 +++++------- python/greeter/docker-compose.yml | 30 +++++++++++++------------- python/showcase/docker-compose.yml | 30 +++++++++++++------------- 15 files changed, 153 insertions(+), 173 deletions(-) diff --git a/go/greeter/README.md b/go/greeter/README.md index 44b18c6..5241620 100644 --- a/go/greeter/README.md +++ b/go/greeter/README.md @@ -2,11 +2,11 @@ This is a simple example of a stateful functions application implemented in `Go`. -In this example, we imagine a service that computes personalized greetings. +In this example, we imagine a service that computes personalized greetings. Our service, consist out of the following components: * `kafka ingress` - This component forwards messages produced to the `names` kafka topic, -to the `person` stateful function. Messages produced to this topic has the following +to the `person` stateful function. Messages produced to this topic has the following schema `{ "name" : "bob"}`. * `person` - This function is triggered by the ingress defined above. @@ -14,7 +14,7 @@ This function keeps track of the number of visits, and triggers the next functio * `greeter` - This function, computes a personalized greeting, based on the name and the number of visits of that user. The output of that computation is forward to a Kafka egress defined below. - + * `kafka egress` - This wraps a Kafka producer that emits `utf-8` greetings to the `greetings` Kafka topic. @@ -30,11 +30,7 @@ docker-compose up To observe the customized greeting, as they appear in the `greetings` Kafka topic, run in a separate terminal: ``` -docker-compose exec kafka kafka-console-consumer \ - --bootstrap-server kafka:9092 \ - --isolation-level read_committed \ - --from-beginning \ - --topic greetings +docker-compose exec kafka rpk topic consume greetings ``` Try adding few more input lines to [input-example.json](input-example.json), and restart @@ -42,7 +38,7 @@ the producer service. ``` docker-compose restart producer -``` +``` Feeling curious? add the following print to the `person` function at [greeter.go](greeter.go): ```fmt.Printf("Hello there %d!", ctx.Self().Id)```. diff --git a/go/greeter/docker-compose.yml b/go/greeter/docker-compose.yml index f72a966..8e4e8a3 100644 --- a/go/greeter/docker-compose.yml +++ b/go/greeter/docker-compose.yml @@ -47,24 +47,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Simple Kafka JSON producer to simulate ingress events diff --git a/go/showcase/docker-compose.yml b/go/showcase/docker-compose.yml index f91c0a1..c493c19 100644 --- a/go/showcase/docker-compose.yml +++ b/go/showcase/docker-compose.yml @@ -36,24 +36,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Forward a port 8000 from the host's machine diff --git a/java/connected-components/README.md b/java/connected-components/README.md index 981a754..17cfa82 100644 --- a/java/connected-components/README.md +++ b/java/connected-components/README.md @@ -54,10 +54,7 @@ $ docker-compose up You can take a look at what messages are being sent to the Kafka egress: ``` -$ docker-compose exec kafka kafka-console-consumer \ - --bootstrap-server kafka:9092 \ - --topic connected-component-changes \ - --from-beginning +$ docker-compose exec kafka rpk topic consume connected-component-changes ``` You can also try modifying the function code in the `src/main/java` directory, and do a zero-downtime upgrade of the diff --git a/java/connected-components/docker-compose.yml b/java/connected-components/docker-compose.yml index d66fc1f..3bbb2e6 100644 --- a/java/connected-components/docker-compose.yml +++ b/java/connected-components/docker-compose.yml @@ -48,24 +48,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Simple Kafka JSON producer to simulate ingress events diff --git a/java/greeter/README.md b/java/greeter/README.md index 315b278..739b5f5 100644 --- a/java/greeter/README.md +++ b/java/greeter/README.md @@ -54,10 +54,7 @@ $ docker-compose up You can take a look at what messages are being sent to the Kafka egress: ``` -$ docker-compose exec kafka kafka-console-consumer \ - --bootstrap-server kafka:9092 \ - --topic greetings \ - --from-beginning +$ docker-compose exec kafka rpk topic consume greetings ``` You can also try modifying the function code in the `src/main/java` directory, and do a zero-downtime upgrade of the diff --git a/java/greeter/docker-compose.yml b/java/greeter/docker-compose.yml index fec846d..cc2e1b1 100644 --- a/java/greeter/docker-compose.yml +++ b/java/greeter/docker-compose.yml @@ -48,26 +48,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - links: - - "zookeeper:zookeeper" - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Simple Kafka JSON producer to simulate ingress events diff --git a/java/shopping-cart/README.md b/java/shopping-cart/README.md index 1d8b539..c1877ed 100644 --- a/java/shopping-cart/README.md +++ b/java/shopping-cart/README.md @@ -63,7 +63,7 @@ Note: `Caller: Optional.empty` in the logs corresponds to the messages that came To see the results produced to the egress: ``` -docker-compose exec kafka bash -c '/usr/bin/kafka-console-consumer --topic receipts --bootstrap-server kafka:9092' +docker-compose exec kafka rpk topic consume receipts' ``` If you want to modify the code, you can do a hot redeploy of your functions service: diff --git a/java/shopping-cart/docker-compose.yml b/java/shopping-cart/docker-compose.yml index 3a2d25e..e4be3aa 100644 --- a/java/shopping-cart/docker-compose.yml +++ b/java/shopping-cart/docker-compose.yml @@ -48,21 +48,21 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" diff --git a/javascript/greeter/README.md b/javascript/greeter/README.md index efc2bd1..ea1fa0a 100644 --- a/javascript/greeter/README.md +++ b/javascript/greeter/README.md @@ -2,11 +2,11 @@ This is a simple example of a stateful functions application implemented in `JavaScript`. -In this example, we imagine a service that computes personalized greetings. +In this example, we imagine a service that computes personalized greetings. Our service, consist out of the following components: * `kafka ingress` - This component forwards messages produced to the `names` kafka topic, -to the `person` stateful function. Messages produced to this topic has the following +to the `person` stateful function. Messages produced to this topic has the following schema `{ "name" : "bob"}`. * `person` - This function is triggered by the ingress defined above. @@ -14,7 +14,7 @@ This function keeps track of the number of visits, and triggers the next functio * `greeter` - This function, computes a personalized greeting, based on the name and the number of visits of that user. The output of that computation is forward to a Kafka egress defined below. - + * `kafka egress` - This wraps a Kafka producer that emits `utf-8` greetings to the `greetings` Kafka topic. @@ -30,11 +30,7 @@ docker-compose up To observe the customized greeting, as they appear in the `greetings` Kafka topic, run in a separate terminal: ``` -docker-compose exec kafka kafka-console-consumer \ - --bootstrap-server kafka:9092 \ - --isolation-level read_committed \ - --from-beginning \ - --topic greetings +docker-compose exec kafka rpk topic consume greetings ``` Try adding few more input lines to [input-example.json](input-example.json), and restart @@ -42,7 +38,7 @@ the producer service. ``` docker-compose restart producer -``` +``` Feeling curious? add the following log to the `person` function at [functions.js](functions.js): ```console.log(`Hello there ${context.self.id}!`);```. diff --git a/javascript/greeter/docker-compose.yml b/javascript/greeter/docker-compose.yml index f72a966..8e4e8a3 100644 --- a/javascript/greeter/docker-compose.yml +++ b/javascript/greeter/docker-compose.yml @@ -47,24 +47,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Simple Kafka JSON producer to simulate ingress events diff --git a/javascript/showcase/docker-compose.yml b/javascript/showcase/docker-compose.yml index e07f6d6..2494a6f 100644 --- a/javascript/showcase/docker-compose.yml +++ b/javascript/showcase/docker-compose.yml @@ -36,24 +36,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Forward a port 8000 from the host's machine diff --git a/python/greeter/README.md b/python/greeter/README.md index 72996b9..0a5aad8 100644 --- a/python/greeter/README.md +++ b/python/greeter/README.md @@ -2,11 +2,11 @@ This is a simple example of a stateful functions application implemented in `Python`. -In this example, we imagine a service that computes personalized greetings. +In this example, we imagine a service that computes personalized greetings. Our service, consist out of the following components: * `kafka ingress` - This component forwards messages produced to the `names` kafka topic, -to the `person` stateful function. Messages produced to this topic has the following +to the `person` stateful function. Messages produced to this topic has the following schema `{ "name" : "bob"}`. * `person` - This function is triggered by the ingress defined above. @@ -14,7 +14,7 @@ This function keeps track of the number of visits, and triggers the next functio * `greeter` - This function, computes a personalized greeting, based on the name and the number of visits of that user. The output of that computation is forward to a Kafka egress defined below. - + * `kafka egress` - This wraps a Kafka producer that emits `utf-8` greetings to the `greetings` Kafka topic. @@ -30,11 +30,7 @@ docker-compose up To observe the customized greeting, as they appear in the `greetings` Kafka topic, run in a separate terminal: ``` -docker-compose exec kafka kafka-console-consumer \ - --bootstrap-server kafka:9092 \ - --isolation-level read_committed \ - --from-beginning \ - --topic greetings +docker-compose exec kafka rpk topic consume greetings ``` Try adding few more input lines to [input-example.json](input-example.json), and restart @@ -42,7 +38,7 @@ the producer service. ``` docker-compose restart producer -``` +``` Feeling curious? add the following print to the `person` function at [functions.py](functions.py): ```print(f"Hello there {context.address.id}!", flush=True)```. diff --git a/python/greeter/docker-compose.yml b/python/greeter/docker-compose.yml index f72a966..8e4e8a3 100644 --- a/python/greeter/docker-compose.yml +++ b/python/greeter/docker-compose.yml @@ -47,24 +47,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Simple Kafka JSON producer to simulate ingress events diff --git a/python/showcase/docker-compose.yml b/python/showcase/docker-compose.yml index e07f6d6..2494a6f 100644 --- a/python/showcase/docker-compose.yml +++ b/python/showcase/docker-compose.yml @@ -36,24 +36,24 @@ services: # Kafka for ingress and egress ############################################################### - zookeeper: - image: confluentinc/cp-zookeeper:5.4.3 - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ports: - - "2181:2181" - kafka: - image: confluentinc/cp-kafka:5.4.3 + image: docker.vectorized.io/vectorized/redpanda:v21.8.1 + command: + - redpanda start + - --smp 1 + - --memory 512M + - --overprovisioned + - --set redpanda.default_topic_replications=1 + - --set redpanda.auto_create_topics_enabled=true + - --kafka-addr INSIDE://0.0.0.0:9094,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr INSIDE://kafka:9094,OUTSIDE://kafka:9092 + - --pandaproxy-addr 0.0.0.0:8089 + - --advertise-pandaproxy-addr kafka:8089 + hostname: kafka ports: + - "8089:8089" - "9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + - "9094:9094" ############################################################### # Forward a port 8000 from the host's machine
