devbv commented on a change in pull request #2744: Deploy kafka & zookeeper
cluster with ansible
URL:
https://github.com/apache/incubator-openwhisk/pull/2744#discussion_r152811236
##########
File path: ansible/roles/kafka/tasks/deploy.yml
##########
@@ -1,57 +1,40 @@
---
# This role will install Kafka with Zookeeper in group 'kafka' in the
environment inventory
-- name: "pull the zookeeper:{{ zookeeper.version }} image"
- shell: "docker pull zookeeper:{{ zookeeper.version }}"
- retries: "{{ docker.pull.retries }}"
- delay: "{{ docker.pull.delay }}"
-
-- name: (re)start zookeeper
- docker_container:
- name: zookeeper
- image: zookeeper:{{ zookeeper.version }}
- state: started
- recreate: true
- restart_policy: "{{ docker.restart.policy }}"
- ports:
- - "{{ zookeeper.port }}:2181"
-
-- name: wait until the Zookeeper in this host is up and running
- action: shell (echo ruok; sleep 1) | nc {{ ansible_host }} {{ zookeeper.port
}}
- register: result
- until: (result.rc == 0) and (result.stdout == 'imok')
- retries: 36
- delay: 5
-
-- name: "pull the ches/kafka:{{ kafka.version }} image"
- shell: "docker pull ches/kafka:{{ kafka.version }}"
+- name: "pull the wurstmeister/kafka:{{ kafka.version }} image"
+ shell: "docker pull wurstmeister/kafka:{{ kafka.version }}"
retries: "{{ docker.pull.retries }}"
delay: "{{ docker.pull.delay }}"
- name: (re)start kafka
+ vars:
+ zookeeper_idx: "{{ groups['kafkas'].index(inventory_hostname) %
(groups['zookeepers'] | length) }}"
docker_container:
- name: kafka
- image: ches/kafka:{{ kafka.version }}
+ name: kafka{{ groups['kafkas'].index(inventory_hostname) }}
+ image: wurstmeister/kafka:{{ kafka.version }}
state: started
recreate: true
restart_policy: "{{ docker.restart.policy }}"
- links:
- - "zookeeper:zookeeper"
env:
+ "KAFKA_DEFAULT_REPLICATION_FACTOR": "{{ kafka.replicationFactor }}"
+ "KAFKA_BROKER_ID": "{{ groups['kafkas'].index(inventory_hostname) }}"
"KAFKA_ADVERTISED_HOST_NAME": "{{ ansible_host }}"
+ "KAFKA_ADVERTISED_PORT": "{{ kafka.port +
groups['kafkas'].index(inventory_hostname) }}"
"KAFKA_HEAP_OPTS": "-Xmx{{ kafka.heap }} -Xms{{ kafka.heap }}"
+ "KAFKA_ZOOKEEPER_CONNECT": "{{ zookeeper_connect_string }}"
+ "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "{{ kafka.replicationFactor }}"
Review comment:
New kafka image 'wurstmeister/kafka-docker' use 1 as default value for it.
(Previous image, ches/kafka set its value as 3)
So even though we set replication factor for each topic, one instance fail
makes the whole kafka cluster fail.
https://travis-ci.org/apache/incubator-openwhisk/builds/305709471
This is the test result without setting
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR. You can see that kafka just stucked.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
