Repository: bigtop Updated Branches: refs/heads/master 4747169cc -> f482fd9fb
BIGTOP-2486: Add Kafka Charm (closes #127) Signed-off-by: Kevin W Monroe <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f482fd9f Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f482fd9f Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f482fd9f Branch: refs/heads/master Commit: f482fd9fb2a8d9f76bd5ca1c23f28d526516e52a Parents: 4747169 Author: Konstantinos Tsakalozos <[email protected]> Authored: Thu Jun 2 16:18:37 2016 +0300 Committer: Kevin W Monroe <[email protected]> Committed: Sat Oct 8 12:57:34 2016 -0500 ---------------------------------------------------------------------- .../src/charm/kafka/layer-kafka/README.md | 227 +++++++++++++++++++ .../src/charm/kafka/layer-kafka/actions.yaml | 44 ++++ .../kafka/layer-kafka/actions/create-topic | 50 ++++ .../kafka/layer-kafka/actions/kafkautils.py | 38 ++++ .../charm/kafka/layer-kafka/actions/list-topics | 42 ++++ .../charm/kafka/layer-kafka/actions/list-zks | 36 +++ .../charm/kafka/layer-kafka/actions/read-topic | 55 +++++ .../charm/kafka/layer-kafka/actions/smoke-test | 64 ++++++ .../charm/kafka/layer-kafka/actions/write-topic | 54 +++++ .../src/charm/kafka/layer-kafka/config.yaml | 10 + .../src/charm/kafka/layer-kafka/copyright | 16 ++ .../src/charm/kafka/layer-kafka/icon.svg | 90 ++++++++ .../src/charm/kafka/layer-kafka/layer.yaml | 20 ++ .../lib/charms/layer/bigtop_kafka.py | 88 +++++++ .../src/charm/kafka/layer-kafka/metadata.yaml | 30 +++ .../charm/kafka/layer-kafka/reactive/kafka.py | 89 ++++++++ .../charm/kafka/layer-kafka/tests/01-deploy.py | 58 +++++ .../kafka/layer-kafka/tests/02-smoke-test.py | 59 +++++ .../layer-kafka/tests/10-config-changed.py | 98 ++++++++ .../charm/kafka/layer-kafka/tests/tests.yaml | 3 + 20 files changed, 1171 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/README.md ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/README.md b/bigtop-packages/src/charm/kafka/layer-kafka/README.md new file mode 100644 index 0000000..1db627e --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/README.md @@ -0,0 +1,227 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +# Overview + +Apache Kafka is an open-source message broker project developed by the Apache +Software Foundation written in Scala. The project aims to provide a unified, +high-throughput, low-latency platform for handling real-time data feeds. Learn +more at [kafka.apache.org](http://kafka.apache.org/). + +This charm deploys the Kafka component of the Apache Bigtop platform. + + +# Deploying / Using + +A working Juju installation is assumed to be present. If Juju is not yet set +up, please follow the +[getting-started](https://jujucharms.com/docs/2.0/getting-started) +instructions prior to deploying this charm. + +Kafka requires the Zookeeper distributed coordination service. Deploy and +relate them as follows: + + juju deploy kafka + juju deploy zookeeper + juju add-relation kafka zookeeper + +Once deployed, there are a number of actions available in this charm. +> **Note**: Actions described below assume Juju 2.0 or greater. If using an +earlier version of Juju, the action syntax is: +`juju action do kafka/0 <action_name> <action_args>; juju action fetch <id>`. + +List the zookeeper servers that our kafka brokers +are connected to. The following will list `<ip>:<port>` information for each +zookeeper unit in the environment (e.g.: `10.0.3.221:2181`). + + juju run-action kafka/0 list-zks + juju show-action-output <id> # <-- id from above command + +Create a Kafka topic with: + + juju run-action kafka/0 create-topic topic=<topic_name> \ + partitions=<#> replication=<#> + juju show-action-output <id> # <-- id from above command + +List topics with: + + juju run-action kafka/0 list-topics + juju show-action-output <id> # <-- id from above command + +Write to a topic with: + + juju run-action kafka/0 write-topic topic=<topic_name> data=<data> + juju show-action-output <id> # <-- id from above command + +Read from a topic with: + + juju run-action kafka/0 read-topic topic=<topic_name> partition=<#> + juju show-action-output <id> # <-- id from above command + + +# Verifying + +## Status +Apache Bigtop charms provide extended status reporting to indicate when they +are ready: + + juju status + +This is particularly useful when combined with `watch` to track the on-going +progress of the deployment: + + watch -n 0.5 juju status + +The message column will provide information about a given unit's state. +This charm is ready for use once the status message indicates that it is +ready. + +## Smoke Test +This charm provides a `smoke-test` action that can be used to verify the +application is functioning as expected. The test will verify connectivity +between Kafka and Zookeeper, and will test creation and listing of Kafka +topics. Run the action as follows: + + juju run-action slave/0 smoke-test + +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action do kafka/0 smoke-test`. + +Watch the progress of the smoke test actions with: + + watch -n 0.5 juju show-action-status + +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action status`. + +Eventually, the action should settle to `status: completed`. If it +reports `status: failed`, the application is not working as expected. Get +more information about a specific smoke test with: + + juju show-action-output <action-id> + +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action fetch <action-id>`. + + +# Scaling + +Expanding a cluster with many brokers is as easy as adding more Kafka units: + + juju add-unit kafka + +After adding additional brokers, topics may be created with +replication up to the number of ready units. For example, if there are two +ready units, create a replicated topic as follows: + + juju run-action kafka/0 create-topic topic=my-replicated-topic \ + partitions=1 replication=2 + +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action do kafka/0 create-topic <args>`. + +Query the description of the recently created topic: + + juju run --unit kafka/0 'kafka-topics.sh --describe \ + --topic my-replicated-topic --zookeeper <zookeeperip>:2181' + +An expected response should be similar to: + + Topic: my-replicated-topic PartitionCount:1 ReplicationFactor:2 Configs: + Topic: my-replicated-topic Partition: 0 Leader: 2 Replicas: 2,0 Isr: 2,0 + + +# Connecting External Clients + +By default, this charm does not expose Kafka outside of the provider's network. +To allow external clients to connect to Kafka, first expose the service: + + juju expose kafka + +Next, ensure the external client can resolve the short hostname of the kafka +unit. A simple way to do this is to add an `/etc/hosts` entry on the external +kafka client machine. Gather the needed info from juju: + + user@juju-client$ juju run --unit kafka/0 'hostname -s' + kafka-0 + user@juju-client$ juju status --format=yaml kafka/0 | grep public-address + public-address: 40.784.149.135 + +Update `/etc/hosts` on the external kafka client: + + user@kafka-client$ echo "40.784.149.135 kafka-0" | sudo tee -a /etc/hosts + +The external kafka client should now be able to access Kafka by using +`kafka-0:9092` as the broker. + + +# Network Interfaces + +In some network environments, kafka may need to be restricted to +listen for incoming connections on a specific network interface +(e.g.: for security reasons). To do so, configure kafka with either a +network interface name or a CIDR range specifying a subnet. For example: + + juju config kafka network_interface=eth0 + juju config kafka network_interface=10.0.2.0/24 + +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju set-config kafka network_interface=eth0`. + +Each kafka machine in the cluster will lookup the IP address of that +network interface, or find the first network interface with an IP +address in the specified subnet, and bind kafka to that address. + +If a mistake is made and an invalid name for the network interface is +configured, recover by re-configuring with the correct name and then +run "juju resolved" on each unit: + + juju config kafka network_interface=eth0 + juju resolved kafka/0 + +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju set-config kafka network_interface=eth0; +juju resolved -r kafka/0`. + +To go back to listening on any network interface on the +machine, simply pass ``0.0.0.0`` to ``network_interface``. + + juju config kafka network_interface=0.0.0.0 + + +# Network-Restricted Environments + +Charms can be deployed in environments with limited network access. To deploy +in this environment, configure a Juju model with appropriate +proxy and/or mirror options. See +[Configuring Models](https://jujucharms.com/docs/2.0/models-config) for more +information. + + +# Contact Information + +- <[email protected]> + + +# Resources + +- [Apache Bigtop](http://bigtop.apache.org/) home page +- [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) +- [Apache Kafka home page](http://kafka.apache.org/) +- [Apache Kafka issue tracker](https://issues.apache.org/jira/browse/KAFKA) +- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) +- [Juju community](https://jujucharms.com/community) http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions.yaml b/bigtop-packages/src/charm/kafka/layer-kafka/actions.yaml new file mode 100644 index 0000000..5db0d58 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions.yaml @@ -0,0 +1,44 @@ +create-topic: + description: Create a new Kafka topic + params: + topic: + type: string + description: Topic name + partitions: + type: integer + description: Number of partitions for the topic being created + replication: + type: integer + description: Replication factor for each partition in the topic + required: [topic, partitions, replication] + additionalProperties: false +list-topics: + description: List all Kafka topics +list-zks: + description: List ip:port info for connected Zookeeper servers +read-topic: + description: Consume an existing kafka topic + params: + topic: + type: string + description: Topic name + partition: + type: integer + description: Partition to consume + required: [topic, partition] + additionalProperties: false +smoke-test: + description: > + Verify that Kafka is working as expected by listing zookeepers, then + creating/listing/deleting a topic +write-topic: + description: Write to a kafka topic + params: + topic: + type: string + description: Topic name + data: + type: string + description: Data to write to topic + required: [topic, data] + additionalProperties: false http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic new file mode 100755 index 0000000..b402fe8 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kafkautils +import subprocess + +from charmhelpers.core import hookenv, host +from charms.reactive import is_state +from jujubigdata.utils import run_as + + +if not is_state('kafka.started'): + kafkautils.fail('Kafka service not yet ready') + + +# Grab the business +topic_name = hookenv.action_get('topic') +topic_partitions = hookenv.action_get('partitions') +topic_replication = hookenv.action_get('replication') + +# Create the topic if kafka is running +if host.service_available('kafka-server') and host.service_running('kafka-server'): + zookeepers = kafkautils.get_zookeepers() + try: + output = run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--create', + '--topic', topic_name, + '--partitions', topic_partitions, + '--replication-factor', topic_replication, + capture_output=True) + except subprocess.CalledProcessError as e: + kafkautils.fail('Kafka command failed', e.output) + else: + hookenv.action_set({'output': output}) +else: + kafkautils.fail('Kafka service is not running', 'Please start kafka-server') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py b/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py new file mode 100644 index 0000000..52e5299 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import sys + +from charmhelpers.core import hookenv + + +def fail(msg, output='<No output>'): + hookenv.action_set({'output': output}) + hookenv.action_fail(msg) + sys.exit() + + +def get_zookeepers(): + cfg = '/etc/kafka/conf/server.properties' + print(cfg) + file = open(cfg, "r") + + for line in file: + if re.search('^zookeeper.connect=.*', line): + zks = line.split("=")[1].strip('\n') + return zks + + return None http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics new file mode 100755 index 0000000..0ef3a3f --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kafkautils +import subprocess + +from charmhelpers.core import hookenv, host +from charms.reactive import is_state +from jujubigdata.utils import run_as + + +if not is_state('kafka.started'): + kafkautils.fail('Kafka service not yet ready') + + +# List topics if kafka is running +if host.service_available('kafka-server') and host.service_running('kafka-server'): + zookeepers = kafkautils.get_zookeepers() + try: + output = run_as('kafka', '/usr/lib/kafka/bin/kafka-topics.sh', + '--zookeeper', zookeepers, '--list', + capture_output=True) + except subprocess.CalledProcessError as e: + kafkautils.fail('Kafka command failed', e.output) + else: + hookenv.action_set({'output': output}) +else: + kafkautils.fail('Kafka service is not running') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks new file mode 100755 index 0000000..517d60c --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kafkautils + +from charmhelpers.core import hookenv, host +from charms.reactive import is_state + + +if not is_state('kafka.started'): + kafkautils.fail('Kafka service not yet ready', 'Please deploy kafka and required relations') + + +# List zookeepers if kafka is running +if host.service_available('kafka-server') and host.service_running('kafka-server'): + zookeepers = kafkautils.get_zookeepers() + if zookeepers: + hookenv.action_set({'output': zookeepers}) + else: + kafkautils.fail('No zookeeper.connect string found', 'Please relate kafka to zookeeper') +else: + kafkautils.fail('Kafka service is not running', 'Please start kafka-server') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic new file mode 100755 index 0000000..b385f66 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append('lib') + +import kafkautils +import subprocess + +from charmhelpers.core import hookenv, host +from charms.layer.apache_bigtop_base import get_layer_opts +from charms.reactive import is_state +from jujubigdata import utils + + +if not is_state('kafka.started'): + kafkautils.fail('Kafka service not yet ready') + + +# Grab the business +topic_name = hookenv.action_get('topic') +topic_partition = hookenv.action_get('partition') + +# Read the topic if kafka is running +if host.service_available('kafka-server') and host.service_running('kafka-server'): + host = subprocess.check_output(['hostname', '-s']).decode('utf8').strip() + port = get_layer_opts().port('kafka') + zookeepers = kafkautils.get_zookeepers() + try: + output = utils.run_as('kafka', '/usr/lib/kafka/bin/kafka-simple-consumer-shell.sh', + '--broker-list', '{}:{}'.format(host, port), + '--topic', topic_name, + '--partition', topic_partition, + '--no-wait-at-logend', + capture_output=True) + except subprocess.CalledProcessError as e: + kafkautils.fail('Kafka command failed', e.output) + else: + hookenv.action_set({'output': output}) +else: + kafkautils.fail('Kafka service is not running') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test b/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test new file mode 100755 index 0000000..5c1435a --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kafkautils +import subprocess + +from charmhelpers.core import hookenv, host +from charms.reactive import is_state +from jujubigdata.utils import run_as + + +if not is_state('kafka.started'): + kafkautils.fail('Kafka service not yet ready') + + +# Define smoke test params +topic_name = "smoketest" +topic_partitions = 1 +topic_replication = 1 + +# Smoke only when kafka is running +if host.service_available('kafka-server') and host.service_running('kafka-server'): + # List ZKs + zookeepers = kafkautils.get_zookeepers() + if not zookeepers: + kafkautils.fail('No zookeeper.connect string found') + + # Create a topic + try: + output = run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--create', + '--topic', topic_name, + '--partitions', topic_partitions, + '--replication-factor', topic_replication, + capture_output=True) + except subprocess.CalledProcessError as e: + kafkautils.fail('Kafka command failed', e.output) + + # List topics + try: + output = run_as('kafka', 'kafka-topics.sh', + '--zookeeper', zookeepers, '--list', + capture_output=True) + except subprocess.CalledProcessError as e: + kafkautils.fail('Kafka command failed', e.output) + + # If we haven't failed yet, we passed + hookenv.action_set({'outcome': 'success'}) +else: + kafkautils.fail('Kafka service is not running') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic new file mode 100755 index 0000000..b879b65 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append('lib') + +import kafkautils +import subprocess + +from charmhelpers.core import hookenv, host +from charms.layer.apache_bigtop_base import get_layer_opts +from charms.reactive import is_state +from jujubigdata import utils + + +if not is_state('kafka.started'): + kafkautils.fail('Kafka service not yet ready') + + +# Grab the business +topic_name = hookenv.action_get('topic') +data = hookenv.action_get('data') + +# Write to the topic if kafka is running +if host.service_available('kafka-server') and host.service_running('kafka-server'): + host = subprocess.check_output(['hostname', '-s']).decode('utf8').strip() + port = get_layer_opts().port('kafka') + zookeepers = kafkautils.get_zookeepers() + try: + output = utils.run_as('kafka', 'kafka-console-producer.sh', + '--broker-list', '{}:{}'.format(host, port), + '--topic', topic_name, + capture_output=True, + input=bytes(data, 'UTF-8')) + except subprocess.CalledProcessError as e: + kafkautils.fail('Kafka command failed', e.output) + else: + hookenv.action_set({'output': output}) +else: + kafkautils.fail('Kafka service is not running') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/config.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/config.yaml b/bigtop-packages/src/charm/kafka/layer-kafka/config.yaml new file mode 100644 index 0000000..c8e723a --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/config.yaml @@ -0,0 +1,10 @@ +options: + network_interface: + default: "" + type: string + description: | + Network interface to bind Kafka to. Defaults to accepting + connections on all interfaces. Accepts either the name of an + interface (e.g., 'eth0'), or a CIDR range. If the latter, we\'ll + bind to the first interface that we find with an IP address in + that range. http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/copyright b/bigtop-packages/src/charm/kafka/layer-kafka/copyright new file mode 100644 index 0000000..e900b97 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/copyright @@ -0,0 +1,16 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved. +License: Apache License 2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/icon.svg ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/icon.svg b/bigtop-packages/src/charm/kafka/layer-kafka/icon.svg new file mode 100644 index 0000000..1564f99 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/icon.svg @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/"; + xmlns:cc="http://creativecommons.org/ns#"; + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + xmlns:svg="http://www.w3.org/2000/svg"; + xmlns="http://www.w3.org/2000/svg"; + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"; + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"; + width="75pt" + height="117pt" + viewBox="0 0 75 117" + version="1.1" + id="svg3201" + inkscape:version="0.48.4 r9939" + sodipodi:docname="icon.svg"> + <metadata + id="metadata3221"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage"; /> + </cc:Work> + </rdf:RDF> + </metadata> + <defs + id="defs3219" /> + <sodipodi:namedview + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1" + objecttolerance="10" + gridtolerance="10" + guidetolerance="10" + inkscape:pageopacity="0" + inkscape:pageshadow="2" + inkscape:window-width="1236" + inkscape:window-height="847" + id="namedview3217" + showgrid="false" + inkscape:zoom="4.5641627" + inkscape:cx="35.369926" + inkscape:cy="80.60281" + inkscape:window-x="397" + inkscape:window-y="78" + inkscape:window-maximized="0" + inkscape:current-layer="layer1" /> + <path + d="m 0,0.04322 76.8,0 0,76.8 -76.8,0 0,-76.8 z" + id="path3203" + style="opacity:0;fill:#fffffe;fill-opacity:0.94977172" + inkscape:connector-curvature="0" /> + <path + d="M 13.76256,3.6403482 C 20.992,-0.05524154 33.024,1.9665021 36.42368,7.4934765 40.37632,12.889169 34.816,19.735528 25.77408,20.917066 c 0,1.732924 0.01024,3.459282 0,5.185642 4.6592,0.708923 8.87808,2.258051 12.05248,4.56205 2.21184,-0.892717 4.526079,-1.700102 6.72768,-2.605948 -0.59392,-3.209847 -0.59392,-6.833231 3.10272,-9.340718 5.77536,-4.719589 18.51392,-4.476718 23.72608,0.531692 4.751359,4.076308 3.266559,10.450051 -3.21536,13.403898 -6.03136,3.012923 -15.28832,2.271179 -20.41856,-1.266872 -2.16064,0.833641 -4.43392,1.549128 -6.52288,2.454975 -0.307201,1.122461 0.45056,2.251486 0.6144,3.373948 0.70656,1.956103 -0.78848,3.859693 -0.68608,5.809231 2.11968,0.951795 4.47488,1.68041 6.79936,2.415589 6.36928,-4.653948 19.445759,-3.918768 24.10496,1.575386 5.12,5.087179 0.62464,12.668717 -8.52992,14.211281 -6.85056,1.273436 -14.7968,-0.794256 -17.84832,-4.988717 -1.81248,-2.23836 -1.69984,-4.857437 -0.8704,-7.259898 -2.29376,-0.951795 -4.67968,-1.805128 -6.97344,-2.756923 - 3.16416,2.284307 -7.34208,3.885949 -12.01152,4.516102 -0.06144,1.746051 -0.04096,3.498667 -0.0512,5.251283 4.78208,0.728615 8.98048,2.921025 10.81344,5.868307 2.75456,4.299487 0.07168,9.872411 -6.51264,12.025436 -6.2464,2.422154 -15.06304,1.234052 -19.33312,-2.546872 -4.89472,-3.846564 -3.84,-10.095589 2.17088,-13.213539 1.91488,-1.102768 4.41344,-1.634461 6.78912,-2.19241 0.03072,-1.68041 0.04096,-3.367384 0.07168,-5.047794 C 14.98112,50.009169 10.21952,48.597887 7.26016,45.919734 1.44384,41.180451 2.2528,33.671118 9.20576,29.581682 c 2.78528,-1.93641 6.77888,-2.776616 10.567679,-3.577436 -0.03072,-1.68041 -0.04096,-3.360821 -0.07168,-5.034667 C 15.59552,20.181887 11.60192,18.678708 9.5641599,16.164655 5.82656,12.081784 7.69024,6.3644508 13.76256,3.6403482 z" + id="path3205" + inkscape:connector-curvature="0" + style="fill:#201f1f" /> + <path + d="M 18.95424,7.4869124 C 23.58272,5.3338867 30.53568,8.2614765 29.85984,11.799528 29.48096,14.924041 23.5008,17.182092 19.2,15.462297 14.42816,13.926297 14.27456,9.1410662 18.95424,7.4869124 z" + id="path3207" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 55.76704,20.844861 c 4.51584,-1.673846 10.91584,0.761436 10.56768,4.135384 0.235519,3.649642 -7.33184,5.96677 -11.64288,3.557744 -4.106241,-1.897025 -3.38944,-6.255589 1.0752,-7.693128 z" + id="path3209" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 18.78016,32.305784 c 7.311359,-2.303999 16.35328,2.829129 13.8752,7.75877 -1.44384,4.555487 -11.17184,6.721641 -16.5376,3.472409 -6.05184,-2.894768 -4.352,-9.570461 2.6624,-11.231179 z" + id="path3211" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 54.69184,48.348451 c 4.66944,-2.619077 12.759039,0.347897 11.601919,4.253538 -0.409599,3.629949 -8.345599,5.218462 -12.103679,2.651898 -3.2256,-1.772308 -2.8672,-5.303795 0.50176,-6.905436 z" + id="path3213" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <path + d="m 20.67456,61.030297 c 5.10976,-1.13559 10.78272,2.566565 8.82688,5.809231 -1.269761,3.190153 -8.48896,4.483282 -11.84768,1.857641 -4.06528,-2.19241 -2.03776,-6.872615 3.0208,-7.666872 z" + id="path3215" + inkscape:connector-curvature="0" + style="fill:#fffffe" /> + <g + inkscape:groupmode="layer" + id="layer1" + inkscape:label="Alpha" + style="opacity:1" /> +</svg> http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/layer.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/layer.yaml b/bigtop-packages/src/charm/kafka/layer-kafka/layer.yaml new file mode 100644 index 0000000..daa7f58 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/layer.yaml @@ -0,0 +1,20 @@ +repo: [email protected]:juju-solutions/layer-apache-bigtop-kafka.git +includes: + - 'layer:apache-bigtop-base' + - 'interface:zookeeper' + - 'interface:kafka' +options: + apache-bigtop-base: + groups: + - kafka + users: + kafka: + groups: ['kafka'] + ports: + # Ports that need to be exposed, overridden, or manually specified. + # Only expose ports serving a UI or external API (i.e., namenode and + # resourcemanager). Communication among units within the cluster does + # not need ports to be explicitly opened. + kafka: + port: 9092 + exposed_on: 'kafka' http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/lib/charms/layer/bigtop_kafka.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/lib/charms/layer/bigtop_kafka.py b/bigtop-packages/src/charm/kafka/layer-kafka/lib/charms/layer/bigtop_kafka.py new file mode 100755 index 0000000..c2f1fc4 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/lib/charms/layer/bigtop_kafka.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from charmhelpers.core import hookenv +from charmhelpers.core import host +from jujubigdata import utils +from charms.layer.apache_bigtop_base import Bigtop +from charms import layer +from subprocess import check_output + + +class Kafka(object): + """ + This class manages Kafka. + """ + def __init__(self): + self.dist_config = utils.DistConfig( + data=layer.options('apache-bigtop-base')) + + def open_ports(self): + for port in self.dist_config.exposed_ports('kafka'): + hookenv.open_port(port) + + def close_ports(self): + for port in self.dist_config.exposed_ports('kafka'): + hookenv.close_port(port) + + def configure_kafka(self, zk_units, network_interface=None): + # Get ip:port data from our connected zookeepers + zks = [] + for unit in zk_units: + ip = utils.resolve_private_address(unit['host']) + zks.append("%s:%s" % (ip, unit['port'])) + zks.sort() + zk_connect = ",".join(zks) + service, unit_num = os.environ['JUJU_UNIT_NAME'].split('/', 1) + kafka_port = self.dist_config.port('kafka') + + roles = ['kafka-server'] + override = { + 'kafka::server::broker_id': unit_num, + 'kafka::server::port': kafka_port, + 'kafka::server::zookeeper_connection_string': zk_connect, + } + if network_interface: + ip = Bigtop().get_ip_for_interface(network_interface) + override['kafka::server::bind_addr'] = ip + + bigtop = Bigtop() + bigtop.render_site_yaml(roles=roles, overrides=override) + bigtop.trigger_puppet() + self.set_advertise() + self.restart() + + def restart(self): + self.stop() + self.start() + + def start(self): + host.service_start('kafka-server') + + def stop(self): + host.service_stop('kafka-server') + + def set_advertise(self): + short_host = check_output(['hostname', '-s']).decode('utf8').strip() + + # Configure server.properties + # NB: We set the advertised.host.name below to our short hostname + # to kafka (admin will still have to expose kafka and ensure the + # external client can resolve the short hostname to our public ip). + kafka_server_conf = '/etc/kafka/conf/server.properties' + utils.re_edit_in_place(kafka_server_conf, { + r'^#?advertised.host.name=.*': 'advertised.host.name=%s' % short_host, + }) http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml b/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml new file mode 100644 index 0000000..5a11394 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml @@ -0,0 +1,30 @@ +name: kafka +summary: High-performance distributed messaging system from Apache Bigtop +maintainer: Juju Big Data <[email protected]> +description: | + Fast + A single Kafka broker can handle hundreds of megabytes of reads and writes per + second from thousands of clients. + + Scalable + Kafka is designed to allow a single cluster to serve as the central data + backbone for a large organization. It can be elastically and transparently + expanded without downtime. Data streams are partitioned and spread over a + cluster of machines to allow data streams larger than the capability of any + single machine and to allow clusters of co-ordinated consumers. + + Durable + Messages are persisted on disk and replicated within the cluster to prevent + data loss. Each broker can handle terabytes of messages without performance + impact. + + Distributed by Design + Kafka has a modern cluster-centric design that offers strong durability and + fault-tolerance guarantees. +tags: [] +provides: + client: + interface: kafka +requires: + zookeeper: + interface: zookeeper http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py b/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py new file mode 100644 index 0000000..6e7d325 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core import hookenv +from charms.layer.apache_bigtop_base import get_layer_opts +from charms.layer.bigtop_kafka import Kafka +from charms.reactive import set_state, remove_state, when, when_not +from charms.reactive.helpers import data_changed + + +@when('bigtop.available') +@when_not('zookeeper.joined') +def waiting_for_zookeeper(): + hookenv.status_set('blocked', 'waiting for relation to zookeeper') + + +@when('bigtop.available', 'zookeeper.joined') +@when_not('kafka.started', 'zookeeper.ready') +def waiting_for_zookeeper_ready(zk): + hookenv.status_set('waiting', 'waiting for zookeeper to become ready') + + +@when('bigtop.available', 'zookeeper.ready') +@when_not('kafka.started') +def configure_kafka(zk): + hookenv.status_set('maintenance', 'setting up kafka') + data_changed( # Prime data changed for network interface + 'kafka.network_interface', hookenv.config().get('network_interface')) + kafka = Kafka() + zks = zk.zookeepers() + kafka.configure_kafka(zks) + kafka.open_ports() + set_state('kafka.started') + hookenv.status_set('active', 'ready') + + +@when('kafka.started', 'zookeeper.ready') +def configure_kafka_zookeepers(zk): + """Configure ready zookeepers and restart kafka if needed. + + As zks come and go, server.properties will be updated. When that file + changes, restart Kafka and set appropriate status messages. + + This method also handles the restart if our network_interface + config has changed. + + """ + zks = zk.zookeepers() + network_interface = hookenv.config().get('network_interface') + if not data_changed('zookeepers', zks) and not data_changed( + 'kafka.network_interface', network_interface): + return + + hookenv.log('Checking Zookeeper configuration') + hookenv.status_set('maintenance', 'updating zookeeper instances') + kafka = Kafka() + kafka.configure_kafka(zks, network_interface) + hookenv.status_set('active', 'ready') + + +@when('kafka.started') +@when_not('zookeeper.ready') +def stop_kafka_waiting_for_zookeeper_ready(): + hookenv.status_set('maintenance', 'zookeeper not ready, stopping kafka') + kafka = Kafka() + kafka.close_ports() + kafka.stop() + remove_state('kafka.started') + hookenv.status_set('waiting', 'waiting for zookeeper to become ready') + + +@when('client.joined', 'zookeeper.ready') +def serve_client(client, zookeeper): + kafka_port = get_layer_opts().port('kafka') + client.send_port(kafka_port) + client.send_zookeepers(zookeeper.zookeepers()) + hookenv.log('Sent Kafka configuration to client') http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py new file mode 100755 index 0000000..af11f9c --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Trivial deployment test for Apache Kafka. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='trusty') + cls.d.add('kafka', 'kafka') + cls.d.add('openjdk', 'openjdk') + cls.d.add('zk', 'zookeeper') + + cls.d.configure('openjdk', {'java-type': 'jdk', + 'java-major': '8'}) + + cls.d.relate('kafka:zookeeper', 'zk:zookeeper') + cls.d.relate('kafka:java', 'openjdk:java') + try: + cls.d.relate('zk:java', 'openjdk:java') + except ValueError: + # No need to related older versions of the zookeeper charm + # to java. + pass + + cls.d.setup(timeout=900) + cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800) + cls.kafka = cls.d.sentry['kafka'][0] + + def test_deploy(self): + """ + Simple test to make sure the Kafka java process is running. + """ + output, retcode = self.kafka.run("pgrep -a java") + assert 'Kafka' in output, "Kafka daemon is not started" + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py new file mode 100755 index 0000000..330cde9 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Smoke test of Apache Kafka. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='trusty') + cls.d.add('kafka', 'kafka') + cls.d.add('openjdk', 'openjdk') + cls.d.add('zk', 'zookeeper') + + cls.d.configure('openjdk', {'java-type': 'jdk', + 'java-major': '8'}) + + cls.d.relate('kafka:zookeeper', 'zk:zookeeper') + cls.d.relate('kafka:java', 'openjdk:java') + try: + cls.d.relate('zk:java', 'openjdk:java') + except ValueError: + # No need to related older versions of the zookeeper charm + # to java. + pass + + cls.d.setup(timeout=900) + cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800) + cls.kafka = cls.d.sentry['kafka'][0] + + def test_kafka(self): + """ + Validate Kafka by running the smoke-test action. + """ + smk_uuid = self.kafka.action_do("smoke-test") + output = self.d.action_fetch(smk_uuid, full_output=True) + assert "completed" in output['status'] + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py new file mode 100755 index 0000000..dd20c53 --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet +import re + + +class TestConfigChanged(unittest.TestCase): + """ + Test to verify that we update network interface bindings successfully. + + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='trusty') + cls.d.log.debug("foo!") + cls.d.add('kafka', 'kafka') + cls.d.add('openjdk', 'openjdk') + cls.d.add('zk', 'zookeeper') + + cls.d.configure('openjdk', {'java-type': 'jdk', + 'java-major': '8'}) + + cls.d.relate('kafka:zookeeper', 'zk:zookeeper') + cls.d.relate('kafka:java', 'openjdk:java') + try: + cls.d.relate('zk:java', 'openjdk:java') + except ValueError: + # No need to related older versions of the zookeeper charm + # to java. + pass + + cls.d.setup(timeout=900) + cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800) + cls.kafka = cls.d.sentry['kafka'][0] + + def test_bind_network_interface(self): + """ + Test to verify that we update network interface bindings successfully. + + """ + self.d.configure('kafka', {'network_interface': 'eth0'}) + self.d.sentry.wait_for_messages({'kafka': 'updating zookeeper instances'}, timeout=600) + + self.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=600) + ret = self.kafka.run( + 'grep host.name /etc/kafka/conf/server.properties')[0] + # Correct line should start with host.name (no comment hash + # mark), followed by an equals sign and something that looks + # like an IP address (we aren't too strict about it being a + # valid ip address.) + matcher = re.compile("^host\.name=\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}.*") + + self.assertTrue('host.name' in ret) + self.assertTrue(matcher.match(ret)) + + # Verify that smoke tests still run + smk_uuid = self.kafka.action_do("smoke-test") + output = self.d.action_fetch(smk_uuid, full_output=True) + assert "completed" in output['status'] + + def test_reset_network_interface(self): + """ + Verify that we can reset the network interface to 0. + + """ + self.d.configure('kafka', {'network_interface': '0.0.0.0'}) + self.d.sentry.wait_for_messages({'kafka': 'updating zookeeper instances'}, timeout=600) + self.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=600) + ret = self.kafka.run( + 'grep host.name /etc/kafka/conf/server.properties')[0] + + matcher = re.compile("^host\.name=0\.0\.0\.0.*") + self.assertTrue(matcher.match(ret)) + + # Verify that smoke tests still run + smk_uuid = self.kafka.action_do("smoke-test") + output = self.d.action_fetch(smk_uuid, full_output=True) + assert "completed" in output['status'] + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/bigtop/blob/f482fd9f/bigtop-packages/src/charm/kafka/layer-kafka/tests/tests.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/tests.yaml b/bigtop-packages/src/charm/kafka/layer-kafka/tests/tests.yaml new file mode 100644 index 0000000..3b6ce3e --- /dev/null +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/tests.yaml @@ -0,0 +1,3 @@ +reset: false +packages: + - amulet
