Repository: bigtop Updated Branches: refs/heads/master f4515e8c6 -> 39390b270
BIGTOP-2577: kafka charm test updates (closes #157) Signed-off-by: Kevin W Monroe <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/39390b27 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/39390b27 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/39390b27 Branch: refs/heads/master Commit: 39390b2700405d2374b41a7662d4623cd0850c7d Parents: f4515e8 Author: Kevin W Monroe <[email protected]> Authored: Sat Oct 29 19:06:38 2016 +0000 Committer: Kevin W Monroe <[email protected]> Committed: Wed Nov 16 15:51:17 2016 -0600 ---------------------------------------------------------------------- .../src/charm/kafka/layer-kafka/README.md | 38 ++++---- .../charm/kafka/layer-kafka/actions/read-topic | 13 ++- .../charm/kafka/layer-kafka/actions/write-topic | 13 ++- .../charm/kafka/layer-kafka/reactive/kafka.py | 5 +- .../charm/kafka/layer-kafka/tests/01-deploy.py | 29 ++---- .../kafka/layer-kafka/tests/02-smoke-test.py | 37 +++----- .../layer-kafka/tests/10-config-changed.py | 96 +++++++++++--------- 7 files changed, 113 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/README.md ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/README.md b/bigtop-packages/src/charm/kafka/layer-kafka/README.md index 1db627e..0d41677 100644 --- a/bigtop-packages/src/charm/kafka/layer-kafka/README.md +++ b/bigtop-packages/src/charm/kafka/layer-kafka/README.md @@ -19,17 +19,19 @@ Apache Kafka is an open-source message broker project developed by the Apache Software Foundation written in Scala. The project aims to provide a unified, high-throughput, low-latency platform for handling real-time data feeds. Learn -more at [kafka.apache.org](http://kafka.apache.org/). +more at [kafka.apache.org][]. -This charm deploys the Kafka component of the Apache Bigtop platform. +This charm deploys the Kafka component of the [Apache Bigtop][] platform. +[kafka.apache.org]: http://kafka.apache.org/ +[Apache Bigtop]: http://bigtop.apache.org/ -# Deploying / Using + +# Deploying A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the -[getting-started](https://jujucharms.com/docs/2.0/getting-started) -instructions prior to deploying this charm. +up, please follow the [getting-started][] instructions prior to deploying this +charm. Kafka requires the Zookeeper distributed coordination service. Deploy and relate them as follows: @@ -38,6 +40,17 @@ relate them as follows: juju deploy zookeeper juju add-relation kafka zookeeper +## Network-Restricted Environments +Charms can be deployed in environments with limited network access. To deploy +in this environment, configure a Juju model with appropriate proxy and/or +mirror options. See [Configuring Models][] for more information. + +[getting-started]: https://jujucharms.com/docs/stable/getting-started +[Configuring Models]: https://jujucharms.com/docs/stable/models-config + + +# Using + Once deployed, there are a number of actions available in this charm. > **Note**: Actions described below assume Juju 2.0 or greater. If using an earlier version of Juju, the action syntax is: @@ -83,7 +96,7 @@ are ready: This is particularly useful when combined with `watch` to track the on-going progress of the deployment: - watch -n 0.5 juju status + watch -n 2 juju status The message column will provide information about a given unit's state. This charm is ready for use once the status message indicates that it is @@ -102,7 +115,7 @@ of Juju, the syntax is `juju action do kafka/0 smoke-test`. Watch the progress of the smoke test actions with: - watch -n 0.5 juju show-action-status + watch -n 2 juju show-action-status > **Note**: The above assumes Juju 2.0 or greater. If using an earlier version of Juju, the syntax is `juju action status`. @@ -202,15 +215,6 @@ machine, simply pass ``0.0.0.0`` to ``network_interface``. juju config kafka network_interface=0.0.0.0 -# Network-Restricted Environments - -Charms can be deployed in environments with limited network access. To deploy -in this environment, configure a Juju model with appropriate -proxy and/or mirror options. See -[Configuring Models](https://jujucharms.com/docs/2.0/models-config) for more -information. - - # Contact Information - <[email protected]> http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic index b385f66..8b7a27d 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic @@ -15,16 +15,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -sys.path.append('lib') - import kafkautils import subprocess +import sys +sys.path.append('lib') # Add our helpers to our path. -from charmhelpers.core import hookenv, host -from charms.layer.apache_bigtop_base import get_layer_opts -from charms.reactive import is_state -from jujubigdata import utils +from charmhelpers.core import hookenv, host # noqa: E402 +from charms.layer.apache_bigtop_base import get_layer_opts # noqa: E402 +from charms.reactive import is_state # noqa: E402 +from jujubigdata import utils # noqa: E402 if not is_state('kafka.started'): http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic index b879b65..6a56265 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic @@ -15,16 +15,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -sys.path.append('lib') - import kafkautils import subprocess +import sys +sys.path.append('lib') # Add our helpers to our path. -from charmhelpers.core import hookenv, host -from charms.layer.apache_bigtop_base import get_layer_opts -from charms.reactive import is_state -from jujubigdata import utils +from charmhelpers.core import hookenv, host # noqa: E402 +from charms.layer.apache_bigtop_base import get_layer_opts # noqa: E402 +from charms.reactive import is_state # noqa: E402 +from jujubigdata import utils # noqa: E402 if not is_state('kafka.started'): http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py b/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py index 6e7d325..97f96d6 100644 --- a/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py +++ b/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py @@ -14,7 +14,7 @@ # limitations under the License. from charmhelpers.core import hookenv -from charms.layer.apache_bigtop_base import get_layer_opts +from charms.layer.apache_bigtop_base import get_layer_opts, get_package_version from charms.layer.bigtop_kafka import Kafka from charms.reactive import set_state, remove_state, when, when_not from charms.reactive.helpers import data_changed @@ -44,6 +44,9 @@ def configure_kafka(zk): kafka.open_ports() set_state('kafka.started') hookenv.status_set('active', 'ready') + # set app version string for juju status output + kafka_version = get_package_version('kafka') or 'unknown' + hookenv.application_version_set(kafka_version) @when('kafka.started', 'zookeeper.ready') http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py index af11f9c..62d6a55 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py @@ -15,34 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import amulet +import unittest class TestDeploy(unittest.TestCase): """ - Trivial deployment test for Apache Kafka. + Trivial deployment test for Apache Bigtop Kafka. """ @classmethod def setUpClass(cls): - cls.d = amulet.Deployment(series='trusty') - cls.d.add('kafka', 'kafka') - cls.d.add('openjdk', 'openjdk') - cls.d.add('zk', 'zookeeper') - - cls.d.configure('openjdk', {'java-type': 'jdk', - 'java-major': '8'}) - - cls.d.relate('kafka:zookeeper', 'zk:zookeeper') - cls.d.relate('kafka:java', 'openjdk:java') - try: - cls.d.relate('zk:java', 'openjdk:java') - except ValueError: - # No need to related older versions of the zookeeper charm - # to java. - pass - - cls.d.setup(timeout=900) + cls.d = amulet.Deployment(series='xenial') + cls.d.add('kafka', charm='kafka') + cls.d.add('zookeeper', charm='cs:xenial/zookeeper') + + cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper') + + cls.d.setup(timeout=1800) cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800) cls.kafka = cls.d.sentry['kafka'][0] http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py index 330cde9..f396bdb 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py @@ -15,34 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import amulet +import unittest class TestDeploy(unittest.TestCase): """ - Smoke test of Apache Kafka. + Smoke test for Apache Bigtop Kafka. """ @classmethod def setUpClass(cls): - cls.d = amulet.Deployment(series='trusty') - cls.d.add('kafka', 'kafka') - cls.d.add('openjdk', 'openjdk') - cls.d.add('zk', 'zookeeper') - - cls.d.configure('openjdk', {'java-type': 'jdk', - 'java-major': '8'}) - - cls.d.relate('kafka:zookeeper', 'zk:zookeeper') - cls.d.relate('kafka:java', 'openjdk:java') - try: - cls.d.relate('zk:java', 'openjdk:java') - except ValueError: - # No need to related older versions of the zookeeper charm - # to java. - pass - - cls.d.setup(timeout=900) + cls.d = amulet.Deployment(series='xenial') + cls.d.add('kafka', charm='kafka') + cls.d.add('zookeeper', charm='cs:xenial/zookeeper') + + cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper') + + cls.d.setup(timeout=1800) cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800) cls.kafka = cls.d.sentry['kafka'][0] @@ -50,9 +39,11 @@ class TestDeploy(unittest.TestCase): """ Validate Kafka by running the smoke-test action. """ - smk_uuid = self.kafka.action_do("smoke-test") - output = self.d.action_fetch(smk_uuid, full_output=True) - assert "completed" in output['status'] + uuid = self.kafka.run_action('smoke-test') + result = self.d.action_fetch(uuid, full_output=True) + # action status=completed on success + if (result['status'] != "completed"): + self.fail('Kafka smoke-test failed: %s' % result) if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/bigtop/blob/39390b27/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py index dd20c53..4fd44ce 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py +++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py @@ -15,83 +15,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import amulet import re +import time +import unittest class TestConfigChanged(unittest.TestCase): """ - Test to verify that we update network interface bindings successfully. - + Test to verify that we can bind to listen for client connections + on a specific interface. """ @classmethod def setUpClass(cls): - cls.d = amulet.Deployment(series='trusty') - cls.d.log.debug("foo!") - cls.d.add('kafka', 'kafka') - cls.d.add('openjdk', 'openjdk') - cls.d.add('zk', 'zookeeper') - - cls.d.configure('openjdk', {'java-type': 'jdk', - 'java-major': '8'}) - - cls.d.relate('kafka:zookeeper', 'zk:zookeeper') - cls.d.relate('kafka:java', 'openjdk:java') - try: - cls.d.relate('zk:java', 'openjdk:java') - except ValueError: - # No need to related older versions of the zookeeper charm - # to java. - pass - - cls.d.setup(timeout=900) + cls.d = amulet.Deployment(series='xenial') + cls.d.add('kafka', charm='kafka') + cls.d.add('zookeeper', charm='cs:xenial/zookeeper') + + cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper') + + cls.d.setup(timeout=1800) cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800) - cls.kafka = cls.d.sentry['kafka'][0] + cls.unit = cls.d.sentry['kafka'][0] def test_bind_network_interface(self): """ - Test to verify that we update network interface bindings successfully. - + Verify that we update client port bindings successfully. """ - self.d.configure('kafka', {'network_interface': 'eth0'}) - self.d.sentry.wait_for_messages({'kafka': 'updating zookeeper instances'}, timeout=600) - - self.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=600) - ret = self.kafka.run( + network_interface = None + # Regular expression should handle interfaces in the format + # eth[n], and in the format en[foo] (the "predicatble + # interface names" in v197+ of systemd). + ethernet_interface = re.compile('^e[thn]+.*') + interfaces, _ = self.unit.run( + "ifconfig -a | sed 's/[ \t].*//;/^$/d'") + interfaces = interfaces.split() # Splits on newlines + for interface in interfaces: + if ethernet_interface.match(interface): + network_interface = interface + break + + if network_interface is None: + raise Exception( + "Could not find any interface on the unit that matched my " + "criteria.") + self.d.configure('kafka', {'network_interface': network_interface}) + + # NB: we used to watch for a maintenance status message, but every now + # and then, we'd miss it. Wait 2m to let the config-changed hook settle. + time.sleep(120) + ret = self.unit.run( 'grep host.name /etc/kafka/conf/server.properties')[0] + # Correct line should start with host.name (no comment hash # mark), followed by an equals sign and something that looks # like an IP address (we aren't too strict about it being a # valid ip address.) matcher = re.compile("^host\.name=\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}.*") - - self.assertTrue('host.name' in ret) self.assertTrue(matcher.match(ret)) # Verify that smoke tests still run - smk_uuid = self.kafka.action_do("smoke-test") - output = self.d.action_fetch(smk_uuid, full_output=True) - assert "completed" in output['status'] + smk_uuid = self.unit.run_action('smoke-test') + result = self.d.action_fetch(smk_uuid, full_output=True) + # actions set status=completed on success + if (result['status'] != "completed"): + self.fail('Kafka test failed after setting nic config: %s' % result) def test_reset_network_interface(self): """ - Verify that we can reset the network interface to 0. - + Verify that we can reset the client port bindings to 0.0.0.0 """ self.d.configure('kafka', {'network_interface': '0.0.0.0'}) - self.d.sentry.wait_for_messages({'kafka': 'updating zookeeper instances'}, timeout=600) - self.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=600) - ret = self.kafka.run( + + # NB: we used to watch for a maintenance status message, but every now + # and then, we'd miss it. Wait 2m to let the config-changed hook settle. + time.sleep(120) + ret = self.unit.run( 'grep host.name /etc/kafka/conf/server.properties')[0] matcher = re.compile("^host\.name=0\.0\.0\.0.*") self.assertTrue(matcher.match(ret)) # Verify that smoke tests still run - smk_uuid = self.kafka.action_do("smoke-test") - output = self.d.action_fetch(smk_uuid, full_output=True) - assert "completed" in output['status'] + smk_uuid = self.unit.run_action('smoke-test') + result = self.d.action_fetch(smk_uuid, full_output=True) + # actions set status=completed on success + if (result['status'] != "completed"): + self.fail('Kafka test failed after resetting nic config: %s' % result) if __name__ == '__main__':
