Repository: bigtop Updated Branches: refs/heads/master d39fa855a -> 9e324fe9f
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e324fe9/bigtop-packages/src/charm/giraph/layer-giraph/reactive/giraph.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/giraph/layer-giraph/reactive/giraph.py b/bigtop-packages/src/charm/giraph/layer-giraph/reactive/giraph.py new file mode 100644 index 0000000..49ba0c7 --- /dev/null +++ b/bigtop-packages/src/charm/giraph/layer-giraph/reactive/giraph.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from jujubigdata import utils +from path import Path + +from charms.reactive import is_state, when, when_not, set_state +from charms.layer.apache_bigtop_base import Bigtop, get_package_version +from charmhelpers.core import hookenv + + +def get_good_jars(dir, prefix=True): + """ + Walk a directory (non-recursively) and return a list of (good) jars. + + Some jars included in giraph have classes that are incompatible with yarn + nodemanagers. Filter these out when constructing a list of good + jars. If omitting the entire .jar is too coarse, the jar will need to be + reconstructed with the offending .class removed. + + param: str dir: Directory to walk + param: bool prefix: When true, prepend the directory to each jar entry + """ + # Known incompatible jars: + # - hive-exec-0.11.0 protobuf class + # java.lang.VerifyError: ... overrides final method getUnknownFields + bad_jars = ['hive-exec-0.11.0.jar'] + good_jars = [] + for file in os.listdir(dir): + if file.endswith('.jar') and file not in bad_jars: + good_jars.append(Path(dir / file) if prefix else file) + + return good_jars + + +@when('bigtop.available') +def report_status(): + """Set juju status based on the deployment topology.""" + giraph_joined = is_state('giraph.joined') + giraph_installed = is_state('giraph.installed') + if not giraph_joined: + hookenv.status_set('blocked', + 'waiting for relation to a giraph host') + elif giraph_installed: + hookenv.status_set('active', + 'ready') + + +@when('bigtop.available', 'giraph.joined') +@when_not('giraph.installed') +def install_giraph(giraph): + """Install giraph when prerequisite states are present.""" + hookenv.status_set('maintenance', 'installing giraph') + bigtop = Bigtop() + bigtop.render_site_yaml( + roles=[ + 'giraph-client', + ], + ) + bigtop.trigger_puppet() + giraph_home = Path('/usr/lib/giraph') + giraph_libdir = Path(giraph_home / 'lib') + giraph_examples = Path('{}/resources/giraph-examples-1.1.0.jar'.format( + hookenv.charm_dir())) + + # Gather a list of all the giraph jars (needed for -libjars) + giraph_jars = [giraph_examples] + giraph_jars.extend(get_good_jars(giraph_home, prefix=True)) + giraph_jars.extend(get_good_jars(giraph_libdir, prefix=True)) + + # Update environment with appropriate giraph bits. HADOOP_CLASSPATH can + # use wildcards (and it should for readability), but GIRAPH_JARS, which + # is intended to be used as 'hadoop jar -libjars $GIRAPH_JARS', needs to + # be a comma-separate list of jars. + with utils.environment_edit_in_place('/etc/environment') as env: + cur_cp = env['HADOOP_CLASSPATH'] if 'HADOOP_CLASSPATH' in env else "" + env['GIRAPH_HOME'] = giraph_home + env['HADOOP_CLASSPATH'] = "{ex}:{home}/*:{libs}/*:{cp}".format( + ex=giraph_examples, + home=giraph_home, + libs=giraph_libdir, + cp=cur_cp + ) + env['GIRAPH_JARS'] = ','.join(j for j in giraph_jars) + + set_state('giraph.installed') + report_status() + # set app version string for juju status output + giraph_version = get_package_version('giraph') or 'unknown' + hookenv.application_version_set(giraph_version) http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e324fe9/bigtop-packages/src/charm/giraph/layer-giraph/resources/giraph-examples-1.1.0.jar ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/giraph/layer-giraph/resources/giraph-examples-1.1.0.jar b/bigtop-packages/src/charm/giraph/layer-giraph/resources/giraph-examples-1.1.0.jar new file mode 100644 index 0000000..63e0a48 Binary files /dev/null and b/bigtop-packages/src/charm/giraph/layer-giraph/resources/giraph-examples-1.1.0.jar differ http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e324fe9/bigtop-packages/src/charm/giraph/layer-giraph/resources/tiny_graph.txt ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/giraph/layer-giraph/resources/tiny_graph.txt b/bigtop-packages/src/charm/giraph/layer-giraph/resources/tiny_graph.txt new file mode 100644 index 0000000..637e9c7 --- /dev/null +++ b/bigtop-packages/src/charm/giraph/layer-giraph/resources/tiny_graph.txt @@ -0,0 +1,5 @@ +[0,0,[[1,1],[3,3]]] +[1,0,[[0,1],[2,2],[3,1]]] +[2,0,[[1,2],[4,4]]] +[3,0,[[0,3],[1,1],[4,4]]] +[4,0,[[3,4],[2,4]]] http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e324fe9/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-basic-deployment.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-basic-deployment.py b/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-basic-deployment.py new file mode 100755 index 0000000..7fc31fc --- /dev/null +++ b/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-basic-deployment.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Trivial deployment test for Giraph. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='xenial') + cls.d.add('giraph', 'cs:xenial/giraph') + cls.d.setup(timeout=900) + cls.d.sentry.wait(timeout=1800) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e324fe9/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-giraph-test.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-giraph-test.py b/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-giraph-test.py new file mode 100755 index 0000000..1e4d2a9 --- /dev/null +++ b/bigtop-packages/src/charm/giraph/layer-giraph/tests/01-giraph-test.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Deployment and smoke test for Apache Giraph. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='xenial') + cls.d.add('giraph', 'cs:xenial/giraph') + cls.d.add('client', 'cs:xenial/hadoop-client') + cls.d.add('resourcemanager', 'cs:xenial/hadoop-resourcemanager') + cls.d.add('namenode', 'cs:xenial/hadoop-namenode') + cls.d.add('slave', 'cs:xenial/hadoop-slave') + cls.d.add('plugin', 'cs:xenial/hadoop-plugin') + + cls.d.relate('plugin:hadoop-plugin', 'client:hadoop') + cls.d.relate('plugin:namenode', 'namenode:namenode') + cls.d.relate('plugin:resourcemanager', 'resourcemanager:resourcemanager') + cls.d.relate('slave:namenode', 'namenode:datanode') + cls.d.relate('slave:resourcemanager', 'resourcemanager:nodemanager') + cls.d.relate('namenode:namenode', 'resourcemanager:namenode') + cls.d.relate('giraph:giraph', 'client:giraph') + + cls.d.setup(timeout=3600) + cls.d.sentry.wait_for_messages({"giraph": "ready"}, timeout=3600) + cls.giraph = cls.d.sentry['giraph'][0] + + def test_giraph(self): + """ + Validate Giraph by running the smoke-test action. + """ + uuid = self.giraph.run_action('smoke-test') + result = self.d.action_fetch(uuid, full_output=True) + print(result) + # action status=completed on success + if (result['status'] != "completed"): + self.fail('Giraph smoke-test failed: %s' % result) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e324fe9/bigtop-packages/src/charm/giraph/layer-giraph/tests/tests.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/giraph/layer-giraph/tests/tests.yaml b/bigtop-packages/src/charm/giraph/layer-giraph/tests/tests.yaml new file mode 100644 index 0000000..3b6ce3e --- /dev/null +++ b/bigtop-packages/src/charm/giraph/layer-giraph/tests/tests.yaml @@ -0,0 +1,3 @@ +reset: false +packages: + - amulet
