Repository: bigtop Updated Branches: refs/heads/master 08aa3a267 -> 4ad7f6bd2
BIGTOP-2910: zeppelin charm: support bigtop upgrade Fixes #298 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4ad7f6bd Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4ad7f6bd Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4ad7f6bd Branch: refs/heads/master Commit: 4ad7f6bd291c467b3a4af76ae7baba5eb13fcacb Parents: 08aa3a2 Author: Kevin W Monroe <[email protected]> Authored: Thu Oct 19 21:21:50 2017 +0000 Committer: Kevin W Monroe <[email protected]> Committed: Fri Oct 20 11:21:16 2017 -0500 ---------------------------------------------------------------------- .../charm/zeppelin/layer-zeppelin/actions.yaml | 9 ++- .../zeppelin/layer-zeppelin/actions/reinstall | 59 +++++++++++++++++ .../lib/charms/layer/bigtop_zeppelin.py | 70 +++++++++++--------- .../layer-zeppelin/reactive/zeppelin.py | 50 ++++++++++---- .../layer-zeppelin/tests/02-zeppelin-smoke.py | 2 +- .../tests/03-zeppelin-spark-smoke.py | 2 +- .../layer-zeppelin/tests/04-zeppelin-config.py | 48 ++++++++++++++ 7 files changed, 191 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml index 7ce817e..375a8ca 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml @@ -1,7 +1,10 @@ -smoke-test: +reinstall: description: > - Verify that Zeppelin is working by running one of - the example notebook paragraphs, using the REST server. + Reinstall Zeppelin with the version available in the repo. restart: description: > Restart Zeppelin. +smoke-test: + description: > + Verify that Zeppelin is working by running one of the example notebook + paragraphs, using the REST server. http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall new file mode 100755 index 0000000..7ac8e75 --- /dev/null +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.append('lib') + +from charmhelpers.core import hookenv, unitdata # noqa: E402 +from charms.layer.apache_bigtop_base import Bigtop, get_package_version # noqa: E402 +from charms.reactive import is_state # noqa: E402 + + +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) + hookenv.action_fail(msg) + sys.exit() + + +if not is_state('bigtop.version.changed'): + fail('No Bigtop version changes were found; nothing to reinstall.') + +if not unitdata.kv().get('zeppelin.version.repo', False): + fail('Charm is not prepared to run the reinstall action.') + +# Call the base reinstall method and ensure zeppelin packages +# are removed prior to reinstalling new versions via puppet apply. +bigtop = Bigtop() +result = bigtop.reinstall_repo_packages(remove_pkgs='zeppelin') + +if bigtop.check_bigtop_repo_package('zeppelin'): + # Ruh roh. We expect this to be None since we just did a reinstall + # with the current repo. There should be no different version available. + fail('Unexpected zeppelin version found after reinstalling') + +if result == 'success': + # Set appropriate status output + app_version = get_package_version('zeppelin') or 'unknown' + hookenv.application_version_set(app_version) + hookenv.status_set('active', 'reinstall was successful') + + # Remove our version unitdata and report success + unitdata.kv().unset('zeppelin.version.repo') + hookenv.action_set({'outcome': 'success'}) +else: + fail('Reinstall failed; hiera data and package repos have been restored ' + 'to the previous working state.') http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py index 812a644..c60c0f5 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py @@ -44,8 +44,7 @@ class Zeppelin(object): def install(self): ''' - Perform initial one-time setup, workaround upstream bugs, and - trigger puppet. + Perform initial one-time setup and trigger puppet. ''' # Dirs are handled by the bigtop deb, so no need to call out to # dist_config to do that work. However, we want to adjust the @@ -73,36 +72,8 @@ class Zeppelin(object): self._add_override('spark::common::event_log_dir', events_log_dir) self._add_override('spark::common::history_log_dir', events_log_dir) - ########## - # BUG: BIGTOP-2742 - # Default zeppelin init script looks for the literal '$(hostname)' - # string. Symlink it so it exists before the apt install from puppet - # tries to start the service. - import subprocess - host = subprocess.check_output(['hostname']).decode('utf8').strip() - zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host) - utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin') - utils.run_as('root', 'ln', '-sf', - zepp_pid, - '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid') - ########## - self.trigger_bigtop() - ########## - # BUG: BIGTOP-2742 - # Puppet apply will call systemctl daemon-reload, which removes the - # symlink we just created. Now that the bits are on disk, update the - # init script $(hostname) that caused this mess to begin with. - zepp_init_script = '/etc/init.d/zeppelin' - utils.re_edit_in_place(zepp_init_script, { - r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid), - }) - utils.run_as('root', 'systemctl', 'daemon-reload') - self.restart() - self.wait_for_api(30) - ########## - def trigger_bigtop(self): ''' Trigger the Bigtop puppet recipe that handles the Zeppelin service. @@ -124,8 +95,43 @@ class Zeppelin(object): overrides=overrides, ) - bigtop.trigger_puppet() - self.wait_for_api(30) + # NB: during an upgrade, we configure the site.yaml, but do not + # trigger puppet. The user must do that with the 'reinstall' action. + if unitdata.kv().get('zeppelin.version.repo', False): + hookenv.log("An upgrade is available and the site.yaml has been " + "configured. Run the 'reinstall' action to continue.", + level=hookenv.INFO) + else: + #################################################################### + # BUG: BIGTOP-2742 + # Default zeppelin init script looks for the literal '$(hostname)' + # string. Symlink it so it exists before the apt install from puppet + # tries to start the service. + import subprocess + host = subprocess.check_output(['hostname']).decode('utf8').strip() + zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host) + utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin') + utils.run_as('root', 'ln', '-sf', + zepp_pid, + '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid') + #################################################################### + + bigtop.trigger_puppet() + self.wait_for_api(30) + + #################################################################### + # BUG: BIGTOP-2742 + # Puppet apply will call systemctl daemon-reload, which removes the + # symlink we just created. Now that the bits are on disk, update the + # init script $(hostname) that caused this mess to begin with. + zepp_init_script = '/etc/init.d/zeppelin' + utils.re_edit_in_place(zepp_init_script, { + r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid), + }) + utils.run_as('root', 'systemctl', 'daemon-reload') + self.restart() + self.wait_for_api(30) + #################################################################### def reconfigure_zeppelin(self): ''' http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py index 7c9ca06..d68eccc 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py @@ -15,10 +15,9 @@ import hashlib -from charms.reactive import when, when_not -from charms.reactive import is_state, set_state, remove_state -from charmhelpers.core import hookenv -from charms.layer.apache_bigtop_base import get_package_version +from charms.reactive import is_state, remove_state, set_state, when, when_not +from charmhelpers.core import hookenv, unitdata +from charms.layer.apache_bigtop_base import Bigtop, get_package_version from charms.layer.bigtop_zeppelin import Zeppelin from charms.reactive.helpers import data_changed @@ -58,13 +57,19 @@ def update_status(): elif spark_ready: ready_apps.append('spark') - # Set appropriate status based on the apps we checked above - if waiting_apps: - hookenv.status_set('waiting', - 'waiting for: {}'.format(' & '.join(waiting_apps))) + # Set appropriate status + repo_ver = unitdata.kv().get('zeppelin.version.repo', False) + if repo_ver: + # Pending upgrade takes precedent over other status messages + msg = "install version {} with the 'reinstall' action".format(repo_ver) + hookenv.status_set('active', msg) + elif waiting_apps: + # Waiting takes precedent over active status messages + msg = "waiting for: {}".format(' & '.join(waiting_apps)) + hookenv.status_set('waiting', msg) elif ready_apps: - hookenv.status_set('active', - 'ready with: {}'.format(' & '.join(ready_apps))) + msg = "ready with: {}".format(' & '.join(ready_apps)) + hookenv.status_set('active', msg) else: hookenv.status_set('active', 'ready') @@ -83,6 +88,27 @@ def initial_setup(): hookenv.application_version_set(zeppelin_version) +@when('zeppelin.installed', 'bigtop.version.changed') +def check_repo_version(): + """ + Configure a bigtop site.yaml if a new version of zeppelin is available. + + This method will set unitdata if a different version of zeppelin is + available in the newly configured bigtop repo. This unitdata allows us to + configure site.yaml while gating the actual puppet apply. The user must do + the puppet apply by calling the 'reinstall' action. + """ + repo_ver = Bigtop().check_bigtop_repo_package('zeppelin') + if repo_ver: + unitdata.kv().set('zeppelin.version.repo', repo_ver) + unitdata.kv().flush(True) + zeppelin = Zeppelin() + zeppelin.trigger_bigtop() + else: + unitdata.kv().unset('zeppelin.version.repo') + update_status() + + @when('zeppelin.installed', 'hadoop.ready') @when_not('zeppelin.hadoop.configured') def configure_hadoop(hadoop): @@ -170,7 +196,7 @@ def unconfigure_spark(): update_status() -@when('zeppelin.started', 'client.notebook.registered') +@when('zeppelin.installed', 'client.notebook.registered') def register_notebook(client): zeppelin = Zeppelin() for notebook in client.unregistered_notebooks(): @@ -181,7 +207,7 @@ def register_notebook(client): client.reject_notebook(notebook) -@when('zeppelin.started', 'client.notebook.removed') +@when('zeppelin.installed', 'client.notebook.removed') def remove_notebook(client): zeppelin = Zeppelin() for notebook in client.unremoved_notebooks(): http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py index 0887cbe..b1ec990 100755 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py @@ -20,7 +20,7 @@ import re import unittest -class TestDeploy(unittest.TestCase): +class TestSmoke(unittest.TestCase): """ Smoke test for Apache Bigtop Zeppelin. """ http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py index 56f5f96..b7cb918 100755 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py @@ -20,7 +20,7 @@ import re import unittest -class TestDeploy(unittest.TestCase): +class TestSmokeSpark(unittest.TestCase): """ Smoke test for Apache Bigtop Zeppelin using remote Spark resources. """ http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py new file mode 100755 index 0000000..dac6190 --- /dev/null +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import amulet +import re +import unittest + + +class TestConfig(unittest.TestCase): + """ + Test configuring Apache Zeppelin. + """ + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='xenial') + cls.d.add('zeppelin-test-config', charm='zeppelin', + constraints={'mem': '7G'}) + cls.d.setup(timeout=1800) + cls.d.sentry.wait_for_messages({'zeppelin-test-config': re.compile('ready')}, + timeout=1800) + cls.zeppelin = cls.d.sentry['zeppelin-test-config'][0] + + def test_bigtop_upgrade(self): + """ + Validate Zeppelin status is changed when upgrading zeppelin. + """ + self.d.configure('zeppelin-test-config', + {'bigtop_version': 'master'}) + self.d.sentry.wait_for_messages({'zeppelin-test-config': re.compile('reinstall|ready')}, + timeout=900) + + +if __name__ == '__main__': + unittest.main()
