Repository: bigtop Updated Branches: refs/heads/master c29e841d7 -> c1a210371
BIGTOP-2844: zeppelin charm: consistent spark config Closes #257 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c1a21037 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c1a21037 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c1a21037 Branch: refs/heads/master Commit: c1a21037165334716ec2a22debcd011d89f1074d Parents: c29e841 Author: Kevin W Monroe <[email protected]> Authored: Thu Jul 13 20:27:35 2017 +0000 Committer: Kevin W Monroe <[email protected]> Committed: Thu Aug 3 09:50:31 2017 -0500 ---------------------------------------------------------------------- .../lib/charms/layer/bigtop_zeppelin.py | 10 ++++++---- .../layer-zeppelin/tests/03-zeppelin-spark-smoke.py | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1a21037/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py index 44512d2..812a644 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py @@ -111,7 +111,7 @@ class Zeppelin(object): overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.', strip=True) - # The zep deb depends on spark-core, spark-python, and unfortunately, + # The zep deb depends on spark-core which unfortunately brings in # most of hadoop. Include appropriate roles here to ensure these # packages are configured in the same way as our other Bigtop # software deployed with puppet. @@ -156,9 +156,10 @@ class Zeppelin(object): utils.run_as('hdfs', 'hdfs', 'dfs', '-mkdir', '-p', '/user/zeppelin') utils.run_as('hdfs', 'hdfs', 'dfs', '-chown', 'zeppelin', '/user/zeppelin') - # If spark is ready, let it handle the spark_master_url. Otherwise, - # zepp is in local mode; set it to yarn-client since hadoop is here. + # If spark is ready, let configure_spark() trigger bigtop. Otherwise, + # put our spark in yarn-client mode since hadoop is here. if not is_state('spark.ready'): + self._add_override('spark::common::master_url', 'yarn-client') self._add_override('zeppelin::server::spark_master_url', 'yarn-client') self.trigger_bigtop() @@ -166,7 +167,8 @@ class Zeppelin(object): ''' Configure the zeppelin spark interpreter ''' - # TODO: Need Puppet params created to set Spark driver and executor memory + # TODO: Add config for Spark driver and executor memory overrides + self._add_override('spark::common::master_url', master_url) self._add_override('zeppelin::server::spark_master_url', master_url) self.trigger_bigtop() http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1a21037/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py index daddc51..56f5f96 100755 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py @@ -28,14 +28,26 @@ class TestDeploy(unittest.TestCase): def setUpClass(cls): cls.d = amulet.Deployment(series='xenial') cls.d.add('zeppelin') - cls.d.add('spark') + cls.d.add('zeppelin-test-spark', charm='spark') - cls.d.relate('zeppelin:spark', 'spark:client') + cls.d.relate('zeppelin-test-spark:client', 'zeppelin:spark') cls.d.setup(timeout=1800) cls.d.sentry.wait_for_messages({'zeppelin': re.compile('ready with')}, timeout=1800) cls.zeppelin = cls.d.sentry['zeppelin'][0] + @classmethod + def tearDownClass(cls): + # NB: seems to be a remove_service issue with amulet. However, the + # unit does still get removed. Pass OSError for now: + # OSError: juju command failed ['remove-application', ...]: + # ERROR allocation for service ... owned by ... not found + try: + cls.d.remove_service('zeppelin-test-spark') + except OSError as e: + print("IGNORE: Amulet remove_service failed: {}".format(e)) + pass + def test_zeppelin(self): """ Validate Zeppelin by running the smoke-test action.
