Repository: bigtop Updated Branches: refs/heads/master 42146ecbf -> 20bfd7d0a
BIGTOP-2765: fix roles logic for spark/zeppelin charms Closes #206 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/20bfd7d0 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/20bfd7d0 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/20bfd7d0 Branch: refs/heads/master Commit: 20bfd7d0a60bf9c4332484849fa24c0a3f4b5c24 Parents: 42146ec Author: Kevin W Monroe <[email protected]> Authored: Wed May 10 17:26:27 2017 +0000 Committer: Kevin W Monroe <[email protected]> Committed: Thu May 11 11:41:44 2017 -0500 ---------------------------------------------------------------------- .../lib/charms/layer/bigtop_spark.py | 40 +++++++++++--------- .../lib/charms/layer/bigtop_zeppelin.py | 16 ++++---- 2 files changed, 29 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/20bfd7d0/bigtop-packages/src/charm/spark/layer-spark/lib/charms/layer/bigtop_spark.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/spark/layer-spark/lib/charms/layer/bigtop_spark.py b/bigtop-packages/src/charm/spark/layer-spark/lib/charms/layer/bigtop_spark.py index 1be1072..fa8b5e7 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/lib/charms/layer/bigtop_spark.py +++ b/bigtop-packages/src/charm/spark/layer-spark/lib/charms/layer/bigtop_spark.py @@ -18,7 +18,6 @@ from jujubigdata import utils from path import Path from charms.layer.apache_bigtop_base import Bigtop -from charms.reactive import is_state from charms import layer from charmhelpers.core import hookenv, host, unitdata from charmhelpers.fetch.archiveurl import ArchiveUrlFetchHandler @@ -52,14 +51,6 @@ class Spark(object): master = 'yarn-client' return master - def get_roles(self): - roles = ['spark-worker', 'spark-client'] - zk_units = unitdata.kv().get('zookeeper.units', []) - if is_state('leadership.is_leader') or zk_units: - roles.append('spark-master') - roles.append('spark-history-server') - return roles - def install_benchmark(self): """ Install and configure SparkBench. @@ -182,23 +173,27 @@ class Spark(object): :param dict available_hosts: Hosts that Spark should know about. """ - unitdata.kv().set('zookeeper.units', zk_units) - unitdata.kv().set('sparkpeer.units', peers) - unitdata.kv().flush(True) - + # Bootstrap spark if not unitdata.kv().get('spark.bootstrapped', False): self.setup() unitdata.kv().set('spark.bootstrapped', True) + # Set KV based on connected applications + unitdata.kv().set('zookeeper.units', zk_units) + unitdata.kv().set('sparkpeer.units', peers) + unitdata.kv().flush(True) + + # Get our config ready + dc = self.dist_config + events_log_dir = 'file://{}'.format(dc.path('spark_events')) mode = hookenv.config()['spark_execution_mode'] master_ip = utils.resolve_private_address(available_hosts['spark-master']) master_url = self.get_master_url(master_ip) + + # Setup hosts dict hosts = { 'spark': master_ip, } - - dc = self.dist_config - events_log_dir = 'file://{}'.format(dc.path('spark_events')) if 'namenode' in available_hosts: hosts['namenode'] = available_hosts['namenode'] events_log_dir = self.setup_hdfs_logs() @@ -206,14 +201,22 @@ class Spark(object): if 'resourcemanager' in available_hosts: hosts['resourcemanager'] = available_hosts['resourcemanager'] - roles = self.get_roles() + # Setup roles dict. We always include the history server and client. + # Determine other roles based on our execution mode. + roles = ['spark-history-server', 'spark-client'] + if mode == 'standalone': + roles.append('spark-master') + roles.append('spark-worker') + elif mode.startswith('yarn'): + roles.append('spark-on-yarn') + roles.append('spark-yarn-slave') + # Setup overrides dict override = { 'spark::common::master_url': master_url, 'spark::common::event_log_dir': events_log_dir, 'spark::common::history_log_dir': events_log_dir, } - if zk_units: zks = [] for unit in zk_units: @@ -225,6 +228,7 @@ class Spark(object): else: override['spark::common::zookeeper_connection_string'] = None + # Create our site.yaml and trigger puppet bigtop = Bigtop() bigtop.render_site_yaml(hosts, roles, override) bigtop.trigger_puppet() http://git-wip-us.apache.org/repos/asf/bigtop/blob/20bfd7d0/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py index bb7cf08..0bb545a 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py @@ -92,15 +92,6 @@ class Zeppelin(object): self.wait_for_api(30) ########## - ########## - # BUG: BIGTOP-2154 - # The zep deb depends on spark-core and spark-python. However, because - # of the unholy requirement to have hive tightly coupled to spark, - # we need to ensure spark-datanucleus is installed. Do this after the - # initial install so the bigtop repo is available to us. - utils.run_as('root', 'apt-get', 'install', '-qy', 'spark-datanucleus') - ########## - def trigger_bigtop(self): ''' Trigger the Bigtop puppet recipe that handles the Zeppelin service. @@ -108,8 +99,15 @@ class Zeppelin(object): bigtop = Bigtop() overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.', strip=True) + + # The zep deb depends on spark-core, spark-python, and unfortunately, + # most of hadoop. Include appropriate roles here to ensure these + # packages are configured in the same way as our other Bigtop + # software deployed with puppet. bigtop.render_site_yaml( roles=[ + 'spark-client', + 'spark-yarn-slave', 'zeppelin-server', ], overrides=overrides,
