This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 0b23a93 Refactor config (#288)
0b23a93 is described below
commit 0b23a9399f372dd3f17f8cab1f39ed5d57a8eb1e
Author: Min Pae <[email protected]>
AuthorDate: Wed Nov 6 06:05:37 2019 -0800
Refactor config (#288)
The following commits were squashed :
* refactor config.py to separate target specific code
- split config.py to DeployConfig base class and Ec2DeployConfig,
ExistingDeployConfig, and AzureDeployConfig subclasses
- add @abstractmethod annotations for common interfaces to DeployConfig
base class
- move target specific code to respective subclasses
* move config tests to target specific subdirectory
* add config decorators and move ec2 specific config
- add python decorators to annotate config items to allow generic
handling of configs
- moved some ec2 specific configs to Ec2DeployConfig
* add config value validation decorator
* rename host_vars and play_vars to ansible_*
* use config/base.ansible_*_vars to render vars
* add hadoop 3.2.1 checksum and remove azure_vars
- integrate default value handling for host_var and play_var in to base
config class accessor for respective var dictionaries
- remove use of 'azure_vars' and consolidate as host_vars
* remove fstype and force_format from base, existing, and azure config
* remove checks for fstype and force_format from existing and azure tests
* addressing review comments
- made config.shutdown_delay_minutes not abstract so existing and azure
cluster types wouldn't have to implement it
- changed _ansible_vars dictionary values a list of structs rather than
tuples so that each field has an understandable name
- removed instance_tags from base config
---
lib/muchos/config/__init__.py | 37 +++
lib/muchos/config/azure.py | 107 ++++++++
lib/muchos/{config.py => config/base.py} | 416 ++++++++++++-------------------
lib/muchos/config/decorators.py | 111 +++++++++
lib/muchos/config/ec2.py | 184 ++++++++++++++
lib/muchos/config/existing.py | 50 ++++
lib/muchos/config/validators.py | 58 +++++
lib/muchos/existing.py | 42 +---
lib/tests/azure/__init__.py | 17 ++
lib/tests/azure/test_config.py | 84 +++++++
lib/tests/ec2/__init__.py | 17 ++
lib/tests/{ => ec2}/test_config.py | 91 +------
lib/tests/existing/__init__.py | 17 ++
lib/tests/existing/test_config.py | 31 +++
14 files changed, 879 insertions(+), 383 deletions(-)
diff --git a/lib/muchos/config/__init__.py b/lib/muchos/config/__init__.py
new file mode 100644
index 0000000..13294f1
--- /dev/null
+++ b/lib/muchos/config/__init__.py
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from muchos.config.base import BaseConfig, SERVICES, OPTIONAL_SERVICES
+from muchos.config.existing import ExistingDeployConfig
+from muchos.config.ec2 import Ec2DeployConfig
+from muchos.config.azure import AzureDeployConfig
+
+from configparser import ConfigParser
+
+def DeployConfig(deploy_path, config_path, hosts_path, checksums_path,
templates_path, cluster_name):
+ c = ConfigParser()
+ c.read(config_path)
+ cluster_type = c.get('general', 'cluster_type')
+
+ if cluster_type == 'existing':
+ return ExistingDeployConfig(deploy_path, config_path, hosts_path,
checksums_path, templates_path, cluster_name)
+
+ if cluster_type == 'ec2':
+ return Ec2DeployConfig(deploy_path, config_path, hosts_path,
checksums_path, templates_path, cluster_name)
+
+ if cluster_type == 'azure':
+ return AzureDeployConfig(deploy_path, config_path, hosts_path,
checksums_path, templates_path, cluster_name)
\ No newline at end of file
diff --git a/lib/muchos/config/azure.py b/lib/muchos/config/azure.py
new file mode 100644
index 0000000..fe93b55
--- /dev/null
+++ b/lib/muchos/config/azure.py
@@ -0,0 +1,107 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from muchos.config import BaseConfig
+from muchos.config.decorators import *
+from muchos.config.validators import *
+from sys import exit
+from muchos.util import get_ephemeral_devices, get_arch
+import os
+import json
+import glob
+
+
+class AzureDeployConfig(BaseConfig):
+ def __init__(self, deploy_path, config_path, hosts_path, checksums_path,
templates_path, cluster_name):
+ super(AzureDeployConfig, self).__init__(deploy_path, config_path,
hosts_path, checksums_path, templates_path, cluster_name)
+
+ def verify_config(self, action):
+ self._verify_config(action)
+
+ proxy = self.get('general', 'proxy_hostname')
+ cluster_type = self.get('general', 'cluster_type')
+ if cluster_type not in ['azure']:
+ if not proxy:
+ exit("ERROR - proxy.hostname must be set in muchos.props")
+
+ if proxy not in self.node_d:
+ exit("ERROR - The proxy (set by property proxy_hostname={0})
cannot be found in 'nodes' section of "
+ "muchos.props".format(proxy))
+
+ def verify_launch(self):
+ pass
+
+ def node_type_map(self):
+ return {}
+
+ def mount_root(self):
+ return self.get('azure', 'mount_root')
+
+ def data_dirs_common(self, nodeType):
+ data_dirs = []
+
+ num_disks = int(self.get("azure", "numdisks"))
+ range_var = num_disks + 1
+ for diskNum in range(1, range_var):
+ data_dirs.append(self.get("azure", "mount_root") +
+ str(diskNum))
+
+ return data_dirs
+
+ def metrics_drive_ids(self):
+ drive_ids = []
+ range_var = int(self.get("azure", "numdisks")) + 1
+ for i in range(1, range_var):
+ drive_ids.append(self.get("azure", "metrics_drive_root") +
+ str(i))
+ return drive_ids
+
+ @ansible_host_var
+ @default(None)
+ def azure_fileshare_mount(self):
+ return self.get('azure', 'azure_fileshare_mount')
+
+ @ansible_host_var
+ @default(None)
+ def azure_fileshare(self):
+ return self.get('azure', 'azure_fileshare')
+
+ @ansible_host_var
+ @default(None)
+ def azure_fileshare_username(self):
+ return self.get('azure', 'azure_fileshare_username')
+
+ @ansible_host_var
+ @default(None)
+ def azure_fileshare_password(self):
+ return self.get('azure', 'azure_fileshare_password')
+
+ @ansible_host_var(name='az_omsIntegrationNeeded')
+ @default(False)
+ @is_valid(is_in([True, False]))
+ def omsIntegrationNeeded(self):
+ return self.getboolean('azure', 'az_omsIntegrationNeeded')
+
+ @ansible_host_var(name='az_logs_id')
+ @default(None)
+ def logs_id(self):
+ return self.get('azure', 'az_logs_id')
+
+ @ansible_host_var(name='az_logs_key')
+ @default(None)
+ def logs_key(self):
+ return self.get('azure', 'az_logs_key')
\ No newline at end of file
diff --git a/lib/muchos/config.py b/lib/muchos/config/base.py
similarity index 61%
rename from lib/muchos/config.py
rename to lib/muchos/config/base.py
index c62c282..9c6696a 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config/base.py
@@ -15,9 +15,14 @@
# limitations under the License.
#
+from abc import ABCMeta, abstractmethod
+
+from collections import ChainMap
from configparser import ConfigParser
from sys import exit
-from .util import get_ephemeral_devices, get_arch
+from muchos.config.decorators import *
+from muchos.config.validators import *
+from muchos.util import get_ephemeral_devices, get_arch
import os
import json
import glob
@@ -27,39 +32,137 @@ SERVICES = ['zookeeper', 'namenode', 'resourcemanager',
'accumulomaster', 'mesos
OPTIONAL_SERVICES = ['fluo', 'fluo_yarn', 'metrics', 'mesosmaster', 'spark',
'client', 'swarmmanager', 'journalnode', 'zkfc']
+_HOST_VAR_DEFAULTS = {
+ 'accumulo_home': '"{{ install_dir }}/accumulo-{{ accumulo_version }}"',
+ 'accumulo_instance': None,
+ 'accumulo_major_version': '"{{ accumulo_version.split(\'.\')[0] }}"',
+ 'accumulo_password': None,
+ 'accumulo_tarball': 'accumulo-{{ accumulo_version }}-bin.tar.gz',
+ 'accumulo_version': None,
+ 'cluster_type': None,
+ 'cluster_group': None,
+ 'cluster_user': None,
+ 'default_data_dirs': None,
+ 'download_software': None,
+ 'fluo_home': '"{{ install_dir }}/fluo-{{ fluo_version }}"',
+ 'fluo_tarball': 'fluo-{{ fluo_version }}-bin.tar.gz',
+ 'fluo_version': None,
+ 'fluo_yarn_home': '"{{ install_dir }}/fluo-yarn-{{ fluo_yarn_version }}"',
+ 'fluo_yarn_tarball': 'fluo-yarn-{{ fluo_yarn_version }}-bin.tar.gz',
+ 'fluo_yarn_version': None,
+ 'hadoop_home': '"{{ install_dir }}/hadoop-{{ hadoop_version }}"',
+ 'hadoop_tarball': 'hadoop-{{ hadoop_version }}.tar.gz',
+ 'hadoop_version': None,
+ 'hadoop_major_version': '"{{ hadoop_version.split(\'.\')[0] }}"',
+ 'hdfs_root': "{% if hdfs_ha %}hdfs://{{ nameservice_id }}{% else %}hdfs://{{
groups[\'namenode\'][0] }}:8020{% endif %}",
+ 'hdfs_ha': None,
+ 'nameservice_id': None,
+ 'install_dir': None,
+ 'install_hub': None,
+ 'java_home': '"/usr/lib/jvm/java"',
+ 'java_package': '"java-1.8.0-openjdk-devel"',
+ 'journal_quorum': "{% for host in groups['journalnode'] %}{{ host }}:8485{%
if not loop.last %};{% endif %}{% endfor %}",
+ 'maven_home': '"{{ install_dir }}/apache-maven-{{ maven_version }}"',
+ 'maven_tarball': 'apache-maven-{{ maven_version }}-bin.tar.gz',
+ 'maven_version': '3.6.1',
+ 'spark_home': '"{{ install_dir }}/spark-{{ spark_version
}}-bin-without-hadoop"',
+ 'spark_tarball': 'spark-{{ spark_version }}-bin-without-hadoop.tgz',
+ 'spark_version': None,
+ 'tarballs_dir': '"{{ user_home }}/tarballs"',
+ 'user_home': None,
+ 'worker_data_dirs': None,
+ 'zookeeper_connect': "{% for host in groups['zookeepers'] %}{{ host
}}:2181{% if not loop.last %},{% endif %}{% endfor %}",
+ 'zookeeper_client_port': '"2181"',
+ 'zookeeper_home': '"{{ install_dir }}/zookeeper-{{ zookeeper_version }}"',
+ 'zookeeper_tarball': 'zookeeper-{{ zookeeper_version }}.tar.gz',
+ 'zookeeper_version': None
+}
+
+_PLAY_VAR_DEFAULTS = {
+ 'accumulo_dcache_size': None,
+ 'accumulo_icache_size': None,
+ 'accumulo_imap_size': None,
+ 'accumulo_sha256': None,
+ 'accumulo_tserv_mem': None,
+ 'fluo_sha256': None,
+ 'fluo_worker_instances_multiplier': None,
+ 'fluo_worker_mem_mb': None,
+ 'fluo_worker_threads': None,
+ 'fluo_yarn_sha256': None,
+ 'hadoop_sha256': None,
+ 'hub_version': '2.2.3',
+ 'hub_home': '"{{ install_dir }}/hub-linux-amd64-{{ hub_version }}"',
+ 'hub_tarball': 'hub-linux-amd64-{{ hub_version }}.tgz',
+ 'hub_sha256':
'54c35a459a4241b7ae4c28bcfea0ceef849dd2f8a9dd2b82ba2ba964a743e6bc',
+ 'maven_sha256':
'2528c35a99c30f8940cc599ba15d34359d58bec57af58c1075519b8cd33b69e7',
+ 'metrics_drive_ids': None,
+ 'mount_root': None,
+ 'node_type_map': None,
+ 'spark_sha256': None,
+ 'shutdown_delay_minutes': None,
+ 'twill_reserve_mem_mb': None,
+ 'yarn_nm_mem_mb': None,
+ 'zookeeper_sha256': None
+}
+
+_EXTRA_VAR_DEFAULTS = {}
-class DeployConfig(ConfigParser):
+class BaseConfig(ConfigParser, metaclass=ABCMeta):
def __init__(self, deploy_path, config_path, hosts_path, checksums_path,
templates_path, cluster_name):
- ConfigParser.__init__(self)
+ super(BaseConfig, self).__init__()
self.optionxform = str
self.deploy_path = deploy_path
self.read(config_path)
self.hosts_path = hosts_path
self.cluster_name = cluster_name
- self.sg_name = cluster_name + '-group'
- self.ephemeral_root = 'ephemeral'
self.cluster_type = self.get('general', 'cluster_type')
- self.metrics_drive_root = 'media-' + self.ephemeral_root
self.node_d = None
self.hosts = None
self.checksums_path = checksums_path
self.checksums_d = None
- self.init_nodes()
- self.cluster_template_d = None
- self.init_template(templates_path)
-
+ self._init_nodes()
+
+ def ansible_host_vars(self):
+ return dict(
+ ChainMap(self._ansible_vars_from_decorators('host'),
+ getattr(self, 'HOST_VAR_DEFAULTS', {}),
+ _HOST_VAR_DEFAULTS))
+
+ def ansible_play_vars(self):
+ software_checksums = {
+ '{}_sha256'.format(k): self.checksum(k) for
+ k in ['accumulo', 'fluo', 'fluo_yarn', 'hadoop', 'spark',
'zookeeper']
+ }
+ return dict(
+ ChainMap(self._ansible_vars_from_decorators('play'),
+ software_checksums,
+ getattr(self, 'PLAY_VAR_DEFAULTS', {}),
+ _PLAY_VAR_DEFAULTS))
+
+ def ansible_extra_vars(self):
+ return dict(
+ ChainMap(self._ansible_vars_from_decorators('extra'),
+ getattr(self, 'EXTRA_VAR_DEFAULTS', {}),
+ _EXTRA_VAR_DEFAULTS))
+
+ def _ansible_vars_from_decorators(self, var_type):
+ # only render play_vars for base and cluster specific config
+ f = ["{}deployconfig".format(self.get_cluster_type()), "baseconfig"]
+ return {
+ v.var_name: getattr(self, v.property_name)() for v in
+ # filter out any classes that are not baseconfig or
+ # the cluster specific config
+ filter(lambda t: t.class_name.lower() in f,
+ # get all ansible vars of var_type
+ get_ansible_vars(var_type))}
+
+ @abstractmethod
def verify_config(self, action):
- proxy = self.get('general', 'proxy_hostname')
- cluster_type = self.get('general', 'cluster_type')
- if cluster_type not in ['azure']:
- if not proxy:
- exit("ERROR - proxy.hostname must be set in muchos.props")
-
- if proxy not in self.node_d:
- exit("ERROR - The proxy (set by property proxy_hostname={0})
cannot be found in 'nodes' section of "
- "muchos.props".format(proxy))
+ raise NotImplementedError()
+ # helper for verify_config to call for common checks
+ def _verify_config(self, action):
if action in ['launch', 'setup']:
for service in SERVICES:
if service not in OPTIONAL_SERVICES:
@@ -71,11 +174,11 @@ class DeployConfig(ConfigParser):
if self.java_product_version() >= 11 and
StrictVersion(self.version('accumulo')) <= StrictVersion("1.9.3"):
exit("ERROR - Java 11 is not supported with Accumulo version
'{0}'".format(self.version('accumulo')))
+ @abstractmethod
def verify_launch(self):
- self.verify_instance_type(self.get('ec2', 'default_instance_type'))
- self.verify_instance_type(self.get('ec2', 'worker_instance_type'))
+ raise NotImplementedError()
- def init_nodes(self):
+ def _init_nodes(self):
self.node_d = {}
for (hostname, value) in self.items('nodes'):
if hostname in self.node_d:
@@ -88,27 +191,12 @@ class DeployConfig(ConfigParser):
exit('Unknown service "%s" declared for node %s' %
(service, hostname))
self.node_d[hostname] = service_list
- def default_ephemeral_devices(self):
- return get_ephemeral_devices(self.get('ec2', 'default_instance_type'))
-
- def worker_ephemeral_devices(self):
- return get_ephemeral_devices(self.get('ec2', 'worker_instance_type'))
-
- def max_ephemeral(self):
- return max((len(self.default_ephemeral_devices()),
len(self.worker_ephemeral_devices())))
-
+ @abstractmethod
+ @ansible_play_var
def node_type_map(self):
- if self.cluster_template_d:
- return self.cluster_template_d['devices']
-
- node_types = {}
- if self.get_cluster_type() == 'ec2':
- node_list = [('default', self.default_ephemeral_devices()),
('worker', self.worker_ephemeral_devices())]
-
- for (ntype, devices) in node_list:
- node_types[ntype] = {'mounts': self.mounts(len(devices)),
'devices': devices}
- return node_types
+ raise NotImplementedError()
+ @is_valid(is_in(['default', 'worker']))
def node_type(self, hostname):
if 'worker' in self.node_d[hostname]:
return 'worker'
@@ -123,77 +211,36 @@ class DeployConfig(ConfigParser):
mounts.append(self.mount_root() + str(i))
return mounts
+ @abstractmethod
+ @ansible_play_var
def mount_root(self):
- if self.get_cluster_type() == 'ec2':
- return '/media/' + self.ephemeral_root
- elif self.get_cluster_type() == 'existing':
- return self.get('existing', 'mount_root')
- elif self.get_cluster_type() == 'azure':
- return self.get('azure', 'mount_root')
-
- def fstype(self):
- retval = None
- if self.get_cluster_type() == 'ec2':
- retval = self.get('ec2', 'fstype')
- if not retval:
- return 'ext3'
- return retval
-
- def force_format(self):
- retval = 'no'
- if self.get_cluster_type() == 'ec2':
- retval = self.get('ec2', 'force_format')
- if not retval:
- return 'no'
- return retval
+ raise NotImplementedError()
+ @abstractmethod
def data_dirs_common(self, nodeType):
- data_dirs = []
-
- if self.get_cluster_type() == 'ec2':
- data_dirs = self.node_type_map()[nodeType]['mounts']
- elif self.get_cluster_type() == 'existing':
- data_dirs = self.get('existing', 'data_dirs').split(",")
- elif self.get_cluster_type() == 'azure':
- num_disks = int(self.get("azure", "numdisks"))
- range_var = num_disks + 1
- for diskNum in range(1, range_var):
- data_dirs.append(self.get("azure", "mount_root") +
- str(diskNum))
-
- return data_dirs
+ raise NotImplementedError()
+ @ansible_host_var
def worker_data_dirs(self):
return self.data_dirs_common("worker")
+ @ansible_host_var
def default_data_dirs(self):
return self.data_dirs_common("default")
+ @abstractmethod
+ @ansible_play_var
def metrics_drive_ids(self):
- if self.get_cluster_type() == 'ec2':
- drive_ids = []
- for i in range(0, self.max_ephemeral()):
- drive_ids.append(self.metrics_drive_root + str(i))
- return drive_ids
- elif self.get_cluster_type() == 'existing':
- return self.get("existing", "metrics_drive_ids").split(",")
- elif self.get_cluster_type() == 'azure':
- drive_ids = []
- range_var = int(self.get("azure", "numdisks")) + 1
- for i in range(1, range_var):
- drive_ids.append(self.get("azure", "metrics_drive_root") +
- str(i))
- return drive_ids
+ raise NotImplementedError()
+ @ansible_play_var
def shutdown_delay_minutes(self):
- retval = '0'
- if self.get_cluster_type() == 'ec2':
- retval = self.get("ec2", "shutdown_delay_minutes")
- return retval
+ return '0'
def version(self, software_id):
return self.get('general', software_id + '_version')
+ @ansible_host_var
def java_product_version(self):
java_version_map = {
"java-1.8.0-openjdk": 8,
@@ -238,22 +285,6 @@ class DeployConfig(ConfigParser):
exit('ERROR - Failed to find checksums for %s %s in %s' %
(software, version, self.checksums_path))
return self.checksums_d[key]
- def verify_instance_type(self, instance_type):
- if not self.cluster_template_d:
- if get_arch(instance_type) == 'pvm':
- exit("ERROR - Configuration contains instance type '{0}' that
uses pvm architecture."
- "Only hvm architecture is
supported!".format(instance_type))
-
- def instance_tags(self):
- retd = {}
- if self.has_option('ec2', 'instance_tags'):
- value = self.get('ec2', 'instance_tags')
- if value:
- for kv in value.split(','):
- (key, val) = kv.split(':')
- retd[key] = val
- return retd
-
def nodes(self):
return self.node_d
@@ -266,6 +297,7 @@ class DeployConfig(ConfigParser):
return True
return False
+ # test method, might want to make private or just move to test module
def get_host_services(self):
retval = []
for (hostname, service_list) in list(self.node_d.items()):
@@ -273,6 +305,7 @@ class DeployConfig(ConfigParser):
retval.sort()
return retval
+ # test method, might want to make private or just move to test module
def get_service_private_ips(self, service):
retval = []
for (hostname, service_list) in list(self.node_d.items()):
@@ -289,6 +322,7 @@ class DeployConfig(ConfigParser):
retval.sort()
return retval
+ # test method, might want to make private or just move to test module
def get_non_proxy(self):
retval = []
proxy_ip = self.get_private_ip(self.get('general', 'proxy_hostname'))
@@ -383,153 +417,13 @@ class DeployConfig(ConfigParser):
print(self.get(section, key))
return
exit("Property '{0}' was not found".format(key))
-
- def init_template(self, templates_path):
- if self.has_option('ec2', 'cluster_template'):
- template_id = self.get('ec2', 'cluster_template')
- template_path = os.path.join(templates_path, template_id)
- if os.path.exists(template_path):
- self.cluster_template_d = {'id': template_id}
- self.load_template_ec2_requests(template_path)
- self.load_template_device_map(template_path)
- self.validate_template()
-
- def load_template_ec2_requests(self, template_dir):
- for json_path in glob.glob(os.path.join(template_dir, '*.json')):
- service = os.path.basename(json_path).rsplit('.', 1)[0]
- if service not in SERVICES:
- exit("ERROR - Template '{0}' has unrecognized option '{1}'.
Must be one of {2}".format(
- self.cluster_template_d['id'], service, str(SERVICES)))
- with open(json_path, 'r') as json_file:
- # load as string, so we can use string.Template to inject
config values
- self.cluster_template_d[service] = json_file.read()
-
- def load_template_device_map(self, template_dir):
- device_map_path = os.path.join(template_dir, 'devices')
- if not os.path.isfile(device_map_path):
- exit("ERROR - template '{0}' is missing 'devices'
config".format(self.cluster_template_d['id']))
- with open(device_map_path, 'r') as json_file:
- self.cluster_template_d['devices'] = json.load(json_file)
-
- def validate_template(self):
- if not self.cluster_template_d:
- exit("ERROR - Template '{0}' is not
defined!".format(self.get('ec2', 'cluster_template')))
-
- if 'worker' not in self.cluster_template_d:
- exit("ERROR - '{0}' template config is invalid. No 'worker' launch
request is defined".format(
- self.cluster_template_d['id']))
-
- if 'worker' not in self.cluster_template_d['devices']:
- exit("ERROR - '{0}' template is invalid. The devices file must
have a 'worker' device map".format(
- self.cluster_template_d['id']))
-
- if 'default' not in self.cluster_template_d['devices']:
- exit("ERROR - '{0}' template is invalid. The devices file must
have a 'default' device map".format(
- self.cluster_template_d['id']))
-
- # Validate the selected launch template for each host
-
- worker_count = 0
- for hostname in self.node_d:
- # first service listed denotes the selected template
- selected_ec2_request = self.node_d[hostname][0]
- if 'worker' == selected_ec2_request:
- worker_count = worker_count + 1
- else:
- if 'worker' in self.node_d[hostname]:
- exit("ERROR - '{0}' node config is invalid. The 'worker'
service should be listed first".format(
- hostname))
- if selected_ec2_request not in self.cluster_template_d:
- if len(self.node_d[hostname]) > 1:
- print('Hint: In template mode, the first service listed
for a host denotes its EC2 template')
- exit("ERROR - '{0}' node config is invalid. No EC2 template
defined for the '{1}' service".format(
- hostname, selected_ec2_request))
-
- if worker_count == 0:
- exit("ERROR - No worker instances are defined for template
'{0}'".format(self.cluster_template_d['id']))
-
-
-HOST_VAR_DEFAULTS = {
- 'accumulo_home': '"{{ install_dir }}/accumulo-{{ accumulo_version }}"',
- 'accumulo_instance': None,
- 'accumulo_major_version': '"{{ accumulo_version.split(\'.\')[0] }}"',
- 'accumulo_password': None,
- 'accumulo_tarball': 'accumulo-{{ accumulo_version }}-bin.tar.gz',
- 'accumulo_version': None,
- 'cluster_type': None,
- 'cluster_group': None,
- 'cluster_user': None,
- 'default_data_dirs': None,
- 'download_software': None,
- 'fluo_home': '"{{ install_dir }}/fluo-{{ fluo_version }}"',
- 'fluo_tarball': 'fluo-{{ fluo_version }}-bin.tar.gz',
- 'fluo_version': None,
- 'fluo_yarn_home': '"{{ install_dir }}/fluo-yarn-{{ fluo_yarn_version }}"',
- 'fluo_yarn_tarball': 'fluo-yarn-{{ fluo_yarn_version }}-bin.tar.gz',
- 'fluo_yarn_version': None,
- 'hadoop_home': '"{{ install_dir }}/hadoop-{{ hadoop_version }}"',
- 'hadoop_tarball': 'hadoop-{{ hadoop_version }}.tar.gz',
- 'hadoop_version': None,
- 'hadoop_major_version': '"{{ hadoop_version.split(\'.\')[0] }}"',
- 'hdfs_root': "{% if hdfs_ha %}hdfs://{{ nameservice_id }}{% else %}hdfs://{{
groups[\'namenode\'][0] }}:8020{% endif %}",
- 'hdfs_ha': None,
- 'nameservice_id': None,
- 'install_dir': None,
- 'install_hub': None,
- 'java_home': '"/usr/lib/jvm/java"',
- 'java_package': '"java-1.8.0-openjdk-devel"',
- 'journal_quorum': "{% for host in groups['journalnode'] %}{{ host }}:8485{%
if not loop.last %};{% endif %}{% endfor %}",
- 'maven_home': '"{{ install_dir }}/apache-maven-{{ maven_version }}"',
- 'maven_tarball': 'apache-maven-{{ maven_version }}-bin.tar.gz',
- 'maven_version': '3.6.1',
- 'spark_home': '"{{ install_dir }}/spark-{{ spark_version
}}-bin-without-hadoop"',
- 'spark_tarball': 'spark-{{ spark_version }}-bin-without-hadoop.tgz',
- 'spark_version': None,
- 'tarballs_dir': '"{{ user_home }}/tarballs"',
- 'user_home': None,
- 'worker_data_dirs': None,
- 'zookeeper_connect': "{% for host in groups['zookeepers'] %}{{ host
}}:2181{% if not loop.last %},{% endif %}{% endfor %}",
- 'zookeeper_client_port': '"2181"',
- 'zookeeper_home': '"{{ install_dir }}/zookeeper-{{ zookeeper_version }}"',
- 'zookeeper_tarball': 'zookeeper-{{ zookeeper_version }}.tar.gz',
- 'zookeeper_version': None
-}
-
-PLAY_VAR_DEFAULTS = {
- 'accumulo_dcache_size': None,
- 'accumulo_icache_size': None,
- 'accumulo_imap_size': None,
- 'accumulo_sha256': None,
- 'accumulo_tserv_mem': None,
- 'fluo_sha256': None,
- 'fluo_worker_instances_multiplier': None,
- 'fluo_worker_mem_mb': None,
- 'fluo_worker_threads': None,
- 'fluo_yarn_sha256': None,
- 'force_format': None,
- 'fstype': None,
- 'hadoop_sha256': None,
- 'hub_version': '2.2.3',
- 'hub_home': '"{{ install_dir }}/hub-linux-amd64-{{ hub_version }}"',
- 'hub_tarball': 'hub-linux-amd64-{{ hub_version }}.tgz',
- 'hub_sha256':
'54c35a459a4241b7ae4c28bcfea0ceef849dd2f8a9dd2b82ba2ba964a743e6bc',
- 'maven_sha256':
'2528c35a99c30f8940cc599ba15d34359d58bec57af58c1075519b8cd33b69e7',
- 'metrics_drive_ids': None,
- 'mount_root': None,
- 'node_type_map': None,
- 'spark_sha256': None,
- 'shutdown_delay_minutes': None,
- 'twill_reserve_mem_mb': None,
- 'yarn_nm_mem_mb': None,
- 'zookeeper_sha256': None
-}
-
-AZURE_VAR_DEFAULTS = {
- 'azure_fileshare_mount': None,
- 'azure_fileshare': None,
- 'azure_fileshare_username': None,
- 'azure_fileshare_password': None,
- 'az_omsIntegrationNeeded': None,
- 'az_logs_id': None,
- 'az_logs_key': None
-}
+
+ def resolve_value(self, config_name, default=None):
+ # listed low to high priority
+ section_priority = ['general', 'ansible-vars', self.get('performance',
'profile')]
+ all_values = [self.get(section, config_name, fallback=None) for
section in section_priority]
+ try:
+ *_, val = filter(None, all_values)
+ except ValueError:
+ return default
+ return val
\ No newline at end of file
diff --git a/lib/muchos/config/decorators.py b/lib/muchos/config/decorators.py
new file mode 100644
index 0000000..57151b8
--- /dev/null
+++ b/lib/muchos/config/decorators.py
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from collections.abc import Iterable
+from functools import wraps
+
+
+# struct to hold information about ansible vars defined via decorators.
+# var_name indicates the desired variable name
+# class_name indicates the class name where the variable was defined
+# property_name indicates the class property/function where the variable was
+# defined
+class _ansible_var(object):
+ def __init__(self, var_name, class_name, property_name):
+ self.var_name = var_name
+ self.class_name = class_name
+ self.property_name = property_name
+
+# each entry of _ansible_vars will contain a list of _ansible_var instances
+_ansible_vars = dict(
+ host=[],
+ play=[],
+ extra=[]
+)
+
+def get_ansible_vars(var_type):
+ return _ansible_vars.get(var_type)
+
+# ansible hosts inventory variables
+def ansible_host_var(name=None):
+ return ansible_var_decorator('host', name)
+
+# ansible group/all variables
+def ansible_play_var(name=None):
+ return ansible_var_decorator('play', name)
+
+# ansible extra variables
+def ansible_extra_var(name=None):
+ return ansible_var_decorator('extra', name)
+
+def ansible_var_decorator(var_type, name):
+ def _decorator(func):
+ ansible_var = _ansible_var(
+ var_name=name if isinstance(name, str) else func.__name__,
+ class_name=func.__qualname__.split('.')[0],
+ property_name=func.__name__)
+ _ansible_vars[var_type].append(ansible_var)
+ return func
+
+ if callable(name):
+ return _decorator(name)
+ return _decorator
+
+def default(val):
+ def _default(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ try:
+ res = func(*args, **kwargs)
+ except:
+ return val
+ else:
+ if res in [None, 0, ''] or len(res) == 0:
+ return val
+ return res
+ return wrapper
+ return _default
+
+def required(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ res = func(*args, **kwargs)
+ if res in [None, 0, ''] or len(res) == 0:
+ raise ConfigMissingError(func.__name__)
+ return res
+ return wrapper
+
+def is_valid(validators):
+ if not isinstance(validators, Iterable):
+ validators = [validators]
+ def _validate(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ res = func(*args, **kwargs)
+ failed_checks = list(filter(lambda f: f(res) is not True,
validators))
+ if len(failed_checks) > 0:
+ raise Exception("{}={} checked validation {}".format(
+ func.__name__, res,
+ [str(v) for v in failed_checks]))
+ return res
+ return wrapper
+ return _validate
+
+class ConfigMissingError(Exception):
+ def __init__(self, name):
+ super(ConfigMissingError, self).__init__("{} is missing from the
configuration".format(name))
+
diff --git a/lib/muchos/config/ec2.py b/lib/muchos/config/ec2.py
new file mode 100644
index 0000000..7eaa4ab
--- /dev/null
+++ b/lib/muchos/config/ec2.py
@@ -0,0 +1,184 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from muchos.config import SERVICES, OPTIONAL_SERVICES
+from muchos.config.base import BaseConfig
+from sys import exit
+from muchos.config.decorators import *
+from muchos.config.validators import *
+from muchos.util import get_ephemeral_devices, get_arch
+import os
+import json
+import glob
+
+
+class Ec2DeployConfig(BaseConfig):
+
+ def __init__(self, deploy_path, config_path, hosts_path, checksums_path,
templates_path, cluster_name):
+ super(Ec2DeployConfig, self).__init__(deploy_path, config_path,
hosts_path, checksums_path, templates_path, cluster_name)
+ self.sg_name = cluster_name + '-group'
+ self.ephemeral_root = 'ephemeral'
+ self.cluster_template_d = None
+ self.metrics_drive_root = 'media-' + self.ephemeral_root
+ self.init_template(templates_path)
+
+ def verify_config(self, action):
+ self._verify_config(action)
+
+ def verify_launch(self):
+ self.verify_instance_type(self.get('ec2', 'default_instance_type'))
+ self.verify_instance_type(self.get('ec2', 'worker_instance_type'))
+
+ def init_nodes(self):
+ self.node_d = {}
+ for (hostname, value) in self.items('nodes'):
+ if hostname in self.node_d:
+ exit('Hostname {0} already exists twice in
nodes'.format(hostname))
+ service_list = []
+ for service in value.split(','):
+ if service in SERVICES:
+ service_list.append(service)
+ else:
+ exit('Unknown service "%s" declared for node %s' %
(service, hostname))
+ self.node_d[hostname] = service_list
+
+ def default_ephemeral_devices(self):
+ return get_ephemeral_devices(self.get('ec2', 'default_instance_type'))
+
+ def worker_ephemeral_devices(self):
+ return get_ephemeral_devices(self.get('ec2', 'worker_instance_type'))
+
+ def max_ephemeral(self):
+ return max((len(self.default_ephemeral_devices()),
len(self.worker_ephemeral_devices())))
+
+ def node_type_map(self):
+ if self.cluster_template_d:
+ return self.cluster_template_d['devices']
+
+ node_types = {}
+
+ node_list = [('default', self.default_ephemeral_devices()), ('worker',
self.worker_ephemeral_devices())]
+
+ for (ntype, devices) in node_list:
+ node_types[ntype] = {'mounts': self.mounts(len(devices)),
'devices': devices}
+
+ return node_types
+
+ def mount_root(self):
+ return '/media/' + self.ephemeral_root
+
+ @ansible_play_var
+ @default('ext3')
+ def fstype(self):
+ return self.get('ec2', 'fstype')
+
+ @ansible_play_var
+ @default('no')
+ def force_format(self):
+ return self.get('ec2', 'force_format')
+
+ def data_dirs_common(self, nodeType):
+ return self.node_type_map()[nodeType]['mounts']
+
+ def metrics_drive_ids(self):
+ drive_ids = []
+ for i in range(0, self.max_ephemeral()):
+ drive_ids.append(self.metrics_drive_root + str(i))
+ return drive_ids
+
+ def shutdown_delay_minutes(self):
+ return self.get("ec2", "shutdown_delay_minutes")
+
+ def verify_instance_type(self, instance_type):
+ if not self.cluster_template_d:
+ if get_arch(instance_type) == 'pvm':
+ exit("ERROR - Configuration contains instance type '{0}' that
uses pvm architecture."
+ "Only hvm architecture is
supported!".format(instance_type))
+
+ def instance_tags(self):
+ retd = {}
+ if self.has_option('ec2', 'instance_tags'):
+ value = self.get('ec2', 'instance_tags')
+ if value:
+ for kv in value.split(','):
+ (key, val) = kv.split(':')
+ retd[key] = val
+ return retd
+
+ def init_template(self, templates_path):
+ if self.has_option('ec2', 'cluster_template'):
+ template_id = self.get('ec2', 'cluster_template')
+ template_path = os.path.join(templates_path, template_id)
+ if os.path.exists(template_path):
+ self.cluster_template_d = {'id': template_id}
+ self.load_template_ec2_requests(template_path)
+ self.load_template_device_map(template_path)
+ self.validate_template()
+
+ def load_template_ec2_requests(self, template_dir):
+ for json_path in glob.glob(os.path.join(template_dir, '*.json')):
+ service = os.path.basename(json_path).rsplit('.', 1)[0]
+ if service not in SERVICES:
+ exit("ERROR - Template '{0}' has unrecognized option '{1}'.
Must be one of {2}".format(
+ self.cluster_template_d['id'], service, str(SERVICES)))
+ with open(json_path, 'r') as json_file:
+ # load as string, so we can use string.Template to inject
config values
+ self.cluster_template_d[service] = json_file.read()
+
+ def load_template_device_map(self, template_dir):
+ device_map_path = os.path.join(template_dir, 'devices')
+ if not os.path.isfile(device_map_path):
+ exit("ERROR - template '{0}' is missing 'devices'
config".format(self.cluster_template_d['id']))
+ with open(device_map_path, 'r') as json_file:
+ self.cluster_template_d['devices'] = json.load(json_file)
+
+ def validate_template(self):
+ if not self.cluster_template_d:
+ exit("ERROR - Template '{0}' is not
defined!".format(self.get('ec2', 'cluster_template')))
+
+ if 'worker' not in self.cluster_template_d:
+ exit("ERROR - '{0}' template config is invalid. No 'worker' launch
request is defined".format(
+ self.cluster_template_d['id']))
+
+ if 'worker' not in self.cluster_template_d['devices']:
+ exit("ERROR - '{0}' template is invalid. The devices file must
have a 'worker' device map".format(
+ self.cluster_template_d['id']))
+
+ if 'default' not in self.cluster_template_d['devices']:
+ exit("ERROR - '{0}' template is invalid. The devices file must
have a 'default' device map".format(
+ self.cluster_template_d['id']))
+
+ # Validate the selected launch template for each host
+
+ worker_count = 0
+ for hostname in self.node_d:
+ # first service listed denotes the selected template
+ selected_ec2_request = self.node_d[hostname][0]
+ if 'worker' == selected_ec2_request:
+ worker_count = worker_count + 1
+ else:
+ if 'worker' in self.node_d[hostname]:
+ exit("ERROR - '{0}' node config is invalid. The 'worker'
service should be listed first".format(
+ hostname))
+ if selected_ec2_request not in self.cluster_template_d:
+ if len(self.node_d[hostname]) > 1:
+ print('Hint: In template mode, the first service listed
for a host denotes its EC2 template')
+ exit("ERROR - '{0}' node config is invalid. No EC2 template
defined for the '{1}' service".format(
+ hostname, selected_ec2_request))
+
+ if worker_count == 0:
+ exit("ERROR - No worker instances are defined for template
'{0}'".format(self.cluster_template_d['id']))
diff --git a/lib/muchos/config/existing.py b/lib/muchos/config/existing.py
new file mode 100644
index 0000000..e418b06
--- /dev/null
+++ b/lib/muchos/config/existing.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from muchos.config import BaseConfig
+from sys import exit
+from muchos.config.decorators import *
+from muchos.config.validators import *
+from muchos.util import get_ephemeral_devices, get_arch
+import os
+import json
+import glob
+
+
+class ExistingDeployConfig(BaseConfig):
+
+ def __init__(self, deploy_path, config_path, hosts_path, checksums_path,
templates_path, cluster_name):
+ super(ExistingDeployConfig, self).__init__(deploy_path, config_path,
hosts_path, checksums_path, templates_path, cluster_name)
+
+ def verify_config(self, action):
+ self._verify_config(action)
+
+ def verify_launch(self):
+ pass
+
+ def node_type_map(self):
+ node_types = {}
+ return node_types
+
+ def mount_root(self):
+ return self.get('existing', 'mount_root')
+
+ def data_dirs_common(self, nodeType):
+ return self.get('existing', 'data_dirs').split(",")
+
+ def metrics_drive_ids(self):
+ return self.get("existing", "metrics_drive_ids").split(",")
diff --git a/lib/muchos/config/validators.py b/lib/muchos/config/validators.py
new file mode 100644
index 0000000..1e8d49b
--- /dev/null
+++ b/lib/muchos/config/validators.py
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class _validator(object):
+ def __init__(self, f, msg):
+ self.f = f
+ self.msg = msg
+
+ def __call__(self, n):
+ return self.f(n)
+
+ def __str__(self):
+ return self.msg
+
+
+def greater_than(val):
+ return _validator(
+ lambda n: n > val,
+ "must be greater than {}".format(val))
+
+def less_than(val):
+ return _validator(
+ lambda n: n < val,
+ "must be less than {}".format(val))
+
+def equals(val):
+ return _validator(
+ lambda n: n == val,
+ "must equal {}".format(val))
+
+def contains(val):
+ return _validator(
+ lambda n: val in n,
+ "must contain {}".format(val))
+
+def is_in(val):
+ return _validator(
+ lambda n: n in val,
+ "must be in {}".format(val))
+
+def is_type(t):
+ return _validator(
+ lambda n: isinstance(n, t),
+ "must be of type {}".format(t))
\ No newline at end of file
diff --git a/lib/muchos/existing.py b/lib/muchos/existing.py
index f1b3128..523b2a7 100644
--- a/lib/muchos/existing.py
+++ b/lib/muchos/existing.py
@@ -23,9 +23,6 @@ from os.path import isfile, join
from sys import exit
from os import listdir
-from .config import HOST_VAR_DEFAULTS, PLAY_VAR_DEFAULTS, AZURE_VAR_DEFAULTS
-
-
class ExistingCluster:
def __init__(self, config):
@@ -38,36 +35,13 @@ class ExistingCluster:
config = self.config
print('Syncing ansible directory on {0} cluster proxy
node'.format(config.cluster_name))
- host_vars = HOST_VAR_DEFAULTS
- play_vars = PLAY_VAR_DEFAULTS
-
- azure_vars = AZURE_VAR_DEFAULTS
-
- for section in ("general", "ansible-vars", config.get('performance',
'profile'), "azure"):
- for (name, value) in config.items(section):
- if name not in ('proxy_hostname', 'proxy_socks_port'):
- if name in host_vars:
- host_vars[name] = value
- if name in play_vars:
- play_vars[name] = value
- if name in azure_vars:
- azure_vars[name] = value
-
- play_vars['accumulo_sha256'] = config.checksum('accumulo')
- play_vars['fluo_sha256'] = config.checksum('fluo')
- play_vars['fluo_yarn_sha256'] = config.checksum('fluo_yarn')
- play_vars['hadoop_sha256'] = config.checksum('hadoop')
- play_vars['spark_sha256'] = config.checksum('spark')
- play_vars['zookeeper_sha256'] = config.checksum('zookeeper')
- play_vars["shutdown_delay_minutes"] = config.shutdown_delay_minutes()
- play_vars["metrics_drive_ids"] = config.metrics_drive_ids()
- play_vars["mount_root"] = config.mount_root()
- play_vars["node_type_map"] = config.node_type_map()
- play_vars["fstype"] = config.fstype()
- play_vars["force_format"] = config.force_format()
- host_vars['worker_data_dirs'] = str(config.worker_data_dirs())
- host_vars['default_data_dirs'] = str(config.default_data_dirs())
- host_vars['java_product_version'] = str(config.java_product_version())
+ host_vars = config.ansible_host_vars()
+ play_vars = config.ansible_play_vars()
+
+ for k,v in host_vars.items():
+ host_vars[k] = self.config.resolve_value(k, default=v)
+ for k,v in play_vars.items():
+ play_vars[k] = self.config.resolve_value(k, default=v)
with open(join(config.deploy_path, "ansible/site.yml"), 'w') as
site_file:
print("- import_playbook: common.yml", file=site_file)
@@ -148,8 +122,6 @@ class ExistingCluster:
print("\n[all:vars]", file=hosts_file)
for (name, value) in sorted(host_vars.items()):
print("{0} = {1}".format(name, value), file=hosts_file)
- for (name, value) in sorted(azure_vars.items()):
- print("{0} = {1}".format(name, value), file=hosts_file)
with open(join(config.deploy_path, "ansible/group_vars/all"), 'w') as
play_vars_file:
for (name, value) in sorted(play_vars.items()):
diff --git a/lib/tests/azure/__init__.py b/lib/tests/azure/__init__.py
new file mode 100644
index 0000000..555eccf
--- /dev/null
+++ b/lib/tests/azure/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
diff --git a/lib/tests/azure/test_config.py b/lib/tests/azure/test_config.py
new file mode 100644
index 0000000..e9dcfd2
--- /dev/null
+++ b/lib/tests/azure/test_config.py
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from muchos.config import AzureDeployConfig
+
+
+def test_azure_cluster():
+ c = AzureDeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
+ '../conf/checksums', '../conf/templates', 'mycluster')
+
+ # since we are sharing a single muchos.props.example file, we need
+ # to stub the cluster type to be azure (as the file itself has a default
of ec2)
+
+ c.cluster_type = 'azure'
+
+ assert c.checksum_ver('accumulo', '1.9.0') ==
'f68a6145029a9ea843b0305c90a7f5f0334d8a8ceeea94734267ec36421fe7fe'
+ assert c.checksum('accumulo') ==
'df172111698c7a73aa031de09bd5589263a6b824482fbb9b4f0440a16602ed47'
+ assert c.get('azure', 'vm_sku') == 'Standard_D8s_v3'
+ assert c.get('azure', 'managed_disk_type') == 'Standard_LRS'
+ assert c.user_home() == '/home/centos'
+ assert c.mount_root() == '/var/data'
+ assert c.worker_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
+ assert c.default_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
+ assert c.metrics_drive_ids() == ['var-data1', 'var-data2', 'var-data3']
+ assert c.shutdown_delay_minutes() == '0'
+ assert c.mounts(2) == ['/var/data0', '/var/data1']
+ assert c.node_type('worker1') == 'worker'
+ assert c.node_type('leader1') == 'default'
+ assert c.has_option('azure', 'resource_group')
+ assert c.has_option('azure', 'vnet')
+ assert c.has_option('azure', 'vnet_cidr')
+ assert c.has_option('azure', 'subnet')
+ assert c.has_option('azure', 'subnet_cidr')
+ assert c.has_option('azure', 'numnodes')
+ assert c.has_option('azure', 'location')
+ assert len(c.nodes()) == 6
+ assert c.get_node('leader1') == ['namenode', 'resourcemanager',
'accumulomaster', 'zookeeper']
+ assert c.get_node('leader2') == ['metrics']
+ assert c.get_node('worker1') == ['worker', 'swarmmanager']
+ assert c.get_node('worker2') == ['worker']
+ assert c.get_node('worker3') == ['worker']
+ assert c.has_service('accumulomaster')
+ assert not c.has_service('fluo')
+ assert c.get_service_hostnames('worker') == ['worker1', 'worker2',
'worker3', 'worker4']
+ assert c.get_service_hostnames('zookeeper') == ['leader1']
+ assert c.get_hosts() == {'leader2': ('10.0.0.1', None), 'leader1':
('10.0.0.0', '23.0.0.0'),
+ 'worker1': ('10.0.0.2', None), 'worker3':
('10.0.0.4', None),
+ 'worker2': ('10.0.0.3', None), 'worker4':
('10.0.0.5', None)}
+ assert c.get_public_ip('leader1') == '23.0.0.0'
+ assert c.get_private_ip('leader1') == '10.0.0.0'
+ assert c.cluster_name == 'mycluster'
+ assert c.get_cluster_type() == 'azure'
+ assert c.version("accumulo").startswith('2.')
+ assert c.version("fluo").startswith('1.')
+ assert c.version("hadoop").startswith('3.')
+ assert c.version("zookeeper").startswith('3.')
+ assert c.get_service_private_ips("worker") == ['10.0.0.2', '10.0.0.3',
'10.0.0.4', '10.0.0.5']
+ assert c.get('general', 'proxy_hostname') == "leader1"
+ assert c.proxy_public_ip() == "23.0.0.0"
+ assert c.proxy_private_ip() == "10.0.0.0"
+ assert c.get('general', 'cluster_user') == "centos"
+ assert c.get('general', 'cluster_group') == "centos"
+ assert c.get_non_proxy() == [('10.0.0.1', 'leader2'), ('10.0.0.2',
'worker1'), ('10.0.0.3', 'worker2'),
+ ('10.0.0.4', 'worker3'), ('10.0.0.5',
'worker4')]
+ assert c.get_host_services() == [('leader1', 'namenode resourcemanager
accumulomaster zookeeper'),
+ ('leader2', 'metrics'),
+ ('worker1', 'worker swarmmanager'),
+ ('worker2', 'worker'),
+ ('worker3', 'worker'),
+ ('worker4', 'worker')]
diff --git a/lib/tests/ec2/__init__.py b/lib/tests/ec2/__init__.py
new file mode 100644
index 0000000..555eccf
--- /dev/null
+++ b/lib/tests/ec2/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
diff --git a/lib/tests/test_config.py b/lib/tests/ec2/test_config.py
similarity index 53%
rename from lib/tests/test_config.py
rename to lib/tests/ec2/test_config.py
index f6501ff..fe06fe5 100644
--- a/lib/tests/test_config.py
+++ b/lib/tests/ec2/test_config.py
@@ -15,11 +15,11 @@
# limitations under the License.
#
-from muchos.config import DeployConfig
+from muchos.config import Ec2DeployConfig
def test_ec2_cluster():
- c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
+ c = Ec2DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
'../conf/checksums', '../conf/templates', 'mycluster')
assert c.checksum_ver('accumulo', '1.9.0') ==
'f68a6145029a9ea843b0305c90a7f5f0334d8a8ceeea94734267ec36421fe7fe'
assert c.checksum('accumulo') ==
'df172111698c7a73aa031de09bd5589263a6b824482fbb9b4f0440a16602ed47'
@@ -81,91 +81,8 @@ def test_ec2_cluster():
('worker4', 'worker')]
-def test_azure_cluster():
- c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
- '../conf/checksums', '../conf/templates', 'mycluster')
-
- # since we are sharing a single muchos.props.example file, we need
- # to stub the cluster type to be azure (as the file itself has a default
of ec2)
-
- c.cluster_type = 'azure'
-
- assert c.checksum_ver('accumulo', '1.9.0') ==
'f68a6145029a9ea843b0305c90a7f5f0334d8a8ceeea94734267ec36421fe7fe'
- assert c.checksum('accumulo') ==
'df172111698c7a73aa031de09bd5589263a6b824482fbb9b4f0440a16602ed47'
- assert c.get('azure', 'vm_sku') == 'Standard_D8s_v3'
- assert c.get('azure', 'managed_disk_type') == 'Standard_LRS'
- assert c.user_home() == '/home/centos'
- assert c.mount_root() == '/var/data'
- assert c.force_format() == 'no'
- assert c.worker_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
- assert c.default_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
- assert c.metrics_drive_ids() == ['var-data1', 'var-data2', 'var-data3']
- assert c.shutdown_delay_minutes() == '0'
- assert c.mounts(2) == ['/var/data0', '/var/data1']
- assert c.node_type('worker1') == 'worker'
- assert c.node_type('leader1') == 'default'
- assert c.has_option('azure', 'resource_group')
- assert c.has_option('azure', 'vnet')
- assert c.has_option('azure', 'vnet_cidr')
- assert c.has_option('azure', 'subnet')
- assert c.has_option('azure', 'subnet_cidr')
- assert c.has_option('azure', 'numnodes')
- assert c.has_option('azure', 'location')
- assert c.instance_tags() == {}
- assert len(c.nodes()) == 6
- assert c.get_node('leader1') == ['namenode', 'resourcemanager',
'accumulomaster', 'zookeeper']
- assert c.get_node('leader2') == ['metrics']
- assert c.get_node('worker1') == ['worker', 'swarmmanager']
- assert c.get_node('worker2') == ['worker']
- assert c.get_node('worker3') == ['worker']
- assert c.has_service('accumulomaster')
- assert not c.has_service('fluo')
- assert c.get_service_hostnames('worker') == ['worker1', 'worker2',
'worker3', 'worker4']
- assert c.get_service_hostnames('zookeeper') == ['leader1']
- assert c.get_hosts() == {'leader2': ('10.0.0.1', None), 'leader1':
('10.0.0.0', '23.0.0.0'),
- 'worker1': ('10.0.0.2', None), 'worker3':
('10.0.0.4', None),
- 'worker2': ('10.0.0.3', None), 'worker4':
('10.0.0.5', None)}
- assert c.get_public_ip('leader1') == '23.0.0.0'
- assert c.get_private_ip('leader1') == '10.0.0.0'
- assert c.cluster_name == 'mycluster'
- assert c.get_cluster_type() == 'azure'
- assert c.version("accumulo").startswith('2.')
- assert c.version("fluo").startswith('1.')
- assert c.version("hadoop").startswith('3.')
- assert c.version("zookeeper").startswith('3.')
- assert c.get_service_private_ips("worker") == ['10.0.0.2', '10.0.0.3',
'10.0.0.4', '10.0.0.5']
- assert c.get('general', 'proxy_hostname') == "leader1"
- assert c.proxy_public_ip() == "23.0.0.0"
- assert c.proxy_private_ip() == "10.0.0.0"
- assert c.get('general', 'cluster_user') == "centos"
- assert c.get('general', 'cluster_group') == "centos"
- assert c.get_non_proxy() == [('10.0.0.1', 'leader2'), ('10.0.0.2',
'worker1'), ('10.0.0.3', 'worker2'),
- ('10.0.0.4', 'worker3'), ('10.0.0.5',
'worker4')]
- assert c.get_host_services() == [('leader1', 'namenode resourcemanager
accumulomaster zookeeper'),
- ('leader2', 'metrics'),
- ('worker1', 'worker swarmmanager'),
- ('worker2', 'worker'),
- ('worker3', 'worker'),
- ('worker4', 'worker')]
-
-
-def test_existing_cluster():
- c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
- '../conf/checksums', '../conf/templates', 'mycluster')
- c.cluster_type = 'existing'
- assert c.get_cluster_type() == 'existing'
- assert c.node_type_map() == {}
- assert c.mount_root() == '/var/data'
- assert c.fstype() is None
- assert c.force_format() == 'no'
- assert c.worker_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
- assert c.default_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
- assert c.metrics_drive_ids() == ['var-data1', 'var-data2', 'var-data3']
- assert c.shutdown_delay_minutes() == '0'
-
-
def test_case_sensitive():
- c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
+ c = Ec2DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
'../conf/checksums', '../conf/templates', 'mycluster')
assert c.has_option('ec2', 'default_instance_type') == True
assert c.has_option('ec2', 'Default_instance_type') == False
@@ -175,7 +92,7 @@ def test_case_sensitive():
def test_ec2_cluster_template():
- c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
+ c = Ec2DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
'../conf/checksums', '../conf/templates', 'mycluster')
c.set('ec2', 'cluster_template', 'example')
diff --git a/lib/tests/existing/__init__.py b/lib/tests/existing/__init__.py
new file mode 100644
index 0000000..555eccf
--- /dev/null
+++ b/lib/tests/existing/__init__.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
diff --git a/lib/tests/existing/test_config.py
b/lib/tests/existing/test_config.py
new file mode 100644
index 0000000..ef9a29a
--- /dev/null
+++ b/lib/tests/existing/test_config.py
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from muchos.config import ExistingDeployConfig
+
+
+def test_existing_cluster():
+ c = ExistingDeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
+ '../conf/checksums', '../conf/templates', 'mycluster')
+ c.cluster_type = 'existing'
+ assert c.get_cluster_type() == 'existing'
+ assert c.node_type_map() == {}
+ assert c.mount_root() == '/var/data'
+ assert c.worker_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
+ assert c.default_data_dirs() == ['/var/data1', '/var/data2', '/var/data3']
+ assert c.metrics_drive_ids() == ['var-data1', 'var-data2', 'var-data3']
+ assert c.shutdown_delay_minutes() == '0'
\ No newline at end of file
