This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new b69220d Several updates to Muchos (#223)
b69220d is described below
commit b69220d35e3d11b204124511c025a5fc6a8ddfec
Author: Mike Walch <[email protected]>
AuthorDate: Fri Jul 13 12:21:27 2018 -0400
Several updates to Muchos (#223)
* AMI is only specified in muchos.props
* Added support for m5d instance types
* Made m5d.large the the default type
---
conf/muchos.props.example | 10 ++++++----
lib/muchos/config.py | 11 +++++------
lib/muchos/main.py | 15 ++++++---------
lib/muchos/util.py | 44 ++++++++++++++------------------------------
lib/tests/test_config.py | 10 +++++-----
lib/tests/test_util.py | 4 +---
6 files changed, 37 insertions(+), 57 deletions(-)
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index c23439a..b729350 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -49,8 +49,13 @@ fluo_yarn_sha256 =
c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff87
accumulo_sha256 =
f9cebff3ff85cacb8c80263725663b047ef239916cb4490c93c62509d62e1e76
[ec2]
+# AWS machine image to use. The default below is for a CentOS 7 image (in
us-east-1).
+# You may need to change this value if a new image has been released or you
are running in a different region.
+# Before using this AMI, subscribe to it on the CentOS product page below or
launching will fail:
+# https://aws.amazon.com/marketplace/pp/B00O7WM7QW
+aws_ami = ami-9887c6e7
# Type of AWS instance launched by default
-default_instance_type = m3.large
+default_instance_type = m5d.large
# Type of AWS instance launched for any node running 'worker' service
# Leave default below to use same instance type set by 'default_instance_type'
property
worker_instance_type = %(default_instance_type)s
@@ -71,9 +76,6 @@ fstype = ext3
force_format = no
# Tags to add instances
#instance_tags = key1:value1,key2:value2
-# Overrides the default CentOS 7 AWS AMIs used by Muchos
-# Only set if you want to use a custom AMI (which should be derived from a
CentOS 7 AMI)
-#aws_ami = ami-xxxxxxxx
# Nodes will be given public IP addresses if true
associate_public_ip = true
# Path to file containing user data that will be executed at launch
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index 96b5b6f..7196aa0 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -14,7 +14,7 @@
from ConfigParser import ConfigParser
from sys import exit
-from util import get_ephemeral_devices, get_arch, get_ami
+from util import get_ephemeral_devices, get_arch
import os
SERVICES = ['zookeeper', 'namenode', 'resourcemanager', 'accumulomaster',
'mesosmaster', 'worker', 'fluo', 'fluo_yarn', 'metrics']
@@ -52,9 +52,9 @@ class DeployConfig(ConfigParser):
if not self.has_service(service):
exit("ERROR - Missing '{0}' service from [nodes]
section of muchos.props".format(service))
- def verify_launch(self, region):
- self.get_image_id(self.get('ec2', 'default_instance_type'), region)
- self.get_image_id(self.get('ec2', 'worker_instance_type'), region)
+ def verify_launch(self):
+ self.verify_instance_type(self.get('ec2', 'default_instance_type'))
+ self.verify_instance_type(self.get('ec2', 'worker_instance_type'))
def init_nodes(self):
self.node_d = {}
@@ -121,11 +121,10 @@ class DeployConfig(ConfigParser):
def sha256(self, software_id):
return self.get('general', software_id + '_sha256')
- def get_image_id(self, instance_type, region):
+ def verify_instance_type(self, instance_type):
if get_arch(instance_type) == 'pvm':
exit("ERROR - Configuration contains instance type '{0}' that uses
pvm architecture."
"Only hvm architecture is supported!".format(instance_type))
- return get_ami(region)
def instance_tags(self):
retd = {}
diff --git a/lib/muchos/main.py b/lib/muchos/main.py
index 97d205a..26c7025 100644
--- a/lib/muchos/main.py
+++ b/lib/muchos/main.py
@@ -56,15 +56,13 @@ class MuchosCluster:
request['InstanceType'] = instance_type
request['InstanceInitiatedShutdownBehavior'] = self.config.get('ec2',
'shutdown_behavior')
- if self.config.has_option('ec2', 'aws_ami'):
- image_id = self.config.get('ec2', 'aws_ami')
- else:
- session = boto3.session.Session()
- image_id = self.config.get_image_id(instance_type,
session.region_name)
+ if not self.config.has_option('ec2', 'aws_ami'):
+ exit('aws_ami property must be set!')
+ image_id = self.config.get('ec2', 'aws_ami')
if not image_id:
- exit('ERROR - Image not found for instance type: '+instance_type)
- request['ImageId'] = image_id
+ exit('aws_ami property was not properly')
+ request['ImageId'] = image_id
request['BlockDeviceMappings'] = get_block_device_map(instance_type)
if self.config.has_option('ec2', 'key_name'):
@@ -155,8 +153,7 @@ class MuchosCluster:
exit("ERROR - A hosts file already exists at {0}. Please delete
before running launch again"
.format(self.config.hosts_path))
- session = boto3.session.Session()
- self.config.verify_launch(session.region_name)
+ self.config.verify_launch()
print "Launching {0} cluster".format(self.config.cluster_name)
diff --git a/lib/muchos/util.py b/lib/muchos/util.py
index 86811cb..1ddea45 100644
--- a/lib/muchos/util.py
+++ b/lib/muchos/util.py
@@ -33,15 +33,10 @@ this could be due to CentOS releasing new images of CentOS
7. When this occurs,
are no longer available to new users. If you think this is the case, go to
the CentOS 7 product
page on AWS Marketplace at the URL below to find the latest AMI:
-https://aws.amazon.com/marketplace/ordering?productId=b7ee8a69-ee97-4a49-9e68-afaee216db2e
+ https://aws.amazon.com/marketplace/pp/B00O7WM7QW
-On the product page, click 'Manual Launch' to find the latest AMI ID for your
EC2 region.
-This should be used to set the 'aws_ami' property in your muchos.props which
will override
-the default AMI IDs used by Muchos. After setting the 'aws_ami' property, run
the launch
-command again.
-
-Also, let us know that this has occured by creating an issue on the Muchos's
GitHub page
-and we'll upgrade the defaults AMIs used by Muchos to be the latest CentOS
images.
+On the product page, find the latest AMI ID for your EC2 region. This should
be used to set the 'aws_ami'
+property in your muchos.props. After setting the 'aws_ami' property, run the
launch command again.
"""
instance_types = {
@@ -76,6 +71,12 @@ instance_types = {
"m3.large": EC2Type("hvm"),
"m3.medium": EC2Type("hvm"),
"m3.xlarge": EC2Type("hvm", 2),
+ "m5d.large": EC2Type("hvm", 1, True),
+ "m5d.xlarge": EC2Type("hvm", 1, True),
+ "m5d.2xlarge": EC2Type("hvm", 1, True),
+ "m5d.4xlarge": EC2Type("hvm", 2, True),
+ "m5d.12xlarge": EC2Type("hvm", 2, True),
+ "m5d.24xlarge": EC2Type("hvm", 4, True),
"r3.2xlarge": EC2Type("hvm", 1),
"r3.4xlarge": EC2Type("hvm", 1),
"r3.8xlarge": EC2Type("hvm", 2),
@@ -87,24 +88,6 @@ instance_types = {
"d2.8xlarge": EC2Type("hvm", 24)
}
-# AMI given region for HVM arch. PVM arch is not supported.
-ami_lookup = {
- "us-east-1": "ami-4bf3d731",
- "us-east-2": "ami-e1496384",
- "us-west-1": "ami-65e0e305",
- "us-west-2": "ami-a042f4d8",
- "ca-central-1": "ami-dcad28b8",
- "eu-central-1": "ami-337be65c",
- "eu-west-1": "ami-6e28b517",
- "eu-west-2": "ami-ee6a718a",
- "eu-west-3": "ami-bfff49c2",
- "ap-northeast-1": "ami-25bd2743",
- "ap-northeast-2": "ami-7248e81c",
- "ap-southeast-1": "ami-d2fa88ae",
- "ap-southeast-2": "ami-b6bb47d4",
- "ap-south-1": "ami-5d99ce32",
- "sa-east-1": "ami-f9adef95"
-}
def verify_type(instance_type):
if instance_type not in instance_types:
@@ -125,7 +108,11 @@ def get_ephemeral_devices(instance_type):
devices = []
ec2_type = instance_types.get(instance_type)
- for i in range(0, ec2_type.ephemeral):
+ start = 0
+ if instance_type.startswith("m5d"):
+ start = 1
+
+ for i in range(start, ec2_type.ephemeral + start):
if ec2_type.has_nvme:
devices.append('/dev/nvme' + str(i) + 'n1')
else:
@@ -148,9 +135,6 @@ def get_block_device_map(instance_type):
return bdm
-def get_ami(region):
- return ami_lookup.get(region)
-
def parse_args(hosts_dir, input_args=None):
parser = OptionParser(
usage="muchos [options] <action>\n\n"
diff --git a/lib/tests/test_config.py b/lib/tests/test_config.py
index 717048e..6ef2f32 100644
--- a/lib/tests/test_config.py
+++ b/lib/tests/test_config.py
@@ -18,12 +18,13 @@ from muchos.config import DeployConfig
def test_defaults():
c = DeployConfig("muchos", '../conf/muchos.props.example',
'../conf/hosts/example/example_cluster',
'mycluster')
- assert c.get('ec2', 'default_instance_type') == 'm3.large'
- assert c.get('ec2', 'worker_instance_type') == 'm3.large'
+ assert c.get('ec2', 'default_instance_type') == 'm5d.large'
+ assert c.get('ec2', 'worker_instance_type') == 'm5d.large'
+ assert c.get('ec2', 'aws_ami') == 'ami-9887c6e7'
assert c.max_ephemeral() == 1
assert c.mounts(2) == ['/media/ephemeral0', '/media/ephemeral1']
- assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0',
], 'devices': ['/dev/xvdb', ]},
- 'worker': {'mounts': ['/media/ephemeral0', ],
'devices': ['/dev/xvdb', ]}}
+ assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0',
], 'devices': ['/dev/nvme1n1', ]},
+ 'worker': {'mounts': ['/media/ephemeral0', ],
'devices': ['/dev/nvme1n1', ]}}
assert c.node_type('worker1') == 'worker'
assert c.node_type('leader1') == 'default'
assert not c.has_option('ec2', 'vpc_id')
@@ -59,7 +60,6 @@ def test_defaults():
assert c.get_host_services() == [('leader1', 'namenode zookeeper fluo
fluo_yarn'), ('leader2', 'resourcemanager zookeeper'),
('leader3', 'accumulomaster zookeeper'),
('metrics', 'metrics'),
('worker1', 'worker'), ('worker2',
'worker'), ('worker3', 'worker')]
- assert c.get_image_id('m3.large', 'us-east-1') == 'ami-4bf3d731'
def test_case_sensitive():
diff --git a/lib/tests/test_util.py b/lib/tests/test_util.py
index 6c6015d..9fc3c7c 100644
--- a/lib/tests/test_util.py
+++ b/lib/tests/test_util.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from muchos.util import get_arch, parse_args, get_ami, get_ephemeral_devices
+from muchos.util import get_arch, parse_args, get_ephemeral_devices
def test_util():
@@ -25,8 +25,6 @@ def test_util():
assert get_arch('m1.large') == 'pvm'
assert get_arch('m3.large') == 'hvm'
- assert get_ami('us-east-1') == 'ami-4bf3d731'
-
hosts_dir = '../conf/hosts'
assert parse_args(hosts_dir, ['launch']) is None
assert parse_args(hosts_dir, ['launch', 'mycluster']) is None
