This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2409 by this push:
new a0af74d [DATALAB-2409] - [AWS] SSN configure refactored
a0af74d is described below
commit a0af74d498df68e54f8b10de61c480e6d43c0b88
Author: bodnarmykola <[email protected]>
AuthorDate: Wed Jul 28 16:48:54 2021 +0300
[DATALAB-2409] - [AWS] SSN configure refactored
---
.../src/general/conf/datalab.ini | 13 +
.../src/general/lib/aws/meta_lib.py | 58 ++--
.../src/general/scripts/aws/ssn_configure.py | 294 +++++++--------------
.../src/general/scripts/aws/ssn_prepare.py | 44 +--
4 files changed, 153 insertions(+), 256 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini
b/infrastructure-provisioning/src/general/conf/datalab.ini
index 4881a03..7a21340 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -94,6 +94,19 @@ private_subnet_prefix = 24
### Deeplearning native cloud AMI enabled
deeplearning_cloud_ami = true
+[packages]
+
+[pip_packages]
+boto3 = 1.18.8
+bcrypt = 3.2.0
+backoff = 1.11.1
+argparse = 1.4.0
+fabric = 2.6.0
+awscli = 1.20.8
+pymongo = 3.12.0
+pyyaml = 5.4.1
+jinja2 = 3.0.1
+
#--- [aws] section contains all common parameters related to Amazon ---#
[aws]
### Amazon iam user access_key
diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
index 33d95ec..b3ea91c 100644
--- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
@@ -31,30 +31,38 @@ import subprocess
from botocore.client import Config as botoConfig
from datalab.fab import *
-
-def get_instance_hostname(tag_name, instance_name):
+def get_instance_metadata(tag_name, instance_name, metadata):
try:
- public = ''
- private = ''
ec2 = boto3.resource('ec2')
instances = ec2.instances.filter(
Filters=[{'Name': 'tag:{}'.format(tag_name), 'Values':
[instance_name]},
{'Name': 'instance-state-name', 'Values': ['running']}])
for instance in instances:
- public = getattr(instance, 'public_dns_name')
- private = getattr(instance, 'private_dns_name')
- if public:
- return public
- else:
- return private
- if public == '' and private == '':
- raise Exception("Unable to find instance hostname with instance
name: " + instance_name)
- except Exception as err:
- logging.error("Error with finding instance hostname with instance
name: " + instance_name + " : " + str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout))
- append_result(str({"error": "Error with finding instance hostname",
"error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
+ if metadata == 'hostname':
+ public = getattr(instance, 'public_dns_name')
+ private = getattr(instance, 'private_dns_name')
+ if public:
+ return public
+ else:
+ return private
+ if public == '' and private == '':
+ raise Exception("Unable to find instance hostname with
instance name: " + instance_name)
+ elif metadata == 'ip_address':
+ public = getattr(instance, 'public_ip_address')
+ private = getattr(instance, 'private_ip_address')
+ ips = {'Public': public, 'Private': private}
+ if ips == {}:
+ raise Exception("Unable to find instance IP addresses with
instance name: " + instance_name)
+ return ips
+ except Exception as err:
+ logging.error("Error with finding instance metadata with instance
name: "
+ + instance_name + " : " + str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout))
+ append_result(str({"error": "Error with finding instance metadata",
+ "error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+
def get_vpc_endpoints(vpc_id):
try:
# Returns LIST of Endpoint DICTIONARIES
@@ -107,26 +115,6 @@ def get_bucket_by_name(bucket_name):
traceback.print_exc(file=sys.stdout)
-def get_instance_ip_address(tag_name, instance_name):
- try:
- ec2 = boto3.resource('ec2')
- instances = ec2.instances.filter(
- Filters=[{'Name': 'tag:{}'.format(tag_name), 'Values':
[instance_name]},
- {'Name': 'instance-state-name', 'Values': ['running']}])
- ips = {}
- for instance in instances:
- public = getattr(instance, 'public_ip_address')
- private = getattr(instance, 'private_ip_address')
- ips = {'Public': public, 'Private': private}
- if ips == {}:
- raise Exception("Unable to find instance IP addresses with
instance name: " + instance_name)
- return ips
- except Exception as err:
- logging.error("Error with getting ip address by name: " + str(err) +
"\n Traceback: " + traceback.print_exc(file=sys.stdout))
- append_result(str({"error": "Error with getting ip address by name",
"error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
- traceback.print_exc(file=sys.stdout)
-
-
def get_instance_ip_address_by_id(instance_id):
try:
ec2 = boto3.resource('ec2')
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index eceb451..b683ff6 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@ -33,57 +33,28 @@ import traceback
import subprocess
from fabric import *
+def cleanup_aws_cloud_resources(tag_name, service_base_name):
+ try:
+ params = "--tag_name {} --service_base_name {}".format(tag_name,
service_base_name)
+ subprocess.run("~/scripts/{}.py
{}".format('ssn_terminate_aws_resources', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+
if __name__ == "__main__":
+ #configuring logs
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'],
os.environ['request_id'])
local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" +
local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
- filename=local_log_filepath)
-
- ssn_conf = dict()
- ssn_conf['instance'] = 'ssn'
-
- def clear_resources():
- if ssn_conf['domain_created']:
-
datalab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'],
-
os.environ['ssn_hosted_zone_name'],
-
os.environ['ssn_subdomain'])
- datalab.actions_lib.remove_ec2(ssn_conf['tag_name'],
ssn_conf['instance_name'])
- datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
- datalab.actions_lib.remove_s3(ssn_conf['instance'])
- if ssn_conf['pre_defined_sg']:
- datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
- if ssn_conf['pre_defined_subnet']:
-
datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'],
ssn_conf['tag_name'],
-
ssn_conf['service_base_name'])
- datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
- if ssn_conf['pre_defined_vpc']:
- datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
- datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
- if ssn_conf['pre_defined_vpc2']:
- datalab.actions_lib.remove_peering('*')
- try:
-
datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
- except:
- print("There are no VPC Endpoints")
- datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'],
True)
- datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ handlers=[logging.StreamHandler(),
logging.FileHandler(local_log_filepath)])
+ # deriving variables for ssn node deployment
try:
logging.info('[DERIVING NAMES]')
- print('[DERIVING NAMES]')
+ ssn_conf = dict()
ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] =
datalab.fab.replace_multi_symbols(
- os.environ['conf_service_base_name'][:20], '-', True)
- if 'ssn_hosted_zone_id' in os.environ and 'ssn_hosted_zone_name' in
os.environ and \
- 'ssn_subdomain' in os.environ:
- ssn_conf['domain_created'] = True
- else:
- ssn_conf['domain_created'] = False
- ssn_conf['pre_defined_vpc'] = False
- ssn_conf['pre_defined_subnet'] = False
- ssn_conf['pre_defined_sg'] = False
- ssn_conf['billing_enabled'] = True
+ os.environ['conf_service_base_name'][:20], '-', True)
ssn_conf['role_name'] =
'{}-ssn-role'.format(ssn_conf['service_base_name'])
ssn_conf['role_profile_name'] =
'{}-ssn-profile'.format(ssn_conf['service_base_name'])
ssn_conf['policy_name'] =
'{}-ssn-policy'.format(ssn_conf['service_base_name'])
@@ -97,151 +68,94 @@ if __name__ == "__main__":
ssn_conf['sg_name'] = '{}-ssn-sg'.format(ssn_conf['service_base_name'])
ssn_conf['network_type'] = os.environ['conf_network_type']
ssn_conf['datalab_ssh_user'] = os.environ['conf_os_user']
+ ssn_conf['ssn_datalab_path'] = os.environ['ssn_datalab_path']
+ ssn_conf['conf_tag_resource_id'] = os.environ['conf_tag_resource_id']
+ ssn_conf['instance_hostname'] = (lambda x:
datalab.meta_lib.get_instance_metadata(
+ ssn_conf['tag_name'], ssn_conf['instance_name'], 'ip_address').get(
+ 'Private') if x == 'private' else
datalab.meta_lib.get_instance_metadata(
+ ssn_conf['tag_name'], ssn_conf['instance_name'],
'hostname'))(ssn_conf['network_type'])
+ ssn_conf['initial_user'] = (lambda x: 'ubuntu' if x == 'debian' else
'ec2-user')(os.environ['conf_os_family'])
+ ssn_conf['sudo_group'] = (lambda x: 'sudo' if x == 'debian' else
'wheel')(os.environ['conf_os_family'])
+ ssn_conf['step_cert_sans'] = (lambda x: (lambda x: ' --san {0} --san
{1}'.format(
+ datalab.meta_lib.get_instance_metadata(ssn_conf['tag_name'],
ssn_conf['instance_name'], 'hostname'),
+ datalab.meta_lib.get_instance_metadata(ssn_conf['tag_name'],
ssn_conf['instance_name'], 'ip_address').get(
+ 'Public')) if x == 'public' else ' --san
{0}'.format(datalab.meta_lib.get_instance_metadata(
+ ssn_conf['tag_name'], ssn_conf['instance_name'],
'ip_address').get('Private')))(
+ ssn_conf['network_type']) if x == 'true' else
'')(os.environ['conf_stepcerts_enabled'])
- try:
- if os.environ['aws_vpc_id'] == '':
- raise KeyError
- except KeyError:
- ssn_conf['tag'] = {"Key": ssn_conf['tag_name'], "Value":
"{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_vpc_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
-
ssn_conf['service_base_name'])
- ssn_conf['pre_defined_vpc'] = True
- try:
- if os.environ['aws_subnet_id'] == '':
- raise KeyError
- except KeyError:
- ssn_conf['tag'] = {"Key": ssn_conf['tag_name'], "Value":
"{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_subnet_id'] =
datalab.meta_lib.get_subnet_by_tag(ssn_conf['tag'], True)
- ssn_conf['pre_defined_subnet'] = True
- try:
- if os.environ['conf_duo_vpc_enable'] == 'true' and not
os.environ['aws_vpc2_id']:
- raise KeyError
- except KeyError:
- ssn_conf['tag'] = {"Key": ssn_conf['tag2_name'], "Value":
"{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_vpc2_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
-
ssn_conf['service_base_name'])
- ssn_conf['pre_defined_vpc2'] = True
- try:
- if os.environ['conf_duo_vpc_enable'] == 'true' and not
os.environ['aws_peering_id']:
- raise KeyError
- except KeyError:
- os.environ['aws_peering_id'] =
datalab.meta_lib.get_peering_by_tag(ssn_conf['tag_name'],
-
ssn_conf['service_base_name'])
- ssn_conf['pre_defined_peering'] = True
- try:
- if os.environ['aws_security_groups_ids'] == '':
- raise KeyError
- except KeyError:
- os.environ['aws_security_groups_ids'] =
datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
- ssn_conf['pre_defined_sg'] = True
- try:
- if os.environ['aws_account_id'] == '':
- raise KeyError
- if os.environ['aws_billing_bucket'] == '':
- raise KeyError
- except KeyError:
- ssn_conf['billing_enabled'] = False
- if not ssn_conf['billing_enabled']:
- os.environ['aws_account_id'] = 'None'
- os.environ['aws_billing_bucket'] = 'None'
- try:
- if not os.environ['aws_report_path']:
- raise KeyError
- except KeyError:
- os.environ['aws_report_path'] = ''
except Exception as err:
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to generate variables dictionary.",
str(err))
- clear_resources()
+ traceback.print_exc()
sys.exit(1)
+ #creating datalab ssh user
try:
- if os.environ['conf_os_family'] == 'debian':
- ssn_conf['initial_user'] = 'ubuntu'
- ssn_conf['sudo_group'] = 'sudo'
- if os.environ['conf_os_family'] == 'redhat':
- ssn_conf['initial_user'] = 'ec2-user'
- ssn_conf['sudo_group'] = 'wheel'
-
- if ssn_conf['network_type'] == 'private':
- ssn_conf['instance_hostname'] =
datalab.meta_lib.get_instance_ip_address(
- ssn_conf['tag_name'], ssn_conf['instance_name']).get('Private')
- else:
- ssn_conf['instance_hostname'] =
datalab.meta_lib.get_instance_hostname(
- ssn_conf['tag_name'], ssn_conf['instance_name'])
-
- if os.environ['conf_stepcerts_enabled'] == 'true':
- ssn_conf['step_cert_sans'] = ' --san {0}
'.format(datalab.meta_lib.get_instance_ip_address(
- ssn_conf['tag_name'],
ssn_conf['instance_name']).get('Private'))
- if ssn_conf['network_type'] == 'public':
- ssn_conf['step_cert_sans'] += ' --san {0} --san {1}'.format(
-
datalab.meta_lib.get_instance_hostname(ssn_conf['tag_name'],
ssn_conf['instance_name']),
-
datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
-
ssn_conf['instance_name']).get('Public'))
- else:
- ssn_conf['step_cert_sans'] = ''
-
logging.info('[CREATING DATALAB SSH USER]')
- print('[CREATING DATALAB SSH USER]')
params = "--hostname {} --keyfile {} --initial_user {} --os_user {}
--sudo_group {}".format(
ssn_conf['instance_hostname'], os.environ['conf_key_dir'] +
os.environ['conf_key_name'] + ".pem",
ssn_conf['initial_user'], ssn_conf['datalab_ssh_user'],
ssn_conf['sudo_group'])
-
try:
subprocess.run("~/scripts/{}.py {}".format('create_ssh_user',
params), shell=True, check=True)
except:
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.append_result("Failed creating ssh user 'datalab'.",
str(err))
- clear_resources()
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed creating ssh user", str(err))
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
+ #installing prerequisites to ssn instance
try:
logging.info('[INSTALLING PREREQUISITES TO SSN INSTANCE]')
- print('[INSTALLING PREREQUISITES TO SSN INSTANCE]')
- params = "--hostname {} --keyfile {} --pip_packages 'boto3
bcrypt==3.1.7 backoff argparse fabric==1.14.0 awscli pymongo " \
- "pyyaml jinja2' --user {} --region {}". \
- format(ssn_conf['instance_hostname'], os.environ['conf_key_dir'] +
os.environ['conf_key_name'] + ".pem",
- ssn_conf['datalab_ssh_user'], os.environ['aws_region'])
-
+ pip_packages = 'boto3=={} bcrypt=={} backoff=={} argparse=={}
fabric=={} awscli=={} pymongo=={} pyyaml=={}' \
+ ' jinja2=={}'.format(os.environ['pip_packages_boto3'],
os.environ['pip_packages_bcrypt'],
+
os.environ['pip_packages_backoff'], os.environ['pip_packages_argparse'],
+ os.environ['pip_packages_fabric'],
os.environ['pip_packages_awscli'],
+
os.environ['pip_packages_pymongo'], os.environ['pip_packages_pyyaml'],
+ os.environ['pip_packages_jinja2'])
+ params = "--hostname {} --keyfile {} --pip_packages {} --user {}
--region {}".format(
+ ssn_conf['instance_hostname'],
"{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
+ pip_packages, ssn_conf['datalab_ssh_user'], ssn_conf['region'])
try:
subprocess.run("~/scripts/{}.py
{}".format('install_prerequisites', params), shell=True, check=True)
except:
traceback.print_exc()
raise Exception
except Exception as err:
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed installing software: pip,
packages.", str(err))
- clear_resources()
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
+ #configuring ssn instance
try:
logging.info('[CONFIGURE SSN INSTANCE]')
- print('[CONFIGURE SSN INSTANCE]')
- additional_config = {"nginx_template_dir": "/root/templates/",
"service_base_name":
- ssn_conf['service_base_name'],
+ additional_config = {"nginx_template_dir": "/root/templates/",
+ "service_base_name":
ssn_conf['service_base_name'],
"security_group_id":
os.environ['aws_security_groups_ids'],
"vpc_id": os.environ['aws_vpc_id'], "subnet_id":
os.environ['aws_subnet_id'],
"admin_key": os.environ['conf_key_name']}
- params = "--hostname {} --keyfile {} --additional_config '{}'
--os_user {} --datalab_path {} " \
- "--tag_resource_id {} --step_cert_sans '{}' ".format(
- ssn_conf['instance_hostname'],
- "{}{}.pem".format(os.environ['conf_key_dir'],
os.environ['conf_key_name']),
- json.dumps(additional_config), ssn_conf['datalab_ssh_user'],
os.environ['ssn_datalab_path'],
- os.environ['conf_tag_resource_id'], ssn_conf['step_cert_sans'])
-
+ params = "--hostname {} --keyfile {} --additional_config '{}'
--os_user {} --datalab_path {}" \
+ " --tag_resource_id {} --step_cert_sans '{}' ".format(
+ ssn_conf['instance_hostname'],
"{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
+ json.dumps(additional_config), ssn_conf['datalab_ssh_user'],
ssn_conf['ssn_datalab_path'],
+ ssn_conf['conf_tag_resource_id'], ssn_conf['step_cert_sans'])
try:
subprocess.run("~/scripts/{}.py {}".format('configure_ssn_node',
params), shell=True, check=True)
except:
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.append_result("Failed configuring ssn.", str(err))
- clear_resources()
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to configure SSN node.", str(err))
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
+ #configuring docker at ssn instance
try:
logging.info('[CONFIGURING DOCKER AT SSN INSTANCE]')
- print('[CONFIGURING DOCKER AT SSN INSTANCE]')
additional_config = [{"name": "base", "tag": "latest"},
{"name": "edge", "tag": "latest"},
{"name": "project", "tag": "latest"},
@@ -268,11 +182,23 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.append_result("Unable to configure docker.", str(err))
- clear_resources()
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to configure docker.", str(err))
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
+ #configuring keycloak client for ui
try:
+ logging.info('[CONFIGURE KEYCLOAK CLIENT FOR DATALAB UI]')
+ except Exception as err:
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to configure Keycloak client for
DataLab UI.", str(err))
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
+ sys.exit(1)
+
+ #configuring UI
+ try:
+ logging.info('[CONFIGURE SSN INSTANCE UI]')
cloud_params = [
{
'key': 'KEYCLOAK_REDIRECT_URI',
@@ -348,26 +274,6 @@ if __name__ == "__main__":
'value': os.environ['conf_key_dir']
},
{
- 'key': 'LDAP_HOST',
- 'value': os.environ['ldap_hostname']
- },
- {
- 'key': 'LDAP_DN',
- 'value': os.environ['ldap_dn']
- },
- {
- 'key': 'LDAP_OU',
- 'value': os.environ['ldap_ou']
- },
- {
- 'key': 'LDAP_USER_NAME',
- 'value': os.environ['ldap_service_username']
- },
- {
- 'key': 'LDAP_USER_PASSWORD',
- 'value': os.environ['ldap_service_password']
- },
- {
'key': 'AZURE_RESOURCE_GROUP_NAME',
'value': ''
},
@@ -430,7 +336,7 @@ if __name__ == "__main__":
cloud_params.append(
{
'key': 'PEERING_ID',
- 'value': ''
+ 'value': os.environ['aws_peering_id']
})
if os.environ['conf_stepcerts_enabled'] == 'true':
cloud_params.append(
@@ -621,41 +527,31 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.append_result("Unable to configure UI.", str(err))
- clear_resources()
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to configure Datalab UI.", str(err))
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
sys.exit(1)
+ #ssn deployment summary
try:
logging.info('[SUMMARY]')
- print('[SUMMARY]')
- print("Service base name: {}".format(ssn_conf['service_base_name']))
- print("SSN Name: {}".format(ssn_conf['instance_name']))
- print("SSN Hostname: {}".format(ssn_conf['instance_hostname']))
- print("Role name: {}".format(ssn_conf['role_name']))
- print("Role profile name: {}".format(ssn_conf['role_profile_name']))
- print("Policy name: {}".format(ssn_conf['policy_name']))
- print("Key name: {}".format(os.environ['conf_key_name']))
- print("VPC ID: {}".format(os.environ['aws_vpc_id']))
- print("Subnet ID: {}".format(os.environ['aws_subnet_id']))
- print("Security IDs: {}".format(os.environ['aws_security_groups_ids']))
- print("SSN instance shape:
{}".format(os.environ['aws_ssn_instance_size']))
- print("SSN AMI name: {}".format(ssn_conf['ssn_image_name']))
- print("Region: {}".format(ssn_conf['region']))
- ssn_conf['jenkins_url'] =
"http://{}/jenkins".format(datalab.meta_lib.get_instance_hostname(
- ssn_conf['tag_name'], ssn_conf['instance_name']))
- ssn_conf['jenkins_url_https'] =
"https://{}/jenkins".format(datalab.meta_lib.get_instance_hostname(
- ssn_conf['tag_name'], ssn_conf['instance_name']))
- print("Jenkins URL: {}".format(ssn_conf['jenkins_url']))
- print("Jenkins URL HTTPS: {}".format(ssn_conf['jenkins_url_https']))
- print("DataLab UI HTTP URL:
http://{}".format(datalab.meta_lib.get_instance_hostname(
+ logging.info("Service base name:
{}".format(ssn_conf['service_base_name']))
+ logging.info("SSN Name: {}".format(ssn_conf['instance_name']))
+ logging.info("SSN Hostname: {}".format(ssn_conf['instance_hostname']))
+ logging.info("Role name: {}".format(ssn_conf['role_name']))
+ logging.info("Role profile name:
{}".format(ssn_conf['role_profile_name']))
+ logging.info("Policy name: {}".format(ssn_conf['policy_name']))
+ logging.info("Key name: {}".format(os.environ['conf_key_name']))
+ logging.info("VPC ID: {}".format(os.environ['aws_vpc_id']))
+ logging.info("Subnet ID: {}".format(os.environ['aws_subnet_id']))
+ logging.info("Security IDs:
{}".format(os.environ['aws_security_groups_ids']))
+ logging.info("SSN instance shape:
{}".format(os.environ['aws_ssn_instance_size']))
+ logging.info("SSN AMI name: {}".format(ssn_conf['ssn_image_name']))
+ logging.info("Region: {}".format(ssn_conf['region']))
+ logging.info("DataLab UI HTTP URL:
http://{}".format(datalab.meta_lib.get_instance_hostname(
ssn_conf['tag_name'], ssn_conf['instance_name'])))
- print("DataLab UI HTTPS URL:
https://{}".format(datalab.meta_lib.get_instance_hostname(
+ logging.info("DataLab UI HTTPS URL:
https://{}".format(datalab.meta_lib.get_instance_hostname(
ssn_conf['tag_name'], ssn_conf['instance_name'])))
- try:
- with open('jenkins_creds.txt') as f:
- print(f.read())
- except:
- print("Jenkins is either configured already or have issues in
configuration routine.")
with open("/root/result.json", 'w') as f:
res = {"service_base_name": ssn_conf['service_base_name'],
@@ -674,19 +570,19 @@ if __name__ == "__main__":
"action": "Create SSN instance"}
f.write(json.dumps(res))
- print('Upload response file')
+ logging.info('Upload response file')
params = "--instance_name {} --local_log_filepath {} --os_user {}
--instance_hostname {}". \
format(ssn_conf['instance_name'], local_log_filepath,
ssn_conf['datalab_ssh_user'],
ssn_conf['instance_hostname'])
subprocess.run("~/scripts/{}.py {}".format('upload_response_file',
params), shell=True, check=True)
logging.info('[FINALIZE]')
- print('[FINALIZE]')
params = ""
if os.environ['conf_lifecycle_stage'] == 'prod':
params += "--key_id {}".format(os.environ['aws_access_key'])
subprocess.run("~/scripts/{}.py {}".format('ssn_finalize',
params), shell=True, check=True)
except Exception as err:
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Error with writing results.", str(err))
- clear_resources()
- sys.exit(1)
+ remove_cloud_resources(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
+ sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 5f6a36e..2ada46c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -111,7 +111,7 @@ if __name__ == "__main__":
#creating aws vpc
try:
if 'aws_vpc_id' in os.environ and os.environ['aws_vpc_id'] != '':
- ssn_conf['aws_vpc_id'] = os.environ['aws_vpc_id']
+ ssn_conf['predefined_aws_vpc'] = 'true'
else:
logging.info('[CREATE VPC AND ROUTE TABLE]')
params = "--vpc {} --region {} --infra_tag_name {}
--infra_tag_value {} --vpc_name {}".format(
@@ -122,9 +122,9 @@ if __name__ == "__main__":
except:
traceback.print_exc()
raise Exception
- ssn_conf['aws_vpc_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
+ os.environ['aws_vpc_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
ssn_conf['service_base_name'])
- for cidr in
datalab.meta_lib.get_vpc_cidr_by_id(ssn_conf['aws_vpc_id']):
+ for cidr in
datalab.meta_lib.get_vpc_cidr_by_id(os.environ['aws_vpc_id']):
ssn_conf['allowed_vpc_cidr_ip_ranges'].append({"CidrIp": cidr})
except Exception as err:
logging.error('Error: {0}'.format(err))
@@ -146,7 +146,7 @@ if __name__ == "__main__":
except:
traceback.print_exc()
raise Exception
- ssn_conf['aws_vpc2_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
+ os.environ['aws_vpc2_id'] =
datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
ssn_conf['service_base_name'])
except Exception as err:
logging.error('Error: {0}'.format(err))
@@ -157,11 +157,11 @@ if __name__ == "__main__":
#creating subnet
try:
if 'aws_subnet_id' in os.environ and os.environ['aws_subnet_id'] != '':
- ssn_conf['aws_subnet_id'] = os.environ['aws_subnet_id']
+ ssn_conf['predefined_aws_subnet'] = 'true'
else:
logging.info('[CREATE SUBNET]')
params = "--vpc_id {0} --username {1} --infra_tag_name {2}
--infra_tag_value {3} --prefix {4} " \
- "--ssn {5} --zone {6} --subnet_name
{7}".format(ssn_conf['aws_vpc_id'], 'ssn',
+ "--ssn {5} --zone {6} --subnet_name
{7}".format(os.environ['aws_vpc_id'], 'ssn',
ssn_conf['tag_name'],
ssn_conf['service_base_name'],
'20',
True,
@@ -171,9 +171,9 @@ if __name__ == "__main__":
except:
traceback.print_exc()
raise Exception
- ssn_conf['aws_subnet_id'] =
datalab.meta_lib.get_subnet_by_tag(ssn_conf['subnet_tag'], True,
-
ssn_conf['aws_vpc_id'])
-
datalab.actions_lib.enable_auto_assign_ip(ssn_conf['aws_subnet_id'])
+ os.environ['aws_subnet_id'] =
datalab.meta_lib.get_subnet_by_tag(ssn_conf['subnet_tag'], True,
+
os.environ['aws_vpc_id'])
+
datalab.actions_lib.enable_auto_assign_ip(os.environ['aws_subnet_id'])
except Exception as err:
logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create Subnet", str(err))
@@ -182,14 +182,14 @@ if __name__ == "__main__":
#creating peering connection
try:
- if os.environ['conf_duo_vpc_enable'] == 'true' and
ssn_conf['aws_vpc_id'] and ssn_conf['aws_vpc2_id']:
+ if os.environ['conf_duo_vpc_enable'] == 'true' and
os.environ['aws_vpc_id'] and os.environ['aws_vpc2_id']:
logging.info('[CREATE PEERING CONNECTION]')
- ssn_conf['aws_peering_id'] =
datalab.actions_lib.create_peering_connection(
- ssn_conf['aws_vpc_id'], ssn_conf['aws_vpc2_id'],
ssn_conf['service_base_name'])
- logging.info('PEERING CONNECTION ID:' +
ossn_conf['aws_peering_id'])
- datalab.actions_lib.create_route_by_id(ssn_conf['aws_subnet_id'],
ssn_conf['aws_vpc_id'],
- ssn_conf['aws_peering_id'],
-
datalab.meta_lib.get_cidr_by_vpc(ssn_conf['aws_vpc2_id']))
+ os.environ['aws_peering_id'] =
datalab.actions_lib.create_peering_connection(
+ os.environ['aws_vpc_id'], os.environ['aws_vpc2_id'],
ssn_conf['service_base_name'])
+ logging.info('PEERING CONNECTION ID:' +
os.environ['aws_peering_id'])
+
datalab.actions_lib.create_route_by_id(os.environ['aws_subnet_id'],
os.environ['aws_vpc_id'],
+
os.environ['aws_peering_id'],
+
datalab.meta_lib.get_cidr_by_vpc(os.environ['aws_vpc2_id']))
except Exception as err:
logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create peering connection",
str(err))
@@ -199,7 +199,7 @@ if __name__ == "__main__":
#creating security groups
try:
if 'aws_security_groups_ids' in os.environ and
os.environ['aws_security_groups_ids'] != '':
- ssn_conf['aws_security_groups_ids'] =
os.environ['aws_security_groups_ids']
+ ssn_conf['predefined_security_groups'] = 'true'
else:
logging.info('[CREATE SG FOR SSN]')
ssn_conf['ingress_sg_rules_template'] =
datalab.meta_lib.format_sg([
@@ -246,7 +246,7 @@ if __name__ == "__main__":
])
params = "--name {} --vpc_id {} --security_group_rules '{}'
--egress '{}' --infra_tag_name {} " \
"--infra_tag_value {} --force {} --ssn {}". \
- format(ssn_conf['sg_name'], ssn_conf['aws_vpc_id'],
+ format(ssn_conf['sg_name'], os.environ['aws_vpc_id'],
json.dumps(ssn_conf['ingress_sg_rules_template']),
json.dumps(egress_sg_rules_template),
ssn_conf['service_base_name'], ssn_conf['tag_name'],
False, True)
try:
@@ -254,7 +254,7 @@ if __name__ == "__main__":
except:
traceback.print_exc()
raise Exception
- ssn_conf['aws_security_groups_ids'] =
datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
+ os.environ['aws_security_groups_ids'] =
datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
except Exception as err:
logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to create security group for SSN",
str(err))
@@ -284,7 +284,7 @@ if __name__ == "__main__":
try:
logging.info('[CREATE ENDPOINT AND ROUTE-TABLE]')
params = "--vpc_id {} --region {} --infra_tag_name {}
--infra_tag_value {}".format(
- ssn_conf['aws_vpc_id'], ssn_conf['region'], ssn_conf['tag_name'],
ssn_conf['service_base_name'])
+ os.environ['aws_vpc_id'], ssn_conf['region'],
ssn_conf['tag_name'], ssn_conf['service_base_name'])
try:
subprocess.run("~/scripts/{}.py {}".format('ssn_create_endpoint',
params), shell=True, check=True)
except:
@@ -301,7 +301,7 @@ if __name__ == "__main__":
if os.environ['conf_duo_vpc_enable'] == 'true':
logging.info('[CREATE ENDPOINT AND ROUTE-TABLE FOR NOTEBOOK VPC]')
params = "--vpc_id {} --region {} --infra_tag_name {}
--infra_tag_value {}".format(
- ssn_conf['aws_vpc2_id'], ssn_conf['aws_region'],
ssn_conf['tag2_name'],
+ os.environ['aws_vpc2_id'], ssn_conf['aws_region'],
ssn_conf['tag2_name'],
ssn_conf['service_base_name'])
try:
subprocess.run("~/scripts/{}.py
{}".format('ssn_create_endpoint', params), shell=True, check=True)
@@ -321,7 +321,7 @@ if __name__ == "__main__":
"--subnet_id {5} --iam_profile {6} --infra_tag_name {7}
--infra_tag_value {8} --instance_class {9} " \
"--primary_disk_size {10}".\
format(ssn_conf['instance_name'], ssn_conf['ssn_ami_id'],
os.environ['aws_ssn_instance_size'],
- os.environ['conf_key_name'],
ssn_conf['aws_security_groups_ids'], ssn_conf['aws_subnet_id'],
+ os.environ['conf_key_name'],
os.environ['aws_security_groups_ids'], os.environ['aws_subnet_id'],
ssn_conf['role_profile_name'], ssn_conf['tag_name'],
ssn_conf['instance_name'], 'ssn', '20')
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
