This is an automated email from the ASF dual-hosted git repository. omartushevskyi pushed a commit to branch DLAB-1546 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit a4798b1a55abc56aab711efd2c6dac70878275bd Merge: e8dc4ad 0e8c625 Author: Oleh Martushevskyi <[email protected]> AuthorDate: Thu Feb 27 11:55:28 2020 +0200 Merge branch 'develop' of github.com:apache/incubator-dlab into DLAB-1546 Conflicts: infrastructure-provisioning/src/general/scripts/aws/project_prepare.py infrastructure-provisioning/src/general/scripts/azure/project_prepare.py infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py .../scripts/POST_DEPLOYMENT.md | 2 + .../src/general/conf/dlab.ini | 2 +- .../src/general/files/azure/jupyterlab_Dockerfile | 51 +++ .../files/azure/jupyterlab_description.json | 25 ++ .../general/files/gcp/jupyterlab_description.json | 2 +- .../src/general/lib/aws/actions_lib.py | 19 +- .../src/general/lib/gcp/actions_lib.py | 5 +- .../src/general/lib/os/debian/common_lib.py | 36 +- .../src/general/lib/os/debian/ssn_lib.py | 4 + .../src/general/lib/os/redhat/common_lib.py | 34 +- .../src/general/lib/os/redhat/edge_lib.py | 65 +++- .../src/general/lib/os/redhat/notebook_lib.py | 2 +- .../src/general/lib/os/redhat/ssn_lib.py | 13 +- .../general/scripts/aws/common_create_bucket.py | 10 +- .../src/general/scripts/aws/project_prepare.py | 23 +- .../general/scripts/azure/jupyterlab_configure.py | 398 +++++++++++++++++++++ .../src/general/scripts/azure/project_prepare.py | 10 +- .../src/general/scripts/azure/ssn_configure.py | 1 + .../general/scripts/gcp/common_create_bucket.py | 3 +- .../src/general/scripts/gcp/project_prepare.py | 18 +- .../src/general/scripts/gcp/project_terminate.py | 7 +- .../jenkins_jobs/create_notebook_image/config.xml | 1 + .../jenkins_jobs/create_notebook_server/config.xml | 1 + .../jenkins_jobs/manage_git_credentials/config.xml | 1 + .../jenkins_jobs/start_notebook_server/config.xml | 1 + .../jenkins_jobs/stop_notebook_server/config.xml | 1 + .../terminate_notebook_image/config.xml | 1 + .../terminate_notebook_server/config.xml | 1 + .../src/jupyterlab/Dockerfile_jupyterlab | 8 +- .../scripts/configure_jupyterlab_node.py | 7 - .../reporting-grid/reporting-grid.component.html | 2 +- .../src/app/reporting/reporting.component.ts | 30 +- .../app/reporting/toolbar/toolbar.component.html | 1 - .../app/reporting/toolbar/toolbar.component.scss | 1 + .../src/app/reporting/toolbar/toolbar.component.ts | 14 +- .../computational-resources-list.component.scss | 19 + .../install-libraries.component.ts | 2 + .../src/app/shared/navbar/navbar.component.ts | 4 +- 38 files changed, 709 insertions(+), 116 deletions(-) diff --cc infrastructure-provisioning/src/general/scripts/aws/project_prepare.py index 5b18969,009b81e..5b9a3f4 --- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py @@@ -42,124 -38,111 +42,126 @@@ if __name__ == "__main__" level=logging.DEBUG, filename=local_log_filepath) - create_aws_config_files() - print('Generating infrastructure names and tags') - project_conf = dict() - project_conf['service_base_name'] = os.environ['conf_service_base_name'] = replace_multi_symbols( - os.environ['conf_service_base_name'].lower()[:12], '-', True) - project_conf['endpoint_name'] = os.environ['endpoint_name'] - project_conf['endpoint_tag'] = os.environ['endpoint_name'] - project_conf['project_name'] = os.environ['project_name'] - project_conf['project_tag'] = os.environ['project_name'] - project_conf['key_name'] = os.environ['conf_key_name'] - project_conf['public_subnet_id'] = os.environ['aws_subnet_id'] - project_conf['vpc_id'] = os.environ['aws_vpc_id'] - project_conf['region'] = os.environ['aws_region'] - project_conf['ami_id'] = get_ami_id(os.environ['aws_{}_image_name'.format(os.environ['conf_os_family'])]) - project_conf['instance_size'] = os.environ['aws_edge_instance_size'] - project_conf['sg_ids'] = os.environ['aws_security_groups_ids'] - project_conf['edge_instance_name'] = '{}-{}-{}-edge'.format(project_conf['service_base_name'], - os.environ['project_name'], os.environ['endpoint_name']) - project_conf['tag_name'] = '{}-Tag'.format(project_conf['service_base_name']) - project_conf['bucket_name_tag'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'], - project_conf['project_name'], - project_conf['endpoint_name']) - project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-') - project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format( - project_conf['service_base_name'], project_conf['endpoint_tag']) - project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-') - project_conf['edge_role_name'] = '{}-{}-edge-Role'.format( - project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name']) - project_conf['edge_role_profile_name'] = '{}-{}-edge-Profile'.format( - project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name']) - project_conf['edge_policy_name'] = '{}-{}-edge-Policy'.format( - project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name']) - project_conf['edge_security_group_name'] = '{}-sg'.format(project_conf['edge_instance_name']) - project_conf['notebook_instance_name'] = '{}-{}-nb'.format(project_conf['service_base_name'], - os.environ['project_name']) - project_conf['dataengine_instances_name'] = '{}-{}-dataengine' \ - .format(project_conf['service_base_name'], os.environ['project_name']) - project_conf['notebook_dataengine_role_name'] = '{}-{}-{}-nb-de-Role' \ - .format(project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'],os.environ['endpoint_name']) - project_conf['notebook_dataengine_policy_name'] = '{}-{}-{}-nb-de-Policy' \ - .format(project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'],os.environ['endpoint_name']) - project_conf['notebook_dataengine_role_profile_name'] = '{}-{}-{}-nb-de-Profile' \ - .format(project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name'],os.environ['endpoint_name']) - project_conf['notebook_security_group_name'] = '{}-{}-{}-nb-sg'.format(project_conf['service_base_name'], - os.environ['project_name'],os.environ['endpoint_name']) - project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix'] - project_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(project_conf['service_base_name'], - os.environ['project_name']) - project_conf['dataengine_master_security_group_name'] = '{}-{}-dataengine-master-sg' \ - .format(project_conf['service_base_name'], os.environ['project_name']) - project_conf['dataengine_slave_security_group_name'] = '{}-{}-dataengine-slave-sg' \ - .format(project_conf['service_base_name'], os.environ['project_name']) - project_conf['allowed_ip_cidr'] = list() - for cidr in os.environ['conf_allowed_ip_cidr'].split(','): - project_conf['allowed_ip_cidr'].append({"CidrIp": cidr.replace(' ','')}) - project_conf['network_type'] = os.environ['conf_network_type'] - project_conf['all_ip_cidr'] = '0.0.0.0/0' - project_conf['zone'] = os.environ['aws_region'] + os.environ['aws_zone'] - project_conf['elastic_ip_name'] = '{0}-{1}-edge-EIP'.format(project_conf['service_base_name'], - os.environ['project_name']) - project_conf['provision_instance_ip'] = None - project_conf['local_endpoint'] = False try: - project_conf['provision_instance_ip'] = get_instance_ip_address( - project_conf['tag_name'], '{0}-{1}-endpoint'.format(project_conf['service_base_name'], - os.environ['endpoint_name'])).get('Private') + "/32" - except: - project_conf['provision_instance_ip'] = get_instance_ip_address(project_conf['tag_name'], '{0}-ssn'.format( - project_conf['service_base_name'])).get('Private') + "/32" - project_conf['local_endpoint'] = True - if 'aws_user_predefined_s3_policies' not in os.environ: - os.environ['aws_user_predefined_s3_policies'] = 'None' + dlab.actions_lib.create_aws_config_files() + print('Generating infrastructure names and tags') + project_conf = dict() + project_conf['service_base_name'] = os.environ['conf_service_base_name'] = dlab.fab.replace_multi_symbols( + os.environ['conf_service_base_name'].lower()[:20], '-', True) + project_conf['endpoint_name'] = os.environ['endpoint_name'].lower() + project_conf['endpoint_tag'] = project_conf['endpoint_name'] + project_conf['project_name'] = os.environ['project_name'].lower() + project_conf['project_tag'] = project_conf['project_name'] + project_conf['key_name'] = os.environ['conf_key_name'] + project_conf['public_subnet_id'] = os.environ['aws_subnet_id'] + project_conf['vpc_id'] = os.environ['aws_vpc_id'] + project_conf['region'] = os.environ['aws_region'] + project_conf['ami_id'] = dlab.meta_lib.get_ami_id(os.environ['aws_{}_image_name'.format( + os.environ['conf_os_family'])]) + project_conf['instance_size'] = os.environ['aws_edge_instance_size'] + project_conf['sg_ids'] = os.environ['aws_security_groups_ids'] + project_conf['edge_instance_name'] = '{}-{}-{}-edge'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['tag_name'] = '{}-tag'.format(project_conf['service_base_name']) + project_conf['bucket_name_tag'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-') + project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format( + project_conf['service_base_name'], project_conf['endpoint_tag']) + project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-') + project_conf['edge_role_name'] = '{}-{}-{}-edge-role'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['edge_role_profile_name'] = '{}-{}-{}-edge-profile'.format( + project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) + project_conf['edge_policy_name'] = '{}-{}-{}-edge-policy'.format( + project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) + project_conf['edge_security_group_name'] = '{}-{}-{}-edge-sg'.format(project_conf['edge_instance_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['notebook_instance_name'] = '{}-{}-{}-nb'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['dataengine_instances_name'] = '{}-{}-{}-de'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['notebook_dataengine_role_name'] = '{}-{}-{}-nb-de-role'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['notebook_dataengine_policy_name'] = '{}-{}-{}-nb-de-policy'.format( + project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) + project_conf['notebook_dataengine_role_profile_name'] = '{}-{}-{}-nb-de-profile'.format( + project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) + project_conf['notebook_security_group_name'] = '{}-{}-{}-nb-sg'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix'] + project_conf['private_subnet_name'] = '{0}-{1}-{2}-subnet'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['dataengine_master_security_group_name'] = '{}-{}-{}-de-master-sg'.format( + project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) + project_conf['dataengine_slave_security_group_name'] = '{}-{}-{}-de-slave-sg'.format( + project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) + project_conf['allowed_ip_cidr'] = list() + for cidr in os.environ['conf_allowed_ip_cidr'].split(','): + project_conf['allowed_ip_cidr'].append({"CidrIp": cidr.replace(' ', '')}) + project_conf['network_type'] = os.environ['conf_network_type'] + project_conf['all_ip_cidr'] = '0.0.0.0/0' + project_conf['zone'] = os.environ['aws_region'] + os.environ['aws_zone'] + project_conf['elastic_ip_name'] = '{0}-{1}-{2}-edge-static-ip'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['provision_instance_ip'] = None + project_conf['local_endpoint'] = False + try: + project_conf['provision_instance_ip'] = '{}/32'.format(dlab.meta_lib.get_instance_ip_address( + project_conf['tag_name'], '{0}-{1}-endpoint'.format(project_conf['service_base_name'], + project_conf['endpoint_name'])).get('Private')) + except: + project_conf['provision_instance_ip'] = '{}/32'.format(dlab.meta_lib.get_instance_ip_address( + project_conf['tag_name'], '{0}-ssn'.format(project_conf['service_base_name'])).get('Private')) + project_conf['local_endpoint'] = True + if 'aws_user_predefined_s3_policies' not in os.environ: + os.environ['aws_user_predefined_s3_policies'] = 'None' - try: - if os.environ['conf_user_subnets_range'] == '': - raise KeyError - except KeyError: - os.environ['conf_user_subnets_range'] = '' + try: + if os.environ['conf_user_subnets_range'] == '': + raise KeyError + except KeyError: + os.environ['conf_user_subnets_range'] = '' - # FUSE in case of absence of user's key - try: - project_conf['user_key'] = os.environ['key'] + # FUSE in case of absence of user's key try: - local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'], - project_conf['project_name'])) - except: - print("ADMINSs PUBLIC KEY DOES NOT INSTALLED") - except KeyError: - print("ADMINSs PUBLIC KEY DOES NOT UPLOADED") - sys.exit(1) + project_conf['user_key'] = os.environ['key'] + try: + local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'], + project_conf['project_name'])) + except: + print("ADMINSs PUBLIC KEY DOES NOT INSTALLED") + except KeyError: + print("ADMINSs PUBLIC KEY DOES NOT UPLOADED") + sys.exit(1) - print("Will create exploratory environment with edge node as access point as following: {}". - format(json.dumps(project_conf, sort_keys=True, indent=4, separators=(',', ': ')))) - logging.info(json.dumps(project_conf)) + print("Will create exploratory environment with edge node as access point as following: {}". + format(json.dumps(project_conf, sort_keys=True, indent=4, separators=(',', ': ')))) + logging.info(json.dumps(project_conf)) - if 'conf_additional_tags' in os.environ: - project_conf['bucket_additional_tags'] = ';' + os.environ['conf_additional_tags'] - os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + \ - ';project_tag:{0};endpoint_tag:{1};'.format( - project_conf['project_tag'], project_conf['endpoint_tag']) - else: - project_conf['bucket_additional_tags'] = '' - os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(project_conf['project_tag'], - project_conf['endpoint_tag']) - print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags'])) + if 'conf_additional_tags' in os.environ: ++ project_conf['bucket_additional_tags'] = ';' + os.environ['conf_additional_tags'] + os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + \ + ';project_tag:{0};endpoint_tag:{1};'.format( + project_conf['project_tag'], project_conf['endpoint_tag']) + else: ++ project_conf['bucket_additional_tags'] = '' + os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(project_conf['project_tag'], + project_conf['endpoint_tag']) + print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags'])) + except Exception as err: + dlab.fab.append_result("Failed to generate variables dictionary.", str(err)) + sys.exit(1) if not project_conf['local_endpoint']: # attach project_tag and endpoint_tag to endpoint @@@ -549,19 -533,22 +551,24 @@@ try: logging.info('[CREATE BUCKETS]') print('[CREATE BUCKETS]') - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} " \ - "--bucket_name_tag {}".format(project_conf['shared_bucket_name'], project_conf['tag_name'], - project_conf['shared_bucket_name'], project_conf['region'], - project_conf['shared_bucket_name_tag']) - project_conf['shared_bucket_tags'] = 'endpoint_tag:{0};{1}:{2};{3}:{4}{5}'.format(project_conf['endpoint_tag'], - os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'], - project_conf['tag_name'], project_conf['shared_bucket_name'], project_conf['bucket_additional_tags']).replace(';', ',') ++ project_conf['shared_bucket_tags'] = 'endpoint_tag:{0};{1}:{2};{3}:{4}{5}'.format( ++ project_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'], ++ project_conf['tag_name'], project_conf['shared_bucket_name'], ++ project_conf['bucket_additional_tags']).replace(';', ',') + params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {}". \ - format(project_conf['shared_bucket_name'], project_conf['shared_bucket_tags'], project_conf['region'], project_conf['shared_bucket_name_tag']) ++ format(project_conf['shared_bucket_name'], project_conf['shared_bucket_tags'], project_conf['region'], ++ project_conf['shared_bucket_name_tag']) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) except: traceback.print_exc() raise Exception - - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}" \ - .format(project_conf['bucket_name'], project_conf['tag_name'], project_conf['bucket_name'], - project_conf['region'], project_conf['bucket_name_tag']) - project_conf['bucket_tags'] = 'endpoint_tag:{0};{1}:{2};project_tag:{3};{4}:{5}{6}'.format(project_conf['endpoint_tag'], - os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'], - project_conf['project_tag'], - project_conf['tag_name'], project_conf['bucket_name'], project_conf['bucket_additional_tags']).replace(';', ',') ++ project_conf['bucket_tags'] = 'endpoint_tag:{0};{1}:{2};project_tag:{3};{4}:{5}{6}'.format( ++ project_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'], ++ project_conf['project_tag'], project_conf['tag_name'], project_conf['bucket_name'], ++ project_conf['bucket_additional_tags']).replace(';', ',') + params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {}" \ + .format(project_conf['bucket_name'], project_conf['bucket_tags'], project_conf['region'], project_conf['bucket_name_tag']) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) except: diff --cc infrastructure-provisioning/src/general/scripts/azure/project_prepare.py index 42a4fa3,f5c98ab..010e3df --- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py @@@ -55,12 -49,10 +55,12 @@@ if __name__ == "__main__" project_conf['azure_ad_user_name'] = os.environ['azure_iam_user'] project_conf['key_name'] = os.environ['conf_key_name'] - + project_conf['tag_name'] = project_conf['service_base_name'] + '-tag' project_conf['vpc_name'] = os.environ['azure_vpc_name'] project_conf['subnet_name'] = os.environ['azure_subnet_name'] - project_conf['private_subnet_name'] = project_conf['service_base_name'] + '-' + project_conf['project_name'] + '-subnet' + project_conf['private_subnet_name'] = '{}-{}-{}-subnet'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) if os.environ['conf_network_type'] == 'private': project_conf['static_public_ip_name'] = 'None' else: @@@ -112,17 -95,17 +112,21 @@@ "SBN": project_conf['service_base_name'], "project_tag": project_conf['project_tag'], "endpoint_tag": project_conf['endpoint_tag'], - os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'], + os.environ['conf_billing_tag_key']: - os.environ['conf_billing_tag_value']} ++ os.environ['conf_billing_tag_value'], + project_conf['tag_name']: project_conf['edge_storage_account_name']} project_conf['primary_disk_size'] = '32' - project_conf['shared_storage_account_name'] = '{0}-{1}-shared-storage'.format(project_conf['service_base_name'], - project_conf['endpoint_name']) - project_conf['shared_container_name'] = '{}-{}-shared-container'.format(project_conf['service_base_name'], project_conf['endpoint_name']).lower() + project_conf['shared_storage_account_name'] = '{0}-{1}-shared-bucket'.format(project_conf['service_base_name'], + project_conf['endpoint_name']) + project_conf['shared_container_name'] = '{}-{}-shared-bucket'.format(project_conf['service_base_name'], + project_conf['endpoint_name']) project_conf['shared_storage_account_tags'] = {"Name": project_conf['shared_storage_account_name'], - "SBN": project_conf['service_base_name'], - os.environ['conf_billing_tag_key']: os.environ[ - 'conf_billing_tag_value'], "endpoint_tag": project_conf['endpoint_tag'], - project_conf['tag_name']: project_conf['shared_storage_account_name']} + "SBN": project_conf['service_base_name'], + os.environ['conf_billing_tag_key']: os.environ[ - 'conf_billing_tag_value']} ++ 'conf_billing_tag_value'], "endpoint_tag": ++ project_conf['endpoint_tag'], ++ project_conf['tag_name']: ++ project_conf['shared_storage_account_name']} # FUSE in case of absence of user's key try: diff --cc infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py index 1384e24,82edc16..7858d3f --- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py @@@ -39,108 -35,92 +39,109 @@@ if __name__ == "__main__" logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', level=logging.DEBUG, filename=local_log_filepath) - - print('Generating infrastructure names and tags') - project_conf = dict() - project_conf['edge_unique_index'] = str(uuid.uuid4())[:5] - project_conf['ps_unique_index'] = str(uuid.uuid4())[:5] - project_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-') - project_conf['key_name'] = os.environ['conf_key_name'] - project_conf['user_keyname'] = os.environ['project_name'] - project_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-') - project_conf['project_tag'] = (os.environ['project_name']).lower().replace('_', '-') - project_conf['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-') - project_conf['endpoint_name'] = (os.environ['endpoint_name']).lower().replace('_', '-') try: - if os.environ['gcp_vpc_name'] == '': - raise KeyError - else: - project_conf['vpc_name'] = os.environ['gcp_vpc_name'] - except KeyError: - project_conf['vpc_name'] = project_conf['service_base_name'] + '-ssn-vpc' - project_conf['vpc_cidr'] = os.environ['conf_vpc_cidr'] - project_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(project_conf['service_base_name'], - project_conf['project_name']) - project_conf['subnet_name'] = os.environ['gcp_subnet_name'] - project_conf['region'] = os.environ['gcp_region'] - project_conf['zone'] = os.environ['gcp_zone'] - project_conf['vpc_selflink'] = GCPMeta().get_vpc(project_conf['vpc_name'])['selfLink'] - project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix'] - project_conf['edge_service_account_name'] = '{}-{}-edge'.format(project_conf['service_base_name'], - project_conf['project_name']) - project_conf['edge_role_name'] = '{}-{}-{}-edge'.format(project_conf['service_base_name'], - project_conf['project_name'], project_conf['edge_unique_index']) - project_conf['ps_service_account_name'] = '{}-{}-ps'.format(project_conf['service_base_name'], - project_conf['project_name']) - project_conf['ps_role_name'] = '{}-{}-{}-ps'.format(project_conf['service_base_name'], - project_conf['project_name'], project_conf['ps_unique_index']) - project_conf['ps_policy_path'] = '/root/files/ps_policy.json' - project_conf['ps_roles_path'] = '/root/files/ps_roles.json' - project_conf['instance_name'] = '{0}-{1}-{2}-edge'.format(project_conf['service_base_name'], - project_conf['project_name'], project_conf['endpoint_tag']) - project_conf['ssn_instance_name'] = '{}-ssn'.format(project_conf['service_base_name']) - project_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'], + print('Generating infrastructure names and tags') + project_conf = dict() + project_conf['edge_unique_index'] = str(uuid.uuid4())[:5] + project_conf['ps_unique_index'] = str(uuid.uuid4())[:5] + project_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower() + project_conf['key_name'] = os.environ['conf_key_name'] + project_conf['project_name'] = (os.environ['project_name']).lower() + project_conf['user_keyname'] = project_conf['project_name'] + project_conf['project_tag'] = (project_conf['project_name']) + project_conf['endpoint_name'] = (os.environ['endpoint_name']).lower() + project_conf['endpoint_tag'] = project_conf['endpoint_name'] + try: + if os.environ['gcp_vpc_name'] == '': + raise KeyError + else: + project_conf['vpc_name'] = os.environ['gcp_vpc_name'] + except KeyError: + project_conf['vpc_name'] = project_conf['service_base_name'] + '-vpc' + project_conf['vpc_cidr'] = os.environ['conf_vpc_cidr'] + project_conf['private_subnet_name'] = '{0}-{1}-{2}-subnet'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['subnet_name'] = os.environ['gcp_subnet_name'] + project_conf['region'] = os.environ['gcp_region'] + project_conf['zone'] = os.environ['gcp_zone'] + project_conf['vpc_selflink'] = GCPMeta().get_vpc(project_conf['vpc_name'])['selfLink'] + project_conf['private_subnet_prefix'] = os.environ['conf_private_subnet_prefix'] + project_conf['edge_service_account_name'] = '{}-{}-{}-edge-sa'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['edge_role_name'] = '{}-{}-{}-{}-edge-role'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name'], + project_conf['edge_unique_index']) + project_conf['ps_service_account_name'] = '{}-{}-{}-ps-sa'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['ps_role_name'] = '{}-{}-{}-{}-ps-role'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name'], + project_conf['ps_unique_index']) + project_conf['ps_policy_path'] = '/root/files/ps_policy.json' + project_conf['ps_roles_path'] = '/root/files/ps_roles.json' + project_conf['instance_name'] = '{0}-{1}-{2}-edge'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['ssn_instance_name'] = '{}-ssn'.format(project_conf['service_base_name']) + project_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['shared_bucket_name'] = '{0}-{1}-shared-bucket'.format(project_conf['service_base_name'], + project_conf['endpoint_name']) + project_conf['instance_size'] = os.environ['gcp_edge_instance_size'] + project_conf['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) + project_conf['image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])] + project_conf['static_address_name'] = '{0}-{1}-{2}-static-ip'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['fw_edge_ingress_public'] = '{}-sg-ingress-public'.format(project_conf['instance_name']) + project_conf['fw_edge_ingress_internal'] = '{}-sg-ingress-internal'.format(project_conf['instance_name']) + project_conf['fw_edge_egress_public'] = '{}-sg-egress-public'.format(project_conf['instance_name']) + project_conf['fw_edge_egress_internal'] = '{}-sg-egress-internal'.format(project_conf['instance_name']) + project_conf['ps_firewall_target'] = '{0}-{1}-{2}-ps'.format(project_conf['service_base_name'], + project_conf['project_name'], + project_conf['endpoint_name']) + project_conf['fw_common_name'] = '{}-{}-{}-ps'.format(project_conf['service_base_name'], project_conf['project_name'], project_conf['endpoint_name']) - project_conf['shared_bucket_name'] = '{0}-{1}-shared-bucket'.format(project_conf['service_base_name'], - project_conf['endpoint_name']) - project_conf['instance_size'] = os.environ['gcp_edge_instance_size'] - project_conf['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) - project_conf['image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])] - project_conf['static_address_name'] = '{0}-{1}-ip'.format(project_conf['service_base_name'], project_conf['project_name']) - project_conf['fw_edge_ingress_public'] = '{}-ingress-public'.format(project_conf['instance_name']) - project_conf['fw_edge_ingress_internal'] = '{}-ingress-internal'.format(project_conf['instance_name']) - project_conf['fw_edge_egress_public'] = '{}-egress-public'.format(project_conf['instance_name']) - project_conf['fw_edge_egress_internal'] = '{}-egress-internal'.format(project_conf['instance_name']) - project_conf['ps_firewall_target'] = '{0}-{1}-ps'.format(project_conf['service_base_name'], - project_conf['project_name']) - project_conf['fw_common_name'] = '{}-{}-ps'.format(project_conf['service_base_name'], project_conf['project_name']) - project_conf['fw_ps_ingress'] = '{}-ingress'.format(project_conf['fw_common_name']) - project_conf['fw_ps_egress_private'] = '{}-egress-private'.format(project_conf['fw_common_name']) - project_conf['fw_ps_egress_public'] = '{}-egress-public'.format(project_conf['fw_common_name']) - project_conf['network_tag'] = project_conf['instance_name'] - project_conf['instance_labels'] = {"name": project_conf['instance_name'], - "sbn": project_conf['service_base_name'], - "project_tag": project_conf['project_tag'], - "endpoint_tag": project_conf['endpoint_tag'], - "product": "dlab"} - project_conf['tag_name'] = project_conf['service_base_name'] + '-tag' - project_conf['allowed_ip_cidr'] = os.environ['conf_allowed_ip_cidr'] - if 'conf_user_subnets_range' in os.environ: - project_conf['user_subnets_range'] = os.environ['conf_user_subnets_range'] - else: - project_conf['user_subnets_range'] = '' + project_conf['fw_ps_ingress'] = '{}-sg-ingress'.format(project_conf['fw_common_name']) + project_conf['fw_ps_egress_private'] = '{}-sg-egress-private'.format(project_conf['fw_common_name']) + project_conf['fw_ps_egress_public'] = '{}-sg-egress-public'.format(project_conf['fw_common_name']) + project_conf['network_tag'] = project_conf['instance_name'] + project_conf['instance_labels'] = {"name": project_conf['instance_name'], + "sbn": project_conf['service_base_name'], + "project_tag": project_conf['project_tag'], + "endpoint_tag": project_conf['endpoint_tag'], + "product": "dlab"} ++ project_conf['tag_name'] = project_conf['service_base_name'] + '-tag' + project_conf['allowed_ip_cidr'] = os.environ['conf_allowed_ip_cidr'] + if 'conf_user_subnets_range' in os.environ: + project_conf['user_subnets_range'] = os.environ['conf_user_subnets_range'] + else: + project_conf['user_subnets_range'] = '' - # FUSE in case of absence of user's key - try: - project_conf['user_key'] = os.environ['key'] + # FUSE in case of absence of user's key try: - local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'], os.environ['project_name'])) - except: - print("ADMINSs PUBLIC KEY DOES NOT INSTALLED") - except KeyError: - print("ADMINSs PUBLIC KEY DOES NOT UPLOADED") - sys.exit(1) + project_conf['user_key'] = os.environ['key'] + try: + local('echo "{0}" >> {1}{2}.pub'.format(project_conf['user_key'], os.environ['conf_key_dir'], + project_conf['project_name'])) + except: + print("ADMINSs PUBLIC KEY DOES NOT INSTALLED") + except KeyError: + print("ADMINSs PUBLIC KEY DOES NOT UPLOADED") + sys.exit(1) - print("Will create exploratory environment with edge node as access point as following: ".format(json.dumps( - project_conf, sort_keys=True, indent=4, separators=(',', ': ')))) - logging.info(json.dumps(project_conf)) + print("Will create exploratory environment with edge node as access point as following: ".format(json.dumps( + project_conf, sort_keys=True, indent=4, separators=(',', ': ')))) + logging.info(json.dumps(project_conf)) + except Exception as err: + dlab.fab.append_result("Failed to generate infrastructure names", str(err)) + sys.exit(1) try: logging.info('[CREATE SUBNET]') @@@ -389,14 -367,25 +390,27 @@@ try: logging.info('[CREATE BUCKETS]') print('[CREATE BUCKETS]') - params = "--bucket_name {}".format(project_conf['shared_bucket_name']) + project_conf['shared_bucket_tags'] = { + project_conf['tag_name']: project_conf['shared_bucket_name'], + "endpoint_tag": project_conf['endpoint_tag'], + os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'], + "sbn": project_conf['service_base_name']} - params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'], json.dumps(project_conf['shared_bucket_tags'])) ++ params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'], ++ json.dumps(project_conf['shared_bucket_tags'])) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) except: traceback.print_exc() raise Exception - params = "--bucket_name {}".format(project_conf['bucket_name']) + project_conf['bucket_tags'] = { + project_conf['tag_name']: project_conf['bucket_name'], + "endpoint_tag": project_conf['endpoint_tag'], + os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'], + "sbn": project_conf['service_base_name'], + "project_tag": project_conf['project_tag']} - params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'], json.dumps(project_conf['bucket_tags'])) ++ params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'], ++ json.dumps(project_conf['bucket_tags'])) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
