This is an automated email from the ASF dual-hosted git repository. omartushevskyi pushed a commit to branch DLAB-1546 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit d37f0955ceb8fb240d5fc1ceb6c2a2543aa6b195 Merge: 5c64fa1 51afde3 Author: Oleh Martushevskyi <[email protected]> AuthorDate: Tue Mar 3 13:27:04 2020 +0200 Merge branch 'develop' of github.com:apache/incubator-dlab into DLAB-1546 Conflicts: infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py README.md | 3 -- infrastructure-provisioning/scripts/deploy_dlab.py | 1 - .../src/general/conf/dlab.ini | 2 +- .../src/general/lib/gcp/meta_lib.py | 4 -- .../src/general/lib/os/debian/common_lib.py | 1 + .../src/general/lib/os/fab.py | 27 +++++++++++- .../src/general/scripts/aws/ssn_configure.py | 4 -- .../src/general/scripts/azure/ssn_configure.py | 4 -- ...common_notebook_configure_dataengine-service.py | 12 +++++- .../general/scripts/gcp/common_prepare_notebook.py | 20 ++++++--- .../scripts/gcp/dataengine-service_prepare.py | 12 ++++++ .../src/general/scripts/gcp/dataengine_prepare.py | 13 ++++++ .../src/general/scripts/gcp/ssn_configure.py | 11 ++--- .../src/general/scripts/gcp/ssn_prepare.py | 36 ++++++++-------- .../src/ssn/scripts/configure_ui.py | 2 +- .../terraform/azure/endpoint/main/main.tf | 1 + .../terraform/azure/endpoint/main/network.tf | 7 ++- .../project-list/project-list.component.html | 2 +- .../app/administration/roles/roles.component.ts | 50 ++++++++++++++++++---- .../reporting-grid/reporting-grid.component.html | 7 ++- .../src/app/reporting/reporting.component.ts | 4 +- .../resources-grid/resources-grid.component.scss | 1 + .../confirmation-dialog-type.enum.ts | 3 +- .../confirmation-dialog.component.html | 9 ++-- .../webapp/src/assets/styles/_dialogs.scss | 6 +++ .../webapp/src/dictionary/azure.dictionary.ts | 6 +-- 26 files changed, 173 insertions(+), 75 deletions(-) diff --cc infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py index 0c08cde,010fad9..fb52049 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py @@@ -83,8 -80,10 +91,10 @@@ if __name__ == "__main__" "project_tag": notebook_config['project_tag'], "endpoint_tag": notebook_config['endpoint_tag'], "product": "dlab", - "computational_name": (os.environ['computational_name']).lower().replace('_', '-') + "computational_name": (os.environ['computational_name'].replace('_', '-')) } + if notebook_config['custom_tag'] != '': + notebook_config['cluster_labels'].update({'custom_tag': notebook_config['custom_tag']}) try: logging.info('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]') diff --cc infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py index cc9cadb,39c93e1..63985db --- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py @@@ -40,130 -37,121 +40,140 @@@ if __name__ == "__main__" logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', level=logging.DEBUG, filename=local_log_filepath) - - print('Generating infrastructure names and tags') - notebook_config = dict() - notebook_config['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-') - notebook_config['edge_user_name'] = (os.environ['edge_user_name']).lower().replace('_', '-') - notebook_config['project_name'] = (os.environ['project_name']).lower().replace('_', '-') - notebook_config['project_tag'] = (os.environ['project_name']).lower().replace('_', '-') - notebook_config['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-') - notebook_config['region'] = os.environ['gcp_region'] - notebook_config['zone'] = os.environ['gcp_zone'] - - edge_status = GCPMeta().get_instance_status('{0}-{1}-{2}-edge'.format(notebook_config['service_base_name'], - notebook_config['project_name'], - notebook_config['endpoint_tag'])) - if edge_status != 'RUNNING': - logging.info('ERROR: Edge node is unavailable! Aborting...') - print('ERROR: Edge node is unavailable! Aborting...') - ssn_hostname = GCPMeta().get_private_ip_address(notebook_config['service_base_name'] + '-ssn') - put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'], - ssn_hostname) - append_result("Edge node is unavailable") - sys.exit(1) - - try: - if os.environ['gcp_vpc_name'] == '': - raise KeyError - else: - notebook_config['vpc_name'] = os.environ['gcp_vpc_name'] - except KeyError: - notebook_config['vpc_name'] = '{}-ssn-vpc'.format(notebook_config['service_base_name']) try: - notebook_config['exploratory_name'] = (os.environ['exploratory_name']).lower().replace('_', '-') - except: - notebook_config['exploratory_name'] = '' - notebook_config['subnet_name'] = '{0}-{1}-subnet'.format(notebook_config['service_base_name'], - notebook_config['project_name']) - notebook_config['instance_size'] = os.environ['gcp_notebook_instance_size'] - notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) - notebook_config['notebook_service_account_name'] = '{}-{}-ps'.format(notebook_config['service_base_name'], - notebook_config['project_name']).replace('_', '-') - - if os.environ['conf_os_family'] == 'debian': - initial_user = 'ubuntu' - sudo_group = 'sudo' - if os.environ['conf_os_family'] == 'redhat': - initial_user = 'ec2-user' - sudo_group = 'wheel' - notebook_config['instance_name'] = '{0}-{1}-{2}-nb-{3}'.format(notebook_config['service_base_name'], - notebook_config['project_name'], os.environ['endpoint_name'], - notebook_config['exploratory_name']) - notebook_config['primary_disk_size'] = (lambda x: '30' if x == 'deeplearning' else '12')(os.environ['application']) - notebook_config['secondary_disk_size'] = os.environ['notebook_disk_size'] - - notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled'] - if notebook_config['shared_image_enabled'] == 'false': - notebook_config['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format( - notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'], - os.environ['application']) - notebook_config['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format( - notebook_config['service_base_name'], notebook_config['endpoint_tag'], notebook_config['project_name'], + GCPMeta = dlab.meta_lib.GCPMeta() + GCPActions = dlab.actions_lib.GCPActions() + print('Generating infrastructure names and tags') + notebook_config = dict() + notebook_config['service_base_name'] = (os.environ['conf_service_base_name']) + notebook_config['edge_user_name'] = (os.environ['edge_user_name']) + notebook_config['project_name'] = (os.environ['project_name']).replace('_', '-') + notebook_config['project_tag'] = notebook_config['project_name'] + notebook_config['endpoint_name'] = os.environ['endpoint_name'].replace('_', '-') + notebook_config['endpoint_tag'] = notebook_config['endpoint_name'] + notebook_config['region'] = os.environ['gcp_region'] + notebook_config['zone'] = os.environ['gcp_zone'] + + edge_status = GCPMeta.get_instance_status('{0}-{1}-{2}-edge'.format(notebook_config['service_base_name'], + notebook_config['project_name'], + notebook_config['endpoint_tag'])) + if edge_status != 'RUNNING': + logging.info('ERROR: Edge node is unavailable! Aborting...') + print('ERROR: Edge node is unavailable! Aborting...') + ssn_hostname = GCPMeta.get_private_ip_address(notebook_config['service_base_name'] + '-ssn') + dlab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'], + ssn_hostname) + dlab.fab.append_result("Edge node is unavailable") + sys.exit(1) + + try: + if os.environ['gcp_vpc_name'] == '': + raise KeyError + else: + notebook_config['vpc_name'] = os.environ['gcp_vpc_name'] + except KeyError: + notebook_config['vpc_name'] = '{}-vpc'.format(notebook_config['service_base_name']) + try: + notebook_config['exploratory_name'] = (os.environ['exploratory_name']) + except: + notebook_config['exploratory_name'] = '' + notebook_config['subnet_name'] = '{0}-{1}-{2}-subnet'.format(notebook_config['service_base_name'], + notebook_config['project_name'], + notebook_config['endpoint_tag']) + notebook_config['instance_size'] = os.environ['gcp_notebook_instance_size'] + notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) + notebook_config['notebook_service_account_name'] = '{}-{}-{}-ps-sa'.format(notebook_config['service_base_name'], + notebook_config['project_name'], + notebook_config['endpoint_name']) + + if os.environ['conf_os_family'] == 'debian': + notebook_config['initial_user'] = 'ubuntu' + notebook_config['sudo_group'] = 'sudo' + if os.environ['conf_os_family'] == 'redhat': + notebook_config['initial_user'] = 'ec2-user' + notebook_config['sudo_group'] = 'wheel' + notebook_config['instance_name'] = '{0}-{1}-{2}-nb-{3}'.format(notebook_config['service_base_name'], + notebook_config['project_name'], + notebook_config['endpoint_name'], + notebook_config['exploratory_name']) + notebook_config['primary_disk_size'] = (lambda x: '30' if x == 'deeplearning' else '12')( os.environ['application']) - else: - notebook_config['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format( - notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application']) - notebook_config['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format( - notebook_config['service_base_name'], notebook_config['endpoint_tag'], os.environ['application']) - notebook_config['notebook_primary_image_name'] = (lambda x: os.environ['notebook_primary_image_name'] if x != 'None' - else notebook_config['expected_primary_image_name'])(str(os.environ.get('notebook_primary_image_name'))) - print('Searching pre-configured images') - notebook_config['primary_image_name'] = GCPMeta().get_image_by_name(notebook_config['expected_primary_image_name']) - if notebook_config['primary_image_name'] == '': - notebook_config['primary_image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])] - else: - print('Pre-configured primary image found. Using: {}'.format(notebook_config['primary_image_name'].get('name'))) - notebook_config['primary_image_name'] = 'global/images/{}'.format(notebook_config['primary_image_name'].get('name')) - - notebook_config['secondary_image_name'] = GCPMeta().get_image_by_name(notebook_config['expected_secondary_image_name']) - if notebook_config['secondary_image_name'] == '': - notebook_config['secondary_image_name'] = 'None' - else: - print('Pre-configured secondary image found. Using: {}'.format(notebook_config['secondary_image_name'].get('name'))) - notebook_config['secondary_image_name'] = 'global/images/{}'.format(notebook_config['secondary_image_name'].get('name')) - - notebook_config['gpu_accelerator_type'] = 'None' - - if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'): - notebook_config['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type'] - - notebook_config['network_tag'] = '{0}-{1}-ps'.format(notebook_config['service_base_name'], - notebook_config['project_name']) - - with open('/root/result.json', 'w') as f: - data = {"notebook_name": notebook_config['instance_name'], "error": ""} - json.dump(data, f) - - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - - if '@' in additional_tags['user_tag']: - notebook_config['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - notebook_config['user_tag'] = additional_tags['user_tag'] - - notebook_config['custom_tag'] = additional_tags['custom_tag'] - print('Additional tags will be added: {}'.format(additional_tags)) - notebook_config['labels'] = {"name": notebook_config['instance_name'], - "sbn": notebook_config['service_base_name'], - "project_tag": notebook_config['project_tag'], - "endpoint_tag": notebook_config['endpoint_tag'], - "user": notebook_config['user_tag'], - "product": "dlab" - } - - if notebook_config['custom_tag'] != '': - notebook_config['labels'].update({'custom_tag': notebook_config['custom_tag']}) + notebook_config['secondary_disk_size'] = os.environ['notebook_disk_size'] + + notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled'] + if notebook_config['shared_image_enabled'] == 'false': + notebook_config['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format( + notebook_config['service_base_name'], notebook_config['project_name'], notebook_config['endpoint_tag'], + os.environ['application']) + notebook_config['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format( + notebook_config['service_base_name'], notebook_config['project_name'], notebook_config['endpoint_tag'], + os.environ['application']) + else: + notebook_config['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format( + notebook_config['service_base_name'], notebook_config['endpoint_name'], os.environ['application']) + notebook_config['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format( + notebook_config['service_base_name'], notebook_config['endpoint_name'], os.environ['application']) + notebook_config['notebook_primary_image_name'] = \ + (lambda x: os.environ['notebook_primary_image_name'] if x != 'None' + else notebook_config['expected_primary_image_name'])(str(os.environ.get('notebook_primary_image_name'))) + print('Searching pre-configured images') + notebook_config['primary_image_name'] = GCPMeta.get_image_by_name( + notebook_config['expected_primary_image_name']) + if notebook_config['primary_image_name'] == '': + notebook_config['primary_image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])] + else: + print('Pre-configured primary image found. Using: {}'.format( + notebook_config['primary_image_name'].get('name'))) + notebook_config['primary_image_name'] = 'global/images/{}'.format( + notebook_config['primary_image_name'].get('name')) + + notebook_config['secondary_image_name'] = GCPMeta.get_image_by_name( + notebook_config['expected_secondary_image_name']) + if notebook_config['secondary_image_name'] == '': + notebook_config['secondary_image_name'] = 'None' + else: + print('Pre-configured secondary image found. Using: {}'.format( + notebook_config['secondary_image_name'].get('name'))) + notebook_config['secondary_image_name'] = 'global/images/{}'.format( + notebook_config['secondary_image_name'].get('name')) + + notebook_config['gpu_accelerator_type'] = 'None' + + if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'): + notebook_config['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type'] + + notebook_config['network_tag'] = '{0}-{1}-{2}-ps'.format(notebook_config['service_base_name'], + notebook_config['project_name'], + notebook_config['endpoint_name']) + + with open('/root/result.json', 'w') as f: + data = {"notebook_name": notebook_config['instance_name'], "error": ""} + json.dump(data, f) - additional_tags = \ - os.environ['tags'].replace("': u'", ": ").replace("', u'", ", ").replace("{u'", "" ).replace("'}", "") - print('Additional tags will be added: {}'.format(additional_tags)) ++ additional_tags = json.loads( ++ os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace( ++ "{u'", "{\"").replace("'}", "\"}")) ++ ++ if '@' in additional_tags['user_tag']: ++ notebook_config['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] ++ else: ++ notebook_config['user_tag'] = additional_tags['user_tag'] + ++ notebook_config['custom_tag'] = additional_tags['custom_tag'] ++ print('Additional tags will be added: {}'.format(additional_tags)) + notebook_config['labels'] = {"name": notebook_config['instance_name'], + "sbn": notebook_config['service_base_name'], + "project_tag": notebook_config['project_tag'], + "endpoint_tag": notebook_config['endpoint_tag'], - "user": notebook_config['edge_user_name'], - "product": "dlab", ++ "user": notebook_config['user_tag'], ++ "product": "dlab" + } ++ ++ if notebook_config['custom_tag'] != '': ++ notebook_config['labels'].update({'custom_tag': notebook_config['custom_tag']}) + except Exception as err: + dlab.fab.append_result("Failed to generate variables dictionary.", str(err)) + sys.exit(1) # launching instance for notebook server try: logging.info('[CREATE NOTEBOOK INSTANCE]') diff --cc infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py index 5fe8e61,17fc776..cae7fa7 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py @@@ -43,72 -42,74 +43,84 @@@ if __name__ == "__main__" level=logging.INFO, filename=local_log_filepath) try: - os.environ['exploratory_name'] - except: - os.environ['exploratory_name'] = '' - if os.path.exists('/response/.dataproc_creating_{}'.format(os.environ['exploratory_name'])): - time.sleep(30) - - print('Generating infrastructure names and tags') - dataproc_conf = dict() - try: - dataproc_conf['exploratory_name'] = (os.environ['exploratory_name']).lower().replace('_', '-') - except: - dataproc_conf['exploratory_name'] = '' - try: - dataproc_conf['computational_name'] = (os.environ['computational_name']).lower().replace('_', '-') - except: - dataproc_conf['computational_name'] = '' - dataproc_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-') - dataproc_conf['edge_user_name'] = (os.environ['edge_user_name']).lower().replace('_', '-') - dataproc_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-') - dataproc_conf['project_tag'] = (os.environ['project_name']).lower().replace('_', '-') - dataproc_conf['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-') - dataproc_conf['endpoint_name'] = (os.environ['endpoint_name']).lower().replace('_', '-') - dataproc_conf['key_name'] = os.environ['conf_key_name'] - dataproc_conf['key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) - dataproc_conf['region'] = os.environ['gcp_region'] - dataproc_conf['zone'] = os.environ['gcp_zone'] - dataproc_conf['subnet'] = '{0}-{1}-subnet'.format(dataproc_conf['service_base_name'], dataproc_conf['project_name']) - dataproc_conf['cluster_name'] = '{0}-{1}-des-{2}-{3}'.format(dataproc_conf['service_base_name'], dataproc_conf['project_name'], - dataproc_conf['exploratory_name'], dataproc_conf['computational_name']) - dataproc_conf['cluster_tag'] = '{0}-{1}-ps'.format(dataproc_conf['service_base_name'], dataproc_conf['project_name']) - dataproc_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(dataproc_conf['service_base_name'], + GCPMeta = dlab.meta_lib.GCPMeta() + GCPActions = dlab.actions_lib.GCPActions() + print('Generating infrastructure names and tags') + dataproc_conf = dict() + if 'exploratory_name' in os.environ: + dataproc_conf['exploratory_name'] = os.environ['exploratory_name'].replace('_', '-') + else: + dataproc_conf['exploratory_name'] = '' + if 'computational_name' in os.environ: + dataproc_conf['computational_name'] = os.environ['computational_name'].replace('_', '-') + else: + dataproc_conf['computational_name'] = '' + if os.path.exists('/response/.dataproc_creating_{}'.format(dataproc_conf['exploratory_name'])): + time.sleep(30) + dataproc_conf['service_base_name'] = (os.environ['conf_service_base_name']) + dataproc_conf['edge_user_name'] = (os.environ['edge_user_name']) + dataproc_conf['project_name'] = (os.environ['project_name']).replace('_', '-') + dataproc_conf['project_tag'] = dataproc_conf['project_name'] + dataproc_conf['endpoint_name'] = (os.environ['endpoint_name']).replace('_', '-') + dataproc_conf['endpoint_tag'] = dataproc_conf['endpoint_name'] + dataproc_conf['key_name'] = os.environ['conf_key_name'] + dataproc_conf['key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) + dataproc_conf['region'] = os.environ['gcp_region'] + dataproc_conf['zone'] = os.environ['gcp_zone'] + dataproc_conf['subnet'] = '{0}-{1}-{2}-subnet'.format(dataproc_conf['service_base_name'], + dataproc_conf['project_name'], + dataproc_conf['endpoint_name']) + dataproc_conf['cluster_name'] = '{0}-{1}-{2}-des-{3}'.format(dataproc_conf['service_base_name'], + dataproc_conf['project_name'], + dataproc_conf['endpoint_name'], + dataproc_conf['computational_name']) + dataproc_conf['cluster_tag'] = '{0}-{1}-{2}-ps'.format(dataproc_conf['service_base_name'], dataproc_conf['project_name'], dataproc_conf['endpoint_name']) - dataproc_conf['release_label'] = os.environ['dataproc_version'] - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - - if '@' in additional_tags['user_tag']: - dataproc_conf['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - dataproc_conf['user_tag'] = additional_tags['user_tag'] - - dataproc_conf['custom_tag'] = additional_tags['custom_tag'] - dataproc_conf['cluster_labels'] = { - os.environ['notebook_instance_name']: "not-configured", - "name": dataproc_conf['cluster_name'], - "sbn": dataproc_conf['service_base_name'], - "user": dataproc_conf['user_tag'], - "project_tag": dataproc_conf['project_tag'], - "endpoint_tag": dataproc_conf['endpoint_tag'], - "notebook_name": os.environ['notebook_instance_name'], - "product": "dlab", - "computational_name": dataproc_conf['computational_name'] - } - if dataproc_conf['custom_tag'] != '': - dataproc_conf['cluster_labels'].update({'custom_tag': dataproc_conf['custom_tag']}) - - dataproc_conf['dataproc_service_account_name'] = '{0}-{1}-ps'.format(dataproc_conf['service_base_name'], - dataproc_conf['project_name']) - dataproc_conf['dataproc_unique_index'] = GCPMeta().get_index_by_service_account_name(dataproc_conf['dataproc_service_account_name']) - service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(dataproc_conf['service_base_name'], - dataproc_conf['dataproc_unique_index'], - os.environ['gcp_project_id']) - dataproc_conf['edge_instance_hostname'] = '{0}-{1}-{2}-edge'.format(dataproc_conf['service_base_name'], - dataproc_conf['project_name'], - dataproc_conf['endpoint_name']) - dataproc_conf['dlab_ssh_user'] = os.environ['conf_os_user'] + dataproc_conf['bucket_name'] = '{0}-{1}-{2}-bucket'.format(dataproc_conf['service_base_name'], + dataproc_conf['project_name'], + dataproc_conf['endpoint_name']) + dataproc_conf['release_label'] = os.environ['dataproc_version'] ++ additional_tags = json.loads( ++ os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace( ++ "{u'", "{\"").replace("'}", "\"}")) ++ ++ if '@' in additional_tags['user_tag']: ++ dataproc_conf['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] ++ else: ++ dataproc_conf['user_tag'] = additional_tags['user_tag'] ++ ++ dataproc_conf['custom_tag'] = additional_tags['custom_tag'] + dataproc_conf['cluster_labels'] = { + os.environ['notebook_instance_name']: "not-configured", + "name": dataproc_conf['cluster_name'], + "sbn": dataproc_conf['service_base_name'], + "user": dataproc_conf['edge_user_name'], + "project_tag": dataproc_conf['project_tag'], + "endpoint_tag": dataproc_conf['endpoint_tag'], + "notebook_name": os.environ['notebook_instance_name'], + "product": "dlab", + "computational_name": dataproc_conf['computational_name'] + } ++ if dataproc_conf['custom_tag'] != '': ++ dataproc_conf['cluster_labels'].update({'custom_tag': dataproc_conf['custom_tag']}) + dataproc_conf['dataproc_service_account_name'] = '{0}-{1}-{2}-ps-sa'.format(dataproc_conf['service_base_name'], + dataproc_conf['project_name'], + dataproc_conf['endpoint_name']) + dataproc_conf['dataproc_unique_index'] = GCPMeta.get_index_by_service_account_name( + dataproc_conf['dataproc_service_account_name']) + service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(dataproc_conf['service_base_name'], + dataproc_conf['dataproc_unique_index'], + os.environ['gcp_project_id']) + dataproc_conf['edge_instance_hostname'] = '{0}-{1}-{2}-edge'.format(dataproc_conf['service_base_name'], + dataproc_conf['project_name'], + dataproc_conf['endpoint_name']) + dataproc_conf['dlab_ssh_user'] = os.environ['conf_os_user'] + except Exception as err: + dlab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err)) + sys.exit(1) - edge_status = GCPMeta().get_instance_status(dataproc_conf['edge_instance_hostname']) + edge_status = GCPMeta.get_instance_status(dataproc_conf['edge_instance_hostname']) if edge_status != 'RUNNING': logging.info('ERROR: Edge node is unavailable! Aborting...') print('ERROR: Edge node is unavailable! Aborting...') diff --cc infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py index 1e75647,4046abb..bec4c5a --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py @@@ -40,137 -38,138 +40,150 @@@ if __name__ == "__main__" logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', level=logging.DEBUG, filename=local_log_filepath) + try: + GCPMeta = dlab.meta_lib.GCPMeta() + GCPActions = dlab.actions_lib.GCPActions() + print('Generating infrastructure names and tags') + data_engine = dict() + data_engine['service_base_name'] = (os.environ['conf_service_base_name']) + data_engine['edge_user_name'] = (os.environ['edge_user_name']) + data_engine['project_name'] = (os.environ['project_name']).replace('_', '-') + data_engine['project_tag'] = data_engine['project_name'] + data_engine['endpoint_name'] = os.environ['endpoint_name'].replace('_', '-') + data_engine['endpoint_tag'] = data_engine['endpoint_name'] + data_engine['region'] = os.environ['gcp_region'] + data_engine['zone'] = os.environ['gcp_zone'] + + edge_status = GCPMeta.get_instance_status('{0}-{1}-{2}-edge'.format(data_engine['service_base_name'], + data_engine['project_name'], + data_engine['endpoint_name'])) + if edge_status != 'RUNNING': + logging.info('ERROR: Edge node is unavailable! Aborting...') + print('ERROR: Edge node is unavailable! Aborting...') + ssn_hostname = GCPMeta.get_private_ip_address(data_engine['service_base_name'] + '-ssn') + dlab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'], + ssn_hostname) + dlab.fab.append_result("Edge node is unavailable") + sys.exit(1) - print('Generating infrastructure names and tags') - data_engine = dict() - data_engine['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-') - data_engine['edge_user_name'] = (os.environ['edge_user_name']).lower().replace('_', '-') - data_engine['project_name'] = (os.environ['project_name']).lower().replace('_', '-') - data_engine['project_tag'] = (os.environ['project_name']).lower().replace('_', '-') - data_engine['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-') - data_engine['region'] = os.environ['gcp_region'] - data_engine['zone'] = os.environ['gcp_zone'] - data_engine['endpoint_name'] = os.environ['endpoint_name'] - - edge_status = GCPMeta().get_instance_status('{0}-{1}-{2}-edge'.format(data_engine['service_base_name'], - data_engine['project_name'], - data_engine['endpoint_name'])) - if edge_status != 'RUNNING': - logging.info('ERROR: Edge node is unavailable! Aborting...') - print('ERROR: Edge node is unavailable! Aborting...') - ssn_hostname = GCPMeta().get_private_ip_address(data_engine['service_base_name'] + '-ssn') - put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'], - ssn_hostname) - append_result("Edge node is unavailable") - sys.exit(1) + try: + if os.environ['gcp_vpc_name'] == '': + raise KeyError + else: + data_engine['vpc_name'] = os.environ['gcp_vpc_name'] + except KeyError: + data_engine['vpc_name'] = '{}-vpc'.format(data_engine['service_base_name']) + if 'exploratory_name' in os.environ: + data_engine['exploratory_name'] = os.environ['exploratory_name'].replace('_', '-') + else: + data_engine['exploratory_name'] = '' + if 'computational_name' in os.environ: + data_engine['computational_name'] = os.environ['computational_name'].replace('_', '-') + else: + data_engine['computational_name'] = '' + + data_engine['subnet_name'] = '{0}-{1}-{2}-subnet'.format(data_engine['service_base_name'], + data_engine['project_name'], + data_engine['endpoint_name']) + data_engine['master_size'] = os.environ['gcp_dataengine_master_size'] + data_engine['slave_size'] = os.environ['gcp_dataengine_slave_size'] + data_engine['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) + data_engine['dataengine_service_account_name'] = '{}-{}-{}-ps-sa'.format(data_engine['service_base_name'], + data_engine['project_name'], + data_engine['endpoint_name']) + + if os.environ['conf_os_family'] == 'debian': + initial_user = 'ubuntu' + sudo_group = 'sudo' + if os.environ['conf_os_family'] == 'redhat': + initial_user = 'ec2-user' + sudo_group = 'wheel' + data_engine['cluster_name'] = "{}-{}-{}-de-{}".format(data_engine['service_base_name'], + data_engine['project_name'], + data_engine['endpoint_name'], + data_engine['computational_name']) + data_engine['master_node_name'] = data_engine['cluster_name'] + '-m' + data_engine['slave_node_name'] = data_engine['cluster_name'] + '-s' + data_engine['instance_count'] = int(os.environ['dataengine_instance_count']) + data_engine['notebook_name'] = os.environ['notebook_instance_name'] + + data_engine['primary_disk_size'] = '30' + data_engine['secondary_disk_size'] = os.environ['notebook_disk_size'] + + data_engine['shared_image_enabled'] = os.environ['conf_shared_image_enabled'] + if data_engine['shared_image_enabled'] == 'false': + data_engine['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format( + data_engine['service_base_name'], data_engine['project_name'], data_engine['endpoint_tag'], + os.environ['application']) + data_engine['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format( + data_engine['service_base_name'], data_engine['project_name'], data_engine['endpoint_tag'], + os.environ['application']) + else: + data_engine['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format( + data_engine['service_base_name'], data_engine['endpoint_tag'], os.environ['application']) + data_engine['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format( + data_engine['service_base_name'], data_engine['endpoint_tag'], os.environ['application']) + data_engine['notebook_primary_image_name'] = (lambda x: os.environ['notebook_primary_image_name'] if x != 'None' + else data_engine['expected_primary_image_name'])(str(os.environ.get('notebook_primary_image_name'))) + print('Searching pre-configured images') + data_engine['primary_image_name'] = GCPMeta.get_image_by_name(data_engine['notebook_primary_image_name']) + if data_engine['primary_image_name'] == '': + data_engine['primary_image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])] + else: + print('Pre-configured primary image found. Using: {}'.format(data_engine['primary_image_name'].get('name'))) + data_engine['primary_image_name'] = 'global/images/{}'.format( + data_engine['primary_image_name'].get('name')) - try: - if os.environ['gcp_vpc_name'] == '': - raise KeyError + data_engine['secondary_image_name'] = GCPMeta.get_image_by_name(data_engine['expected_secondary_image_name']) + if data_engine['secondary_image_name'] == '': + data_engine['secondary_image_name'] = 'None' else: - data_engine['vpc_name'] = os.environ['gcp_vpc_name'] - except KeyError: - data_engine['vpc_name'] = '{}-ssn-vpc'.format(data_engine['service_base_name']) - try: - data_engine['exploratory_name'] = (os.environ['exploratory_name']).lower().replace('_', '-') - except: - data_engine['exploratory_name'] = '' - try: - data_engine['computational_name'] = os.environ['computational_name'].lower().replace('_', '-') - except: - data_engine['computational_name'] = '' - - data_engine['subnet_name'] = '{0}-{1}-subnet'.format(data_engine['service_base_name'], - data_engine['project_name']) - data_engine['master_size'] = os.environ['gcp_dataengine_master_size'] - data_engine['slave_size'] = os.environ['gcp_dataengine_slave_size'] - data_engine['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name']) - data_engine['dataengine_service_account_name'] = '{}-{}-ps'.format(data_engine['service_base_name'], - data_engine['project_name']) - - if os.environ['conf_os_family'] == 'debian': - initial_user = 'ubuntu' - sudo_group = 'sudo' - if os.environ['conf_os_family'] == 'redhat': - initial_user = 'ec2-user' - sudo_group = 'wheel' - data_engine['cluster_name'] = data_engine['service_base_name'] + '-' + data_engine['project_name'] + \ - '-de-' + data_engine['exploratory_name'] + '-' + \ - data_engine['computational_name'] - data_engine['master_node_name'] = data_engine['cluster_name'] + '-m' - data_engine['slave_node_name'] = data_engine['cluster_name'] + '-s' - data_engine['instance_count'] = int(os.environ['dataengine_instance_count']) - data_engine['notebook_name'] = os.environ['notebook_instance_name'] - - data_engine['primary_disk_size'] = '30' - data_engine['secondary_disk_size'] = os.environ['notebook_disk_size'] - - data_engine['shared_image_enabled'] = os.environ['conf_shared_image_enabled'] - if data_engine['shared_image_enabled'] == 'false': - data_engine['expected_primary_image_name'] = '{}-{}-{}-{}-primary-image'.format( - data_engine['service_base_name'], data_engine['endpoint_tag'], data_engine['project_name'], - os.environ['application']) - data_engine['expected_secondary_image_name'] = '{}-{}-{}-{}-secondary-image'.format( - data_engine['service_base_name'], data_engine['endpoint_tag'], data_engine['project_name'], - os.environ['application']) - else: - data_engine['expected_primary_image_name'] = '{}-{}-{}-primary-image'.format( - data_engine['service_base_name'], data_engine['endpoint_tag'], os.environ['application']) - data_engine['expected_secondary_image_name'] = '{}-{}-{}-secondary-image'.format( - data_engine['service_base_name'], data_engine['endpoint_tag'], os.environ['application']) - data_engine['notebook_primary_image_name'] = (lambda x: os.environ['notebook_primary_image_name'] if x != 'None' - else data_engine['expected_primary_image_name'])(str(os.environ.get('notebook_primary_image_name'))) - print('Searching pre-configured images') - data_engine['primary_image_name'] = GCPMeta().get_image_by_name(data_engine['notebook_primary_image_name']) - if data_engine['primary_image_name'] == '': - data_engine['primary_image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])] - else: - print('Pre-configured primary image found. Using: {}'.format(data_engine['primary_image_name'].get('name'))) - data_engine['primary_image_name'] = 'global/images/{}'.format( - data_engine['primary_image_name'].get('name')) - - data_engine['secondary_image_name'] = GCPMeta().get_image_by_name(data_engine['expected_secondary_image_name']) - if data_engine['secondary_image_name'] == '': - data_engine['secondary_image_name'] = 'None' - else: - print('Pre-configured secondary image found. Using: {}'.format(data_engine['secondary_image_name'].get('name'))) - data_engine['secondary_image_name'] = 'global/images/{}'.format(data_engine['secondary_image_name'].get('name')) - - with open('/root/result.json', 'w') as f: - data = {"hostname": data_engine['cluster_name'], "error": ""} - json.dump(data, f) - - data_engine['gpu_accelerator_type'] = 'None' - if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'): - data_engine['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type'] - data_engine['network_tag'] = '{0}-{1}-ps'.format(data_engine['service_base_name'], - data_engine['project_name']) - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - - if '@' in additional_tags['user_tag']: - data_engine['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - data_engine['user_tag'] = additional_tags['user_tag'] - - data_engine['custom_tag'] = additional_tags['custom_tag'] - data_engine['slave_labels'] = {"name": data_engine['cluster_name'], - "sbn": data_engine['service_base_name'], - "user": data_engine['user_tag'], - "project_tag": data_engine['project_tag'], - "endpoint_tag": data_engine['endpoint_tag'], - "type": "slave", - "notebook_name": data_engine['notebook_name'], - "product": "dlab"} - data_engine['master_labels'] = {"name": data_engine['cluster_name'], - "sbn": data_engine['service_base_name'], - "user": data_engine['user_tag'], - "project_tag": data_engine['project_tag'], - "endpoint_tag": data_engine['endpoint_tag'], - "type": "master", - "notebook_name": data_engine['notebook_name'], - "product": "dlab"} - if data_engine['custom_tag'] != '': - data_engine['slave_labels'].update({'custom_tag': data_engine['custom_tag']}) - data_engine['master_labels'].update({'custom_tag': data_engine['custom_tag']}) + print('Pre-configured secondary image found. Using: {}'.format( + data_engine['secondary_image_name'].get('name'))) + data_engine['secondary_image_name'] = 'global/images/{}'.format( + data_engine['secondary_image_name'].get('name')) + + with open('/root/result.json', 'w') as f: + data = {"hostname": data_engine['cluster_name'], "error": ""} + json.dump(data, f) + + data_engine['gpu_accelerator_type'] = 'None' + if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'): + data_engine['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type'] + data_engine['network_tag'] = '{0}-{1}-{2}-ps'.format(data_engine['service_base_name'], + data_engine['project_name'], data_engine['endpoint_name']) ++ additional_tags = json.loads( ++ os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace( ++ "{u'", "{\"").replace("'}", "\"}")) ++ ++ if '@' in additional_tags['user_tag']: ++ data_engine['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] ++ else: ++ data_engine['user_tag'] = additional_tags['user_tag'] ++ ++ data_engine['custom_tag'] = additional_tags['custom_tag'] + data_engine['slave_labels'] = {"name": data_engine['cluster_name'], + "sbn": data_engine['service_base_name'], + "user": data_engine['edge_user_name'], + "project_tag": data_engine['project_tag'], + "endpoint_tag": data_engine['endpoint_tag'], + "type": "slave", + "notebook_name": data_engine['notebook_name'], + "product": "dlab"} + data_engine['master_labels'] = {"name": data_engine['cluster_name'], + "sbn": data_engine['service_base_name'], + "user": data_engine['edge_user_name'], + "project_tag": data_engine['project_tag'], + "endpoint_tag": data_engine['endpoint_tag'], + "type": "master", + "notebook_name": data_engine['notebook_name'], + "product": "dlab"} ++ if data_engine['custom_tag'] != '': ++ data_engine['slave_labels'].update({'custom_tag': data_engine['custom_tag']}) ++ data_engine['master_labels'].update({'custom_tag': data_engine['custom_tag']}) + except Exception as err: + dlab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err)) + sys.exit(1) try: logging.info('[CREATE MASTER NODE]') diff --cc infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py index 31a7ce1,1fcbd03..9bee281 --- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py @@@ -42,36 -41,20 +42,36 @@@ if __name__ == "__main__" logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s', level=logging.DEBUG, filename=local_log_filepath) - instance = 'ssn' + + def clear_resources(): + GCPActions.remove_instance(ssn_conf['instance_name'], ssn_conf['zone']) + GCPActions.remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) + GCPActions.remove_role(ssn_conf['role_name']) - if ssn_conf['pre_defined_firewall']: ++ if not ssn_conf['pre_defined_firewall']: + GCPActions.remove_firewall('{}-ingress'.format(ssn_conf['firewall_name'])) + GCPActions.remove_firewall('{}-egress'.format(ssn_conf['firewall_name'])) - if ssn_conf['pre_defined_subnet']: ++ if not ssn_conf['pre_defined_subnet']: + GCPActions.remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_vpc']: + GCPActions.remove_vpc(ssn_conf['vpc_name']) try: + GCPMeta = dlab.meta_lib.GCPMeta() + GCPActions = dlab.actions_lib.GCPActions() logging.info('[DERIVING NAMES]') print('[DERIVING NAMES]') - pre_defined_vpc = False - pre_defined_subnet = False - pre_defined_firewall = False - billing_enabled = True - ssn_conf = dict() + ssn_conf['instance'] = 'ssn' + ssn_conf['pre_defined_vpc'] = False + ssn_conf['pre_defined_subnet'] = False + ssn_conf['pre_defined_firewall'] = False + ssn_conf['billing_enabled'] = True + ssn_conf['ssn_unique_index'] = args.ssn_unique_index - ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = replace_multi_symbols( - os.environ['conf_service_base_name'].lower().replace('_', '-')[:12], '-', True) + ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = dlab.fab.replace_multi_symbols( + os.environ['conf_service_base_name'].replace('_', '-')[:20], '-', True) + ssn_conf['instance_name'] = '{}-ssn'.format(ssn_conf['service_base_name']) + ssn_conf['role_name'] = '{}-{}-ssn-role'.format(ssn_conf['service_base_name'], ssn_conf['ssn_unique_index']) ssn_conf['region'] = os.environ['gcp_region'] ssn_conf['zone'] = os.environ['gcp_zone'] ssn_conf['default_endpoint_name'] = os.environ['default_endpoint_name'] diff --cc infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py index 8b0d7ae,cf4b045..1d765f5 --- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py @@@ -89,6 -81,7 +89,7 @@@ if __name__ == "__main__" if os.environ['gcp_vpc_name'] == '': raise KeyError else: - pre_defined_vpc = True ++ ssn_conf['pre_defined_vpc'] = True ssn_conf['vpc_name'] = os.environ['gcp_vpc_name'] except KeyError: try: @@@ -103,10 -95,11 +103,10 @@@ traceback.print_exc() raise Exception except Exception as err: - print('Error: {0}'.format(err)) - append_result("Failed to create VPC. Exception:" + str(err)) - if not pre_defined_vpc: + dlab.fab.append_result("Failed to create VPC.", str(err)) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_vpc']: try: - GCPActions().remove_vpc(ssn_conf['vpc_name']) + GCPActions.remove_vpc(ssn_conf['vpc_name']) except: print("VPC hasn't been created.") sys.exit(1) @@@ -116,6 -109,7 +116,7 @@@ if os.environ['gcp_subnet_name'] == '': raise KeyError else: - pre_defined_subnet = True ++ ssn_conf['pre_defined_subnet'] = True ssn_conf['subnet_name'] = os.environ['gcp_subnet_name'] except KeyError: try: @@@ -132,13 -125,15 +132,14 @@@ traceback.print_exc() raise Exception except Exception as err: - print('Error: {0}'.format(err)) - append_result("Failed to create Subnet.", str(err)) - if not pre_defined_subnet: + dlab.fab.append_result("Failed to create Subnet.", str(err)) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_subnet']: try: - GCPActions().remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) + GCPActions.remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) except: print("Subnet hasn't been created.") - if not pre_defined_vpc: - GCPActions().remove_vpc(ssn_conf['vpc_name']) ++ if not ssn_conf['pre_defined_vpc']: + GCPActions.remove_vpc(ssn_conf['vpc_name']) sys.exit(1) @@@ -146,6 -141,7 +147,7 @@@ if os.environ['gcp_firewall_name'] == '': raise KeyError else: - pre_defined_firewall = True ++ ssn_conf['pre_defined_firewall'] = True ssn_conf['firewall_name'] = os.environ['gcp_firewall_name'] except KeyError: try: @@@ -193,38 -188,40 +194,39 @@@ traceback.print_exc() raise Exception except Exception as err: - print('Error: {0}'.format(err)) - append_result("Failed to create Firewall.", str(err)) - if not pre_defined_subnet: - GCPActions().remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if not pre_defined_vpc: - GCPActions().remove_vpc(ssn_conf['vpc_name']) + dlab.fab.append_result("Failed to create Firewall.", str(err)) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_subnet']: + GCPActions.remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) ++ if not ssn_conf['pre_defined_vpc']: + GCPActions.remove_vpc(ssn_conf['vpc_name']) sys.exit(1) try: logging.info('[CREATE SERVICE ACCOUNT AND ROLE]') print('[CREATE SERVICE ACCOUNT AND ROLE]') - params = "--service_account_name {} --role_name {} --policy_path {} --roles_path {} --unique_index {} --service_base_name {}".format( - ssn_conf['service_account_name'], ssn_conf['role_name'], - ssn_conf['ssn_policy_path'], ssn_conf['ssn_roles_path'], ssn_conf['ssn_unique_index'], ssn_conf['service_base_name']) + params = "--service_account_name {} --role_name {} --policy_path {} --roles_path {} --unique_index {} " \ - "--service_base_name {}".format( ssn_conf['service_account_name'], ssn_conf['role_name'], - ssn_conf['ssn_policy_path'], ssn_conf['ssn_roles_path'], ++ "--service_base_name {}".format( ssn_conf['service_account_name'], ssn_conf['role_name'], ++ ssn_conf['ssn_policy_path'], ssn_conf['ssn_roles_path'], + ssn_conf['ssn_unique_index'], ssn_conf['service_base_name']) try: local("~/scripts/{}.py {}".format('common_create_service_account', params)) except: traceback.print_exc() raise Exception except Exception as err: - print('Error: {0}'.format(err)) - append_result("Unable to create Service account and role.", str(err)) + dlab.fab.append_result("Unable to create Service account and role.", str(err)) try: - GCPActions().remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) - GCPActions().remove_role(ssn_conf['role_name']) + GCPActions.remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) + GCPActions.remove_role(ssn_conf['role_name']) except: print("Service account hasn't been created") - if ssn_conf['pre_defined_firewall']: - if not pre_defined_firewall: - GCPActions().remove_firewall(ssn_conf['firewall_name'] + '-ingress') - GCPActions().remove_firewall(ssn_conf['firewall_name'] + '-egress') - if not pre_defined_subnet: - GCPActions().remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if not pre_defined_vpc: - GCPActions().remove_vpc(ssn_conf['vpc_name']) ++ if not ssn_conf['pre_defined_firewall']: + GCPActions.remove_firewall('{}-ingress'.format(ssn_conf['firewall_name'])) + GCPActions.remove_firewall('{}-egress'.format(ssn_conf['firewall_name'])) - if ssn_conf['pre_defined_subnet']: ++ if not ssn_conf['pre_defined_subnet']: + GCPActions.remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_vpc']: + GCPActions.remove_vpc(ssn_conf['vpc_name']) sys.exit(1) try: @@@ -237,22 -234,21 +239,22 @@@ traceback.print_exc() raise Exception except Exception as err: - print('Error: {0}'.format(err)) - append_result("Failed to create static ip.", str(err)) + dlab.fab.append_result("Failed to create static ip.", str(err)) try: - GCPActions().remove_static_address(ssn_conf['static_address_name'], ssn_conf['region']) + GCPActions.remove_static_address(ssn_conf['static_address_name'], ssn_conf['region']) except: print("Static IP address hasn't been created.") - GCPActions().remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) - GCPActions().remove_role(ssn_conf['role_name']) - if not pre_defined_firewall: - GCPActions().remove_firewall(ssn_conf['firewall_name'] + '-ingress') - GCPActions().remove_firewall(ssn_conf['firewall_name'] + '-egress') - if not pre_defined_subnet: - GCPActions().remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if not pre_defined_vpc: - GCPActions().remove_vpc(ssn_conf['vpc_name']) + GCPActions.remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) + GCPActions.remove_role(ssn_conf['role_name']) + GCPActions.remove_bucket(ssn_conf['ssn_bucket_name']) + GCPActions.remove_bucket(ssn_conf['shared_bucket_name']) - if ssn_conf['pre_defined_firewall']: ++ if not ssn_conf['pre_defined_firewall']: + GCPActions.remove_firewall('{}-ingress'.format(ssn_conf['firewall_name'])) + GCPActions.remove_firewall('{}-egress'.format(ssn_conf['firewall_name'])) - if ssn_conf['pre_defined_subnet']: ++ if not ssn_conf['pre_defined_subnet']: + GCPActions.remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_vpc']: + GCPActions.remove_vpc(ssn_conf['vpc_name']) sys.exit(1) if os.environ['conf_os_family'] == 'debian': @@@ -282,17 -276,16 +284,17 @@@ traceback.print_exc() raise Exception except Exception as err: - print('Error: {0}'.format(err)) - append_result("Unable to create ssn instance.", str(err)) - GCPActions().remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) - GCPActions().remove_role(ssn_conf['role_name']) - GCPActions().remove_static_address(ssn_conf['static_address_name'], ssn_conf['region']) - if not pre_defined_firewall: - GCPActions().remove_firewall(ssn_conf['firewall_name'] + '-ingress') - GCPActions().remove_firewall(ssn_conf['firewall_name'] + '-egress') - if not pre_defined_subnet: - GCPActions().remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if not pre_defined_vpc: - GCPActions().remove_vpc(ssn_conf['vpc_name']) - sys.exit(1) + dlab.fab.append_result("Unable to create ssn instance.", str(err)) + GCPActions.remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name']) + GCPActions.remove_role(ssn_conf['role_name']) + GCPActions.remove_static_address(ssn_conf['static_address_name'], ssn_conf['region']) + GCPActions.remove_bucket(ssn_conf['ssn_bucket_name']) + GCPActions.remove_bucket(ssn_conf['shared_bucket_name']) - if ssn_conf['pre_defined_firewall']: ++ if not ssn_conf['pre_defined_firewall']: + GCPActions.remove_firewall('{}-ingress'.format(ssn_conf['firewall_name'])) + GCPActions.remove_firewall('{}-egress'.format(ssn_conf['firewall_name'])) - if ssn_conf['pre_defined_subnet']: ++ if not ssn_conf['pre_defined_subnet']: + GCPActions.remove_subnet(ssn_conf['subnet_name'], ssn_conf['region']) - if ssn_conf['pre_defined_vpc']: ++ if not ssn_conf['pre_defined_vpc']: + GCPActions.remove_vpc(ssn_conf['vpc_name']) + sys.exit(1) diff --cc infrastructure-provisioning/terraform/azure/endpoint/main/network.tf index dec5d6e,520b221..738f062 --- a/infrastructure-provisioning/terraform/azure/endpoint/main/network.tf +++ b/infrastructure-provisioning/terraform/azure/endpoint/main/network.tf @@@ -93,6 -92,12 +92,12 @@@ resource "azurerm_network_interface" "e Name = local.endpoint_nif_name "${local.additional_tag[0]}" = local.additional_tag[1] "${var.tag_resource_id}" = "${var.service_base_name}:${local.endpoint_nif_name}" - "${var.service_base_name}-Tag" = local.endpoint_nif_name + "${var.service_base_name}-tag" = local.endpoint_nif_name } } + + resource "azurerm_network_interface_security_group_association" "endpoint-nif-sg" { + network_interface_id = azurerm_network_interface.endpoint-nif.id + network_security_group_id = azurerm_network_security_group.enpoint-sg.id + depends_on = [azurerm_virtual_machine.endpoint_instance] + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
