This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-1487 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit f8de1c2bfce89ddcd95cfe4179b3d14165d4c71c Author: Mykola_Bodnar1 <bodnarmyk...@gmail.com> AuthorDate: Wed Jan 29 15:00:34 2020 +0200 [DLAB-1487]: [Azure] Shared endpoint bucket should be created during project creation --- .../src/general/scripts/azure/project_prepare.py | 39 ++++++++++++ .../src/general/scripts/azure/ssn_prepare.py | 73 +--------------------- 2 files changed, 40 insertions(+), 72 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py index df4cfae..c485ff9 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py @@ -97,6 +97,14 @@ if __name__ == "__main__": "endpoint_tag": project_conf['endpoint_tag'], os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']} project_conf['primary_disk_size'] = '32' + project_conf['default_endpoint_name'] = os.environ['default_endpoint_name'] + project_conf['shared_storage_account_name'] = '{0}-{1}-shared-storage'.format(project_conf['service_base_name'], + project_conf['default_endpoint_name']) + project_conf['shared_container_name'] = '{}-shared-container'.format(project_conf['service_base_name']).lower() + project_conf['shared_storage_account_tags'] = {"Name": project_conf['shared_storage_account_name'], + "SBN": project_conf['service_base_name'], + os.environ['conf_billing_tag_key']: os.environ[ + 'conf_billing_tag_value']} # FUSE in case of absence of user's key try: @@ -672,6 +680,31 @@ if __name__ == "__main__": sys.exit(1) try: + logging.info('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]') + print('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]') + params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \ + format(project_conf['shared_container_name'], json.dumps(project_conf['shared_storage_account_tags']), + project_conf['resource_group_name'], project_conf['region']) + local("~/scripts/{}.py {}".format('common_create_storage_account', params)) + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to create storage account.", str(err)) + AzureActions().remove_subnet(project_conf['resource_group_name'], project_conf['vpc_name'], + project_conf['private_subnet_name']) + AzureActions().remove_security_group(project_conf['resource_group_name'], + project_conf['edge_security_group_name']) + AzureActions().remove_security_group(project_conf['resource_group_name'], + project_conf['notebook_security_group_name']) + AzureActions().remove_security_group(project_conf['resource_group_name'], + project_conf['master_security_group_name']) + AzureActions().remove_security_group(project_conf['resource_group_name'], + project_conf['slave_security_group_name']) + for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']): + if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]: + AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) + sys.exit(1) + + try: logging.info('[CREATE STORAGE ACCOUNT AND CONTAINERS]') print('[CREATE STORAGE ACCOUNT AND CONTAINERS]') @@ -697,6 +730,8 @@ if __name__ == "__main__": for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']): if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]: AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) + if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]: + AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) sys.exit(1) if os.environ['azure_datalake_enable'] == 'true': @@ -726,6 +761,8 @@ if __name__ == "__main__": for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']): if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]: AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) + if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]: + AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) try: for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']): if project_conf['datalake_store_name'] == datalake.tags["Name"]: @@ -777,6 +814,8 @@ if __name__ == "__main__": for storage_account in AzureMeta().list_storage_accounts(project_conf['resource_group_name']): if project_conf['edge_storage_account_name'] == storage_account.tags["Name"]: AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) + if project_conf['shared_storage_account_name'] == storage_account.tags["Name"]: + AzureActions().remove_storage_account(project_conf['resource_group_name'], storage_account.name) if os.environ['azure_datalake_enable'] == 'true': for datalake in AzureMeta().list_datalakes(project_conf['resource_group_name']): if project_conf['datalake_store_name'] == datalake.tags["Name"]: diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py index e9dff4d..bb6c793 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py @@ -62,12 +62,6 @@ if __name__ == "__main__": ssn_conf['vpc_cidr'] = os.environ['conf_vpc_cidr'] ssn_conf['subnet_prefix'] = '20' ssn_conf['ssn_image_name'] = os.environ['azure_{}_image_name'.format(os.environ['conf_os_family'])] - ssn_conf['ssn_storage_account_name'] = '{}-ssn-storage'.format(ssn_conf['service_base_name']) - ssn_conf['ssn_container_name'] = '{}-ssn-container'.format(ssn_conf['service_base_name']).lower() - ssn_conf['default_endpoint_name'] = os.environ['default_endpoint_name'] - ssn_conf['shared_storage_account_name'] = '{0}-{1}-shared-storage'.format(ssn_conf['service_base_name'], - ssn_conf['default_endpoint_name']) - ssn_conf['shared_container_name'] = '{}-shared-container'.format(ssn_conf['service_base_name']).lower() ssn_conf['datalake_store_name'] = '{}-ssn-datalake'.format(ssn_conf['service_base_name']) ssn_conf['datalake_shared_directory_name'] = '{}-shared-folder'.format(ssn_conf['service_base_name']) ssn_conf['instance_name'] = '{}-ssn'.format(ssn_conf['service_base_name']) @@ -82,12 +76,7 @@ if __name__ == "__main__": ssn_conf['instance_tags'] = {"Name": ssn_conf['instance_name'], "SBN": ssn_conf['service_base_name'], os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']} - ssn_conf['ssn_storage_account_tags'] = {"Name": ssn_conf['ssn_storage_account_name'], - "SBN": ssn_conf['service_base_name'], - os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']} - ssn_conf['shared_storage_account_tags'] = {"Name": ssn_conf['shared_storage_account_name'], - "SBN": ssn_conf['service_base_name'], - os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']} + ssn_conf['datalake_store_tags'] = {"Name": ssn_conf['datalake_store_name'], "SBN": ssn_conf['service_base_name'], os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value']} @@ -254,58 +243,6 @@ if __name__ == "__main__": append_result("Failed to create Security group. Exception: " + str(err)) sys.exit(1) - try: - logging.info('[CREATE SSN STORAGE ACCOUNT AND CONTAINER]') - print('[CREATE SSN STORAGE ACCOUNT AND CONTAINER]') - params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \ - format(ssn_conf['ssn_container_name'], json.dumps(ssn_conf['ssn_storage_account_tags']), - ssn_conf['resource_group_name'], ssn_conf['region']) - local("~/scripts/{}.py {}".format('common_create_storage_account', params)) - except Exception as err: - traceback.print_exc() - print('Error: {0}'.format(err)) - if 'azure_resource_group_name' not in os.environ: - AzureActions().remove_resource_group(ssn_conf['service_base_name'], ssn_conf['region']) - if 'azure_vpc_name' not in os.environ: - AzureActions().remove_vpc(ssn_conf['resource_group_name'], ssn_conf['vpc_name']) - if 'azure_subnet_name' not in os.environ: - AzureActions().remove_subnet(ssn_conf['resource_group_name'], ssn_conf['vpc_name'], - ssn_conf['subnet_name']) - if 'azure_security_group_name' not in os.environ: - AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name']) - for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']): - if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]: - AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name) - append_result("Failed to create SSN storage account and container. Exception:" + str(err)) - sys.exit(1) - - try: - logging.info('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]') - print('[CREATE SHARED STORAGE ACCOUNT AND CONTAINER]') - params = "--container_name {} --account_tags '{}' --resource_group_name {} --region {}". \ - format(ssn_conf['shared_container_name'], json.dumps(ssn_conf['shared_storage_account_tags']), - ssn_conf['resource_group_name'], ssn_conf['region']) - local("~/scripts/{}.py {}".format('common_create_storage_account', params)) - except Exception as err: - traceback.print_exc() - print('Error: {0}'.format(err)) - if 'azure_resource_group_name' not in os.environ: - AzureActions().remove_resource_group(ssn_conf['service_base_name'], ssn_conf['region']) - if 'azure_vpc_name' not in os.environ: - AzureActions().remove_vpc(ssn_conf['resource_group_name'], ssn_conf['vpc_name']) - if 'azure_subnet_name' not in os.environ: - AzureActions().remove_subnet(ssn_conf['resource_group_name'], ssn_conf['vpc_name'], - ssn_conf['subnet_name']) - if 'azure_security_group_name' not in os.environ: - AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name']) - for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']): - if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]: - AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name) - if ssn_conf['shared_storage_account_name'] == storage_account.tags["Name"]: - AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name) - append_result("Failed to create SSN storage account and container. Exception:" + str(err)) - sys.exit(1) - if os.environ['azure_datalake_enable'] == 'true': try: logging.info('[CREATE DATA LAKE STORE]') @@ -342,9 +279,6 @@ if __name__ == "__main__": ssn_conf['subnet_name']) if 'azure_security_group_name' not in os.environ: AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name']) - for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']): - if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]: - AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name) for datalake in AzureMeta().list_datalakes(ssn_conf['resource_group_name']): if ssn_conf['datalake_store_name'] == datalake.tags["Name"]: AzureActions().delete_datalake_store(ssn_conf['resource_group_name'], datalake.name) @@ -384,11 +318,6 @@ if __name__ == "__main__": ssn_conf['subnet_name']) if 'azure_security_group_name' not in os.environ: AzureActions().remove_security_group(ssn_conf['resource_group_name'], ssn_conf['security_group_name']) - for storage_account in AzureMeta().list_storage_accounts(ssn_conf['resource_group_name']): - if ssn_conf['ssn_storage_account_name'] == storage_account.tags["Name"]: - AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name) - if ssn_conf['shared_storage_account_name'] == storage_account.tags["Name"]: - AzureActions().remove_storage_account(ssn_conf['resource_group_name'], storage_account.name) for datalake in AzureMeta().list_datalakes(ssn_conf['resource_group_name']): if ssn_conf['datalake_store_name'] == datalake.tags["Name"]: AzureActions().delete_datalake_store(ssn_conf['resource_group_name'], datalake.name) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org