This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-1487 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit f6ad61da70ea835d3b6cd2100f93b3ea2b8b2a4c Author: Mykola_Bodnar1 <[email protected]> AuthorDate: Wed Jan 29 12:25:03 2020 +0200 [DLAB-1487]: [AWS] Shared endpoint bucket should be created during project creation --- .../general/scripts/aws/common_create_policy.py | 3 -- .../src/general/scripts/aws/project_prepare.py | 22 +++++++---- .../src/general/scripts/aws/ssn_prepare.py | 46 ---------------------- 3 files changed, 14 insertions(+), 57 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py index 3cad721..d7f5ee0 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py +++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py @@ -29,7 +29,6 @@ import boto3, botocore parser = argparse.ArgumentParser() parser.add_argument('--bucket_name', type=str, default='') -parser.add_argument('--ssn_bucket_name', type=str, default='') parser.add_argument('--shared_bucket_name', type=str, default='') parser.add_argument('--service_base_name', type=str, default='') parser.add_argument('--username', type=str, default='') @@ -46,8 +45,6 @@ if __name__ == "__main__": handler = open('/root/templates/edge_s3_policy.json', 'r') policy = handler.read() policy = policy.replace('BUCKET_NAME', args.bucket_name) - # Removed for multiple Endpoints per project - # policy = policy.replace('SSN_BUCK', args.ssn_bucket_name) policy = policy.replace('SHARED_BUCK', args.shared_bucket_name) if args.region == 'cn-north-1': policy = policy.replace('aws', 'aws-cn') diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py index 8de0370..019eace 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py @@ -61,10 +61,9 @@ if __name__ == "__main__": project_conf['project_name'], project_conf['endpoint_name']) project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-') - project_conf['ssn_bucket_name'] = '{}-ssn-bucket'.format( - project_conf['service_base_name']).lower().replace('_', '-') - project_conf['shared_bucket_name'] = '{0}-{1}-shared-bucket'.format( - project_conf['service_base_name'], project_conf['endpoint_tag']).lower().replace('_', '-') + project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format( + project_conf['service_base_name'], project_conf['endpoint_tag']) + project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-') project_conf['edge_role_name'] = '{}-{}-edge-Role'.format( project_conf['service_base_name'].lower().replace('-', '_'), os.environ['project_name']) project_conf['edge_role_profile_name'] = '{}-{}-edge-Profile'.format( @@ -521,7 +520,7 @@ if __name__ == "__main__": raise Exception except Exception as err: print('Error: {0}'.format(err)) - append_result("Failed to create bucket.", str(err)) + append_result("Failed to create security group.", str(err)) remove_all_iam_resources('notebook', os.environ['project_name']) remove_all_iam_resources('edge', os.environ['project_name']) remove_sgroups(project_conf['dataengine_instances_name']) @@ -532,6 +531,14 @@ if __name__ == "__main__": try: logging.info('[CREATE BUCKETS]') print('[CREATE BUCKETS]') + params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \ + format(project_conf['shared_bucket_name'], project_conf['tag_name'], project_conf['shared_bucket_name'], project_conf['region'], project_conf['shared_bucket_name_tag']) + try: + local("~/scripts/{}.py {}".format('common_create_bucket', params)) + except: + traceback.print_exc() + raise Exception + params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}" \ .format(project_conf['bucket_name'], project_conf['tag_name'], project_conf['bucket_name'], project_conf['region'], project_conf['bucket_name_tag']) @@ -542,7 +549,7 @@ if __name__ == "__main__": raise Exception except Exception as err: print('Error: {0}'.format(err)) - append_result("Failed to create bucket.", str(err)) + append_result("Failed to create buckets.", str(err)) remove_all_iam_resources('notebook', os.environ['project_name']) remove_all_iam_resources('edge', os.environ['project_name']) remove_sgroups(project_conf['dataengine_instances_name']) @@ -553,10 +560,9 @@ if __name__ == "__main__": try: logging.info('[CREATING BUCKET POLICY FOR PROJECT INSTANCES]') print('[CREATING BUCKET POLICY FOR USER INSTANCES]') - params = '--bucket_name {} --ssn_bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} ' \ + params = '--bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} ' \ '--notebook_role_name {} --service_base_name {} --region {} ' \ '--user_predefined_s3_policies "{}"'.format(project_conf['bucket_name'], - project_conf['ssn_bucket_name'], project_conf['shared_bucket_name'], os.environ['project_name'], project_conf['edge_role_name'], project_conf['notebook_dataengine_role_name'], diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py index 9acf264..8615a25 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py @@ -61,11 +61,7 @@ if __name__ == "__main__": role_name = service_base_name.lower().replace('-', '_') + '-ssn-Role' role_profile_name = service_base_name.lower().replace('-', '_') + '-ssn-Profile' policy_name = service_base_name.lower().replace('-', '_') + '-ssn-Policy' - ssn_bucket_name_tag = service_base_name + '-ssn-bucket' default_endpoint_name = os.environ['default_endpoint_name'] - shared_bucket_name_tag = '{0}-{1}-shared-bucket'.format(service_base_name, default_endpoint_name) - ssn_bucket_name = ssn_bucket_name_tag.lower().replace('_', '-') - shared_bucket_name = shared_bucket_name_tag.lower().replace('_', '-') tag_name = service_base_name + '-Tag' tag2_name = service_base_name + '-secondary-Tag' user_tag = "{0}:{0}-ssn-Role".format(service_base_name) @@ -386,48 +382,6 @@ if __name__ == "__main__": remove_route_tables(tag2_name, True) remove_vpc(os.environ['aws_vpc2_id']) sys.exit(1) - try: - logging.info('[CREATE BUCKETS]') - print('[CREATE BUCKETS]') - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \ - format(ssn_bucket_name, tag_name, ssn_bucket_name, region, ssn_bucket_name_tag) - - try: - local("~/scripts/{}.py {}".format('common_create_bucket', params)) - except: - traceback.print_exc() - raise Exception - - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \ - format(shared_bucket_name, tag_name, shared_bucket_name, region, shared_bucket_name_tag) - - try: - local("~/scripts/{}.py {}".format('common_create_bucket', params)) - except: - traceback.print_exc() - raise Exception - except Exception as err: - print('Error: {0}'.format(err)) - append_result("Unable to create bucket.", str(err)) - remove_all_iam_resources(instance) - if pre_defined_sg: - remove_sgroups(tag_name) - if pre_defined_subnet: - remove_internet_gateways(os.environ['aws_vpc_id'], tag_name, service_base_name) - remove_subnets(service_base_name + "-subnet") - if pre_defined_vpc: - remove_vpc_endpoints(os.environ['aws_vpc_id']) - remove_route_tables(tag_name, True) - remove_vpc(os.environ['aws_vpc_id']) - if pre_defined_vpc2: - remove_peering('*') - try: - remove_vpc_endpoints(os.environ['aws_vpc2_id']) - except: - print("There are no VPC Endpoints") - remove_route_tables(tag2_name, True) - remove_vpc(os.environ['aws_vpc2_id']) - sys.exit(1) try: logging.info('[CREATE SSN INSTANCE]') --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
