This is an automated email from the ASF dual-hosted git repository. omartushevskyi pushed a commit to branch DLAB-518 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit dbbc91b69248d11284baf2e8aab38f4669b9cf0f Author: Oleh Martushevskyi <[email protected]> AuthorDate: Mon Mar 18 14:03:50 2019 +0200 Fixed issue with S3 bucket tagging --- .../src/general/lib/aws/actions_lib.py | 298 ++++++++++++++------- .../general/scripts/aws/common_create_bucket.py | 3 +- .../src/general/scripts/aws/edge_prepare.py | 59 ++-- .../src/general/scripts/aws/ssn_configure.py | 10 +- .../src/general/scripts/aws/ssn_prepare.py | 38 ++- 5 files changed, 262 insertions(+), 146 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py index 2209bee..284c744 100644 --- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py @@ -40,6 +40,7 @@ import dlab.fab import uuid import ast + def backoff_log(err): logging.info("Unable to create Tag: " + \ str(err) + "\n Traceback: " + \ @@ -57,32 +58,35 @@ def put_to_bucket(bucket_name, local_file, destination_file): s3.upload_fileobj(data, bucket_name, destination_file, ExtraArgs={'ServerSideEncryption': 'AES256'}) return True except Exception as err: - logging.info("Unable to upload files to S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to upload files to S3 bucket", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to upload files to S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to upload files to S3 bucket", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) return False -def create_s3_bucket(bucket_name, tag, region): +def create_s3_bucket(bucket_name, tag, region, bucket_name_tag): try: s3 = boto3.resource('s3', config=Config(signature_version='s3v4')) if region == "us-east-1": bucket = s3.create_bucket(Bucket=bucket_name) else: bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': region}) - boto3.client('s3', config=Config(signature_version='s3v4')).put_bucket_encryption(Bucket=bucket_name, ServerSideEncryptionConfiguration={ - 'Rules': [ - { - 'ApplyServerSideEncryptionByDefault': { - 'SSEAlgorithm': 'AES256' - } - }, - ] - }) + boto3.client('s3', config=Config(signature_version='s3v4')).put_bucket_encryption( + Bucket=bucket_name, ServerSideEncryptionConfiguration={ + 'Rules': [ + { + 'ApplyServerSideEncryptionByDefault': { + 'SSEAlgorithm': 'AES256' + } + }, + ] + }) tags = list() tags.append(tag) - tags.append({'Key': os.environ['conf_tag_resource_id'], 'Value': os.environ['conf_service_base_name'] + ':' + - bucket_name}) + tags.append({'Key': os.environ['conf_tag_resource_id'], + 'Value': os.environ['conf_service_base_name'] + ':' + bucket_name_tag}) tags.append({'Key': os.environ['conf_billing_tag_key'], 'Value': os.environ['conf_billing_tag_value']}) if 'conf_additional_tags' in os.environ: for tag in os.environ['conf_additional_tags'].split(';'): @@ -97,8 +101,10 @@ def create_s3_bucket(bucket_name, tag, region): tagging.reload() return bucket.name except Exception as err: - logging.info("Unable to create S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create S3 bucket", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to create S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to create S3 bucket", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -110,7 +116,8 @@ def create_vpc(vpc_cidr, tag): return vpc.id except Exception as err: logging.info("Unable to create VPC: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create VPC", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to create VPC", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -120,8 +127,10 @@ def enable_vpc_dns(vpc_id): client.modify_vpc_attribute(VpcId=vpc_id, EnableDnsHostnames={'Value': True}) except Exception as err: - logging.info("Unable to modify VPC attributes: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to modify VPC attributes", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to modify VPC attributes: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to modify VPC attributes", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -132,7 +141,8 @@ def remove_vpc(vpc_id): print("VPC {} has been removed".format(vpc_id)) except Exception as err: logging.info("Unable to remove VPC: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove VPC", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to remove VPC", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -202,7 +212,8 @@ def remove_emr_tag(emr_id, tag): emr.remove_tags(ResourceId=emr_id, TagKeys=tag) except Exception as err: logging.info("Unable to remove Tag: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove Tag", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to remove Tag", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -227,7 +238,8 @@ def create_rt(vpc_id, infra_tag_name, infra_tag_value, secondary): return rt_id except Exception as err: logging.info("Unable to create Route Table: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create Route Table", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to create Route Table", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -240,7 +252,8 @@ def create_subnet(vpc_id, subnet, tag): return subnet.id except Exception as err: logging.info("Unable to create Subnet: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create Subnet", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to create Subnet", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -251,7 +264,8 @@ def create_security_group(security_group_name, vpc_id, security_group_rules, egr time.sleep(10) create_tag(group.id, tag) try: - group.revoke_egress(IpPermissions=[{"IpProtocol": "-1", "IpRanges": [{"CidrIp": "0.0.0.0/0"}], "UserIdGroupPairs": [], "PrefixListIds": []}]) + group.revoke_egress(IpPermissions=[{"IpProtocol": "-1", "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + "UserIdGroupPairs": [], "PrefixListIds": []}]) except: print("Mentioned rule does not exist") for rule in security_group_rules: @@ -261,9 +275,11 @@ def create_security_group(security_group_name, vpc_id, security_group_rules, egr return group.id except Exception as err: logging.info("Unable to create security group: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create security group", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to create security group", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) + def create_route_by_id(subnet_id, vpc_id, peering_id, another_cidr): client = boto3.client('ec2') try: @@ -293,13 +309,16 @@ def create_route_by_id(subnet_id, vpc_id, peering_id, another_cidr): "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) + def create_peer_routes(peering_id, service_base_name): client = boto3.client('ec2') try: route_tables = client.describe_route_tables( - Filters=[{'Name': 'tag:{}-Tag'.format(service_base_name), 'Values': ['{}'.format(service_base_name)]}]).get('RouteTables') + Filters=[{'Name': 'tag:{}-Tag'.format(service_base_name), 'Values': ['{}'.format( + service_base_name)]}]).get('RouteTables') route_tables2 = client.describe_route_tables(Filters=[ - {'Name': 'tag:{}-secondary-Tag'.format(service_base_name), 'Values': ['{}'.format(service_base_name)]}]).get('RouteTables') + {'Name': 'tag:{}-secondary-Tag'.format(service_base_name), 'Values': ['{}'.format( + service_base_name)]}]).get('RouteTables') for table in route_tables: routes = table.get('Routes') routeExists=False @@ -347,8 +366,10 @@ def create_peering_connection(vpc_id, vpc2_id, service_base_name): create_tag(peering.id, json.dumps(tag)) return peering.id except Exception as err: - logging.info("Unable to create peering connection: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create peering connection", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to create peering connection: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to create peering connection", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -358,7 +379,8 @@ def enable_auto_assign_ip(subnet_id): client.modify_subnet_attribute(MapPublicIpOnLaunch={'Value': True}, SubnetId=subnet_id) except Exception as err: logging.info("Unable to create Subnet: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create Subnet", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to create Subnet", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -436,7 +458,8 @@ def create_instance(definitions, instance_tag, primary_disk_size=12): return '' except Exception as err: logging.info("Unable to create EC2: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create EC2", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to create EC2", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) def tag_intance_volume(instance_id, node_name, instance_tag): @@ -489,16 +512,25 @@ def create_iam_role(role_name, role_profile, region, service='ec2'): conn = boto3.client('iam') try: if region == 'cn-north-1': - conn.create_role(RoleName=role_name, - AssumeRolePolicyDocument='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["' + service + '.amazonaws.com.cn"]},"Action":["sts:AssumeRole"]}]}') + conn.create_role( + RoleName=role_name, + AssumeRolePolicyDocument= + '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["' + service + + '.amazonaws.com.cn"]},"Action":["sts:AssumeRole"]}]}') else: - conn.create_role(RoleName=role_name, AssumeRolePolicyDocument='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["' + service + '.amazonaws.com"]},"Action":["sts:AssumeRole"]}]}') + conn.create_role( + RoleName=role_name, AssumeRolePolicyDocument= + '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["' + service + + '.amazonaws.com"]},"Action":["sts:AssumeRole"]}]}') except botocore.exceptions.ClientError as e_role: if e_role.response['Error']['Code'] == 'EntityAlreadyExists': print("IAM role already exists. Reusing...") else: - logging.info("Unable to create IAM role: " + str(e_role.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create IAM role", "error_message": str(e_role.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to create IAM role: " + str(e_role.response['Error']['Message']) + + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) + append_result(str({"error": "Unable to create IAM role", + "error_message": str(e_role.response['Error']['Message']) + "\n Traceback: " + + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) return if service == 'ec2': @@ -510,16 +542,22 @@ def create_iam_role(role_name, role_profile, region, service='ec2'): if e_profile.response['Error']['Code'] == 'EntityAlreadyExists': print("Instance profile already exists. Reusing...") else: - logging.info("Unable to create Instance Profile: " + str(e_profile.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to create Instance Profile", "error_message": str(e_profile.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to create Instance Profile: " + str(e_profile.response['Error']['Message']) + + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) + append_result(str({"error": "Unable to create Instance Profile", + "error_message": str(e_profile.response['Error']['Message']) + "\n Traceback: " + + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) return try: conn.add_role_to_instance_profile(InstanceProfileName=role_profile, RoleName=role_name) time.sleep(30) except botocore.exceptions.ClientError as err: - logging.info("Unable to add IAM role to instance profile: " + str(err.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to add IAM role to instance profile", "error_message": str(err.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to add IAM role to instance profile: " + str(err.response['Error']['Message']) + + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) + append_result(str({"error": "Unable to add IAM role to instance profile", + "error_message": str(err.response['Error']['Message']) + "\n Traceback: " + + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -529,8 +567,11 @@ def attach_policy(role_name, policy_arn): conn.attach_role_policy(PolicyArn=policy_arn, RoleName=role_name) time.sleep(30) except botocore.exceptions.ClientError as err: - logging.info("Unable to attach Policy: " + str(err.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to attach Policy", "error_message": str(err.response['Error']['Message']) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to attach Policy: " + str(err.response['Error']['Message']) + "\n Traceback: " + + traceback.print_exc(file=sys.stdout)) + append_result(str({"error": "Unable to attach Policy", + "error_message": str(err.response['Error']['Message']) + "\n Traceback: " + + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -542,7 +583,8 @@ def create_attach_policy(policy_name, role_name, file_path): conn.put_role_policy(RoleName=role_name, PolicyName=policy_name, PolicyDocument=json_file) except Exception as err: logging.info("Unable to attach Policy: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to attach Policy", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to attach Policy", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -606,7 +648,8 @@ def remove_route_53_record(hosted_zone_id, hosted_zone_name, subdomain): ) else: route53_client = boto3.client('route53') - for record_set in route53_client.list_resource_record_sets(HostedZoneId=hosted_zone_id).get('ResourceRecordSets'): + for record_set in route53_client.list_resource_record_sets( + HostedZoneId=hosted_zone_id).get('ResourceRecordSets'): if record_set['Name'] == "{}.{}.".format(subdomain, hosted_zone_name): for record in record_set['ResourceRecords']: route53_client.change_resource_record_sets( @@ -725,7 +768,8 @@ def remove_ec2(tag_name, tag_value): print("There are no instances with '{}' tag to terminate".format(tag_name)) except Exception as err: logging.info("Unable to remove EC2: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to EC2", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to EC2", "error_message": str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -749,7 +793,8 @@ def stop_ec2(tag_name, tag_value): print("There are no instances with {} name to stop".format(tag_value)) except Exception as err: logging.info("Unable to stop EC2: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to stop EC2", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to stop EC2", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -773,7 +818,8 @@ def start_ec2(tag_name, tag_value): print("There are no instances with {} name to start".format(tag_value)) except Exception as err: logging.info("Unable to start EC2: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to start EC2", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to start EC2", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -790,7 +836,8 @@ def remove_detach_iam_policies(role_name, action=''): client.delete_policy(PolicyArn=policy_arn) print("The IAM policy {} has been deleted successfully".format(policy_arn)) except Exception as err: - logging.info("Unable to remove/detach IAM policy: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) + logging.info("Unable to remove/detach IAM policy: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) append_result(str({"error": "Unable to remove/detach IAM policy", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -802,9 +849,11 @@ def remove_roles_and_profiles(role_name, role_profile_name): client.remove_role_from_instance_profile(InstanceProfileName=role_profile_name, RoleName=role_name) client.delete_instance_profile(InstanceProfileName=role_profile_name) client.delete_role(RoleName=role_name) - print("The IAM role {0} and instance profile {1} have been deleted successfully".format(role_name, role_profile_name)) + print("The IAM role {0} and instance profile {1} have been deleted successfully".format(role_name, + role_profile_name)) except Exception as err: - logging.info("Unable to remove IAM role/profile: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) + logging.info("Unable to remove IAM role/profile: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) append_result(str({"error": "Unable to remove IAM role/profile", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -839,7 +888,8 @@ def remove_all_iam_resources(instance_type, scientist=''): if '-edge-Role' in iam_role: if instance_type == 'edge' and scientist in iam_role: remove_detach_iam_policies(iam_role, 'delete') - role_profile_name = os.environ['conf_service_base_name'] + '-' + '{}'.format(scientist) + '-edge-Profile' + role_profile_name = os.environ['conf_service_base_name'] + '-' + '{}'.format(scientist) + \ + '-edge-Profile' try: client.get_instance_profile(InstanceProfileName=role_profile_name) remove_roles_and_profiles(iam_role, role_profile_name) @@ -849,7 +899,8 @@ def remove_all_iam_resources(instance_type, scientist=''): print("The IAM role {} has been deleted successfully".format(iam_role)) if instance_type == 'all': remove_detach_iam_policies(iam_role, 'delete') - role_profile_name = client.list_instance_profiles_for_role(RoleName=iam_role).get('InstanceProfiles') + role_profile_name = client.list_instance_profiles_for_role( + RoleName=iam_role).get('InstanceProfiles') if role_profile_name: for i in role_profile_name: role_profile_name = i.get('InstanceProfileName') @@ -861,7 +912,8 @@ def remove_all_iam_resources(instance_type, scientist=''): if '-nb-de-Role' in iam_role: if instance_type == 'notebook' and scientist in iam_role: remove_detach_iam_policies(iam_role) - role_profile_name = os.environ['conf_service_base_name'] + '-' + "{}".format(scientist) + '-nb-de-Profile' + role_profile_name = os.environ['conf_service_base_name'] + '-' + "{}".format(scientist) + \ + '-nb-de-Profile' try: client.get_instance_profile(InstanceProfileName=role_profile_name) remove_roles_and_profiles(iam_role, role_profile_name) @@ -871,7 +923,8 @@ def remove_all_iam_resources(instance_type, scientist=''): print("The IAM role {} has been deleted successfully".format(iam_role)) if instance_type == 'all': remove_detach_iam_policies(iam_role) - role_profile_name = client.list_instance_profiles_for_role(RoleName=iam_role).get('InstanceProfiles') + role_profile_name = client.list_instance_profiles_for_role( + RoleName=iam_role).get('InstanceProfiles') if role_profile_name: for i in role_profile_name: role_profile_name = i.get('InstanceProfileName') @@ -908,8 +961,10 @@ def remove_all_iam_resources(instance_type, scientist=''): else: print("There are no instance profiles to delete") except Exception as err: - logging.info("Unable to remove some of the IAM resources: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove some of the IAM resources", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to remove some of the IAM resources: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to remove some of the IAM resources", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -928,7 +983,8 @@ def s3_cleanup(bucket, cluster_name, user_name): s3_res.Object(resource.name, i.key).delete() except Exception as err: logging.info("Unable to clean S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to clean S3 bucket", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to clean S3 bucket", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -941,7 +997,8 @@ def remove_s3(bucket_type='all', scientist=''): bucket_name = (os.environ['conf_service_base_name'] + '-ssn-bucket').lower().replace('_', '-') bucket_list.append((os.environ['conf_service_base_name'] + '-shared-bucket').lower().replace('_', '-')) elif bucket_type == 'edge': - bucket_name = (os.environ['conf_service_base_name'] + '-' + "{}".format(scientist) + '-bucket').lower().replace('_', '-') + bucket_name = (os.environ['conf_service_base_name'] + '-' + "{}".format(scientist) + + '-bucket').lower().replace('_', '-') else: bucket_name = (os.environ['conf_service_base_name']).lower().replace('_', '-') for item in client.list_buckets().get('Buckets'): @@ -960,8 +1017,10 @@ def remove_s3(bucket_type='all', scientist=''): else: print("There are no buckets to delete") except Exception as err: - logging.info("Unable to remove S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove S3 bucket", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to remove S3 bucket: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to remove S3 bucket", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -988,7 +1047,8 @@ def remove_subnets(tag_value): print("There are no private subnets to delete") except Exception as err: logging.info("Unable to remove subnet: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove subnet", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to remove subnet", "error_message": str(err) + "\n Traceback: " + + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -997,8 +1057,11 @@ def remove_peering(tag_value): client = boto3.client('ec2') tag_name = os.environ['conf_service_base_name'] + '-Tag' if os.environ['conf_duo_vpc_enable']=='true': - peering_id = client.describe_vpc_peering_connections(Filters=[{'Name': 'tag-key', 'Values': [tag_name]}, {'Name': 'tag-value', 'Values': [tag_value]}, - {'Name': 'status-code', 'Values': ['active']}]).get('VpcPeeringConnections')[0].get('VpcPeeringConnectionId') + peering_id = client.describe_vpc_peering_connections(Filters=[ + {'Name': 'tag-key', 'Values': [tag_name]}, + {'Name': 'tag-value', 'Values': [tag_value]}, + {'Name': 'status-code', 'Values': + ['active']}]).get('VpcPeeringConnections')[0].get('VpcPeeringConnectionId') if peering_id: client.delete_vpc_peering_connection(VpcPeeringConnectionId=peering_id) print("Peering connection {} has been deleted successfully".format(peering_id)) @@ -1007,8 +1070,10 @@ def remove_peering(tag_value): else: print("There are no peering connections to delete because duo vpc option is disabled") except Exception as err: - logging.info("Unable to remove peering connection: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove peering connection", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to remove peering connection: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to remove peering connection", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) def remove_sgroups(tag_value): @@ -1026,7 +1091,8 @@ def remove_sgroups(tag_value): print("There are no security groups to delete") except Exception as err: logging.info("Unable to remove SG: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove SG", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to remove SG", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -1042,8 +1108,10 @@ def add_inbound_sg_rule(sg_id, rule): print("The following inbound rule is already exist:") print(str(rule)) else: - logging.info("Unable to add inbound rule to SG: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to add inbound rule to SG", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to add inbound rule to SG: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to add inbound rule to SG", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -1059,8 +1127,10 @@ def add_outbound_sg_rule(sg_id, rule): print("The following outbound rule is already exist:") print(str(rule)) else: - logging.info("Unable to add outbound rule to SG: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to add outbound rule to SG", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to add outbound rule to SG: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to add outbound rule to SG", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -1078,8 +1148,10 @@ def deregister_image(image_name='*'): client.delete_snapshot(SnapshotId=device.get('Ebs').get('SnapshotId')) print("Notebook AMI {} has been deregistered successfully".format(image.id)) except Exception as err: - logging.info("Unable to de-register image: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to de-register image", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to de-register image: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to de-register image", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -1093,7 +1165,8 @@ def terminate_emr(id): waiter.wait(ClusterId=id) except Exception as err: logging.info("Unable to remove EMR: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove EMR", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + append_result(str({"error": "Unable to remove EMR", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -1115,15 +1188,17 @@ def remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path, emr_ver if exists('/home/{}/.ensure_dir/dataengine-service_{}_interpreter_ensured'.format(ssh_user, emr_name)): if os.environ['notebook_multiple_clusters'] == 'true': try: - livy_port = sudo("cat /opt/" + emr_version + "/" + emr_name - + "/livy/conf/livy.conf | grep livy.server.port | tail -n 1 | awk '{printf $3}'") + livy_port = sudo("cat /opt/" + emr_version + "/" + emr_name + + "/livy/conf/livy.conf | grep livy.server.port | tail -n 1 | " + "awk '{printf $3}'") process_number = sudo("netstat -natp 2>/dev/null | grep ':" + livy_port + "' | awk '{print $7}' | sed 's|/.*||g'") sudo('kill -9 ' + process_number) sudo('systemctl disable livy-server-' + livy_port) except: print("Wasn't able to find Livy server for this EMR!") - sudo('sed -i \"s/^export SPARK_HOME.*/export SPARK_HOME=\/opt\/spark/\" /opt/zeppelin/conf/zeppelin-env.sh') + sudo('sed -i \"s/^export SPARK_HOME.*/export SPARK_HOME=\/opt\/spark/\" ' + '/opt/zeppelin/conf/zeppelin-env.sh') sudo("rm -rf /home/{}/.ensure_dir/dataengine-service_interpreter_ensure".format(ssh_user)) zeppelin_url = 'http://' + private + ':8080/api/interpreter/setting/' opener = urllib2.build_opener(urllib2.ProxyHandler({})) @@ -1151,7 +1226,8 @@ def remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path, emr_ver if result == '1': zeppelin_restarted = True sudo('sleep 5') - sudo('rm -rf /home/{}/.ensure_dir/dataengine-service_{}_interpreter_ensured'.format(ssh_user, emr_name)) + sudo('rm -rf /home/{}/.ensure_dir/dataengine-service_{}_interpreter_ensured'.format(ssh_user, + emr_name)) if exists('/home/{}/.ensure_dir/rstudio_dataengine-service_ensured'.format(ssh_user)): dlab.fab.remove_rstudio_dataengines_kernel(computational_name, ssh_user) sudo('rm -rf /opt/' + emr_version + '/' + emr_name + '/') @@ -1159,8 +1235,10 @@ def remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path, emr_ver else: print("There are no notebooks to clean kernels.") except Exception as err: - logging.info("Unable to remove kernels on Notebook: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)) - append_result(str({"error": "Unable to remove kernels on Notebook", "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) + logging.info("Unable to remove kernels on Notebook: " + str(err) + "\n Traceback: " + traceback.print_exc( + file=sys.stdout)) + append_result(str({"error": "Unable to remove kernels on Notebook", + "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)})) traceback.print_exc(file=sys.stdout) @@ -1392,22 +1470,24 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p if region == 'cn-north-1': try: local(venv_command + ' && sudo -i ' + pip_command + - ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 -U pip==9.0.3 --no-cache-dir'.format(pip_mirror)) + ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 -U pip==9.0.3 ' + '--no-cache-dir'.format(pip_mirror)) local(venv_command + ' && sudo -i ' + pip_command + ' install pyzmq==17.0.0') local(venv_command + ' && sudo -i ' + pip_command + - ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 ipython ipykernel --no-cache-dir'. - format(pip_mirror)) + ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 ipython ipykernel ' + '--no-cache-dir'.format(pip_mirror)) local(venv_command + ' && sudo -i ' + pip_command + - ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 boto boto3 NumPy=={1} SciPy Matplotlib==2.0.2 pandas Sympy Pillow sklearn --no-cache-dir'. - format(pip_mirror, numpy_version)) + ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 boto boto3 NumPy=={1} SciPy ' + 'Matplotlib==2.0.2 pandas Sympy Pillow sklearn --no-cache-dir'.format(pip_mirror, numpy_version)) # Need to refactor when we add GPU cluster if application == 'deeplearning': local(venv_command + ' && sudo -i ' + pip_command + - ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 mxnet-cu80 opencv-python keras Theano --no-cache-dir'.format(pip_mirror)) + ' install -i https://{0}/simple --trusted-host {0} --timeout 60000 mxnet-cu80 opencv-python ' + 'keras Theano --no-cache-dir'.format(pip_mirror)) python_without_dots = python_version.replace('.', '') local(venv_command + ' && sudo -i ' + pip_command + - ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl --no-cache-dir'. - format(python_without_dots[:2])) + ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl ' + '--no-cache-dir'.format(python_without_dots[:2])) local('sudo rm /etc/pip.conf') local('sudo mv /etc/back_pip.conf /etc/pip.conf') except: @@ -1420,15 +1500,16 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p local(venv_command + ' && sudo -i ' + pip_command + ' install pyzmq==17.0.0') local(venv_command + ' && sudo -i ' + pip_command + ' install ipython ipykernel --no-cache-dir') local(venv_command + ' && sudo -i ' + pip_command + - ' install boto boto3 NumPy=={} SciPy Matplotlib==2.0.2 pandas Sympy Pillow sklearn --no-cache-dir'.format(numpy_version)) + ' install boto boto3 NumPy=={} SciPy Matplotlib==2.0.2 pandas Sympy Pillow sklearn ' + '--no-cache-dir'.format(numpy_version)) # Need to refactor when we add GPU cluster if application == 'deeplearning': local(venv_command + ' && sudo -i ' + pip_command + ' install mxnet-cu80 opencv-python keras Theano --no-cache-dir') python_without_dots = python_version.replace('.', '') local(venv_command + ' && sudo -i ' + pip_command + - ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl --no-cache-dir'. - format(python_without_dots[:2])) + ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl ' + '--no-cache-dir'.format(python_without_dots[:2])) local('sudo rm -rf /usr/bin/python' + python_version[0:3]) local('sudo ln -fs /opt/python/python' + python_version + '/bin/python' + python_version[0:3] + ' /usr/bin/python' + python_version[0:3]) @@ -1440,7 +1521,8 @@ def spark_defaults(args): local(""" sudo bash -c " sed -i '/""" + i + """/d' """ + spark_def_path + """ " """) local(""" sudo bash -c " sed -i '/#/d' """ + spark_def_path + """ " """) local(""" sudo bash -c " sed -i '/^\s*$/d' """ + spark_def_path + """ " """) - local(""" sudo bash -c "sed -i '/spark.driver.extraClassPath/,/spark.driver.extraLibraryPath/s|/usr|/opt/DATAENGINE-SERVICE_VERSION/jars/usr|g' """ + spark_def_path + """ " """) + local(""" sudo bash -c "sed -i '/spark.driver.extraClassPath/,/spark.driver.extraLibraryPath/s|""" + """/usr|/opt/DATAENGINE-SERVICE_VERSION/jars/usr|g' """ + spark_def_path + """ " """) local(""" sudo bash -c "sed -i '/spark.yarn.dist.files/s/\/etc\/spark\/conf/\/opt\/DATAENGINE-SERVICE_VERSION\/CLUSTER\/conf/g' """ + spark_def_path + """ " """) template_file = spark_def_path @@ -1456,7 +1538,8 @@ def spark_defaults(args): endpoint_url = "https://s3.{}.amazonaws.com.cn".format(args.region) else: endpoint_url = 'https://s3-' + args.region + '.amazonaws.com' - local("""bash -c 'echo "spark.hadoop.fs.s3a.endpoint """ + endpoint_url + """ " >> """ + spark_def_path + """'""") + local("""bash -c 'echo "spark.hadoop.fs.s3a.endpoint """ + endpoint_url + """ " >> """ + + spark_def_path + """'""") local('echo "spark.hadoop.fs.s3a.server-side-encryption-algorithm AES256" >> {}'.format(spark_def_path)) @@ -1544,7 +1627,8 @@ def configure_zeppelin_emr_interpreter(emr_version, cluster_name, region, spark_ python_version = python_version[0:5] livy_port = '' livy_path = '/opt/{0}/{1}/livy/'.format(emr_version, cluster_name) - spark_libs = "/opt/{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-core*.jar /opt/{0}/jars/usr/lib/hadoop/hadoop-aws*.jar /opt/" + \ + spark_libs = "/opt/{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-core*.jar /opt/{0}/jars/usr/lib/hadoop" \ + "/hadoop-aws*.jar /opt/" + \ "{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-s3-*.jar /opt/{0}" + \ "/jars/usr/lib/hadoop-lzo/lib/hadoop-lzo-*.jar".format(emr_version) #fix due to: Multiple py4j files found under ..../spark/python/lib @@ -1552,8 +1636,8 @@ def configure_zeppelin_emr_interpreter(emr_version, cluster_name, region, spark_ local('rm /opt/{0}/{1}/spark/python/lib/py4j-src.zip'.format(emr_version, cluster_name)) local('echo \"Configuring emr path for Zeppelin\"') - local('sed -i \"s/^export SPARK_HOME.*/export SPARK_HOME=\/opt\/{0}\/{1}\/spark/\" /opt/zeppelin/conf/zeppelin-env.sh'. - format(emr_version, cluster_name)) + local('sed -i \"s/^export SPARK_HOME.*/export SPARK_HOME=\/opt\/{0}\/{1}\/spark/\" ' + '/opt/zeppelin/conf/zeppelin-env.sh'.format(emr_version, cluster_name)) local('sed -i "s/^export HADOOP_CONF_DIR.*/export HADOOP_CONF_DIR=' + \ '\/opt\/{0}\/{1}\/conf/" /opt/{0}/{1}/spark/conf/spark-env.sh'.format(emr_version, cluster_name)) local('echo \"spark.jars $(ls {0} | tr \'\\n\' \',\')\" >> /opt/{1}/{2}/spark/conf/spark-defaults.conf' @@ -1590,7 +1674,8 @@ def configure_zeppelin_emr_interpreter(emr_version, cluster_name, region, spark_ local('sudo sed -i "s/^/#/g" {}conf/spark-blacklist.conf'.format(livy_path)) local(''' sudo echo "export SPARK_HOME={0}" >> {1}conf/livy-env.sh'''.format(spark_dir, livy_path)) local(''' sudo echo "export HADOOP_CONF_DIR={0}" >> {1}conf/livy-env.sh'''.format(yarn_dir, livy_path)) - local(''' sudo echo "export PYSPARK3_PYTHON=python{0}" >> {1}conf/livy-env.sh'''.format(python_version[0:3], livy_path)) + local(''' sudo echo "export PYSPARK3_PYTHON=python{0}" >> {1}conf/livy-env.sh'''.format(python_version[0:3], + livy_path)) template_file = "/tmp/dataengine-service_interpreter.json" fr = open(template_file, 'r+') text = fr.read() @@ -1657,9 +1742,12 @@ def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir, datalake_ena endpoint_url = "https://s3.{}.amazonaws.com.cn".format(region) else: endpoint_url = 'https://s3-' + region + '.amazonaws.com' - local("""bash -c 'echo "spark.hadoop.fs.s3a.endpoint """ + endpoint_url + """" >> /tmp/{}/notebook_spark-defaults_local.conf'""".format(cluster_name)) - local('echo "spark.hadoop.fs.s3a.server-side-encryption-algorithm AES256" >> /tmp/{}/notebook_spark-defaults_local.conf'.format(cluster_name)) - local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf {1}spark/conf/spark-defaults.conf'.format(cluster_name, cluster_dir)) + local("""bash -c 'echo "spark.hadoop.fs.s3a.endpoint """ + endpoint_url + + """" >> /tmp/{}/notebook_spark-defaults_local.conf'""".format(cluster_name)) + local('echo "spark.hadoop.fs.s3a.server-side-encryption-algorithm AES256" >> ' + '/tmp/{}/notebook_spark-defaults_local.conf'.format(cluster_name)) + local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf {1}spark/conf/spark-defaults.conf'.format(cluster_name, + cluster_dir)) if spark_configs: spark_configurations = ast.literal_eval(spark_configs) new_spark_defaults = list() @@ -1769,10 +1857,14 @@ def ensure_local_spark(os_user, spark_link, spark_version, hadoop_version, local sys.exit(1) -def install_dataengine_spark(cluster_name, spark_link, spark_version, hadoop_version, cluster_dir, os_user, datalake_enabled): - local('wget ' + spark_link + ' -O /tmp/' + cluster_name + '/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz') - local('tar -zxvf /tmp/' + cluster_name + '/spark-' + spark_version + '-bin-hadoop' + hadoop_version + '.tgz -C /opt/' + cluster_name) - local('mv /opt/' + cluster_name + '/spark-' + spark_version + '-bin-hadoop' + hadoop_version + ' ' + cluster_dir + 'spark/') +def install_dataengine_spark(cluster_name, spark_link, spark_version, hadoop_version, cluster_dir, os_user, + datalake_enabled): + local('wget ' + spark_link + ' -O /tmp/' + cluster_name + '/spark-' + spark_version + '-bin-hadoop' + + hadoop_version + '.tgz') + local('tar -zxvf /tmp/' + cluster_name + '/spark-' + spark_version + '-bin-hadoop' + hadoop_version + + '.tgz -C /opt/' + cluster_name) + local('mv /opt/' + cluster_name + '/spark-' + spark_version + '-bin-hadoop' + hadoop_version + ' ' + + cluster_dir + 'spark/') local('chown -R ' + os_user + ':' + os_user + ' ' + cluster_dir + 'spark/') diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py index c14c8f4..9bce147 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py +++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py @@ -30,6 +30,7 @@ import sys parser = argparse.ArgumentParser() parser.add_argument('--bucket_name', type=str, default='') +parser.add_argument('--bucket_name_tag', type=str, default='') parser.add_argument('--infra_tag_name', type=str, default='') parser.add_argument('--infra_tag_value', type=str, default='') parser.add_argument('--region', type=str, default='') @@ -43,7 +44,7 @@ if __name__ == "__main__": bucket = get_bucket_by_name(args.bucket_name) if bucket == '': print("Creating bucket {0} with tag {1}.".format(args.bucket_name, json.dumps(tag))) - bucket = create_s3_bucket(args.bucket_name, tag, args.region) + bucket = create_s3_bucket(args.bucket_name, tag, args.region, args.bucket_name_tag) else: print("REQUESTED BUCKET ALREADY EXISTS") print("BUCKET_NAME {}".format(bucket)) diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py index 9de94dc..71423cd 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/aws/edge_prepare.py @@ -52,8 +52,9 @@ if __name__ == "__main__": edge_conf['sg_ids'] = os.environ['aws_security_groups_ids'] edge_conf['instance_name'] = '{}-{}-edge'.format(edge_conf['service_base_name'], os.environ['edge_user_name']) edge_conf['tag_name'] = '{}-Tag'.format(edge_conf['service_base_name']) - edge_conf['bucket_name'] = '{}-{}-bucket'.format(edge_conf['service_base_name'], - os.environ['edge_user_name']).lower().replace('_', '-') + edge_conf['bucket_name_tag'] = '{}-{}-bucket'.format(edge_conf['service_base_name'], + os.environ['edge_user_name']) + edge_conf['bucket_name'] = edge_conf['bucket_name_tag'].lower().replace('_', '-') edge_conf['ssn_bucket_name'] = '{}-ssn-bucket'.format(edge_conf['service_base_name']).lower().replace('_', '-') edge_conf['shared_bucket_name'] = '{}-shared-bucket'.format(edge_conf['service_base_name']).lower().replace('_', '-') @@ -109,10 +110,11 @@ if __name__ == "__main__": try: logging.info('[CREATE SUBNET]') print('[CREATE SUBNET]') - params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --username {} --prefix {} --user_subnets_range '{}'" \ - .format(edge_conf['vpc2_id'], edge_conf['tag_name'], edge_conf['service_base_name'], - os.environ['edge_user_name'], edge_conf['private_subnet_prefix'], - os.environ['conf_user_subnets_range']) + params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --username {} --prefix {} " \ + "--user_subnets_range '{}'".format(edge_conf['vpc2_id'], edge_conf['tag_name'], + edge_conf['service_base_name'], os.environ['edge_user_name'], + edge_conf['private_subnet_prefix'], + os.environ['conf_user_subnets_range']) try: local("~/scripts/{}.py {}".format('common_create_subnet', params)) except: @@ -380,9 +382,11 @@ if __name__ == "__main__": } ]) - params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} --infra_tag_value {} --force {}".\ - format(edge_conf['notebook_security_group_name'], edge_conf['vpc2_id'], json.dumps(private_sg_ingress), - json.dumps(private_sg_egress), edge_conf['service_base_name'], edge_conf['notebook_instance_name'], True) + params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \ + "--infra_tag_value {} --force {}".format(edge_conf['notebook_security_group_name'], + edge_conf['vpc2_id'], json.dumps(private_sg_ingress), + json.dumps(private_sg_egress), edge_conf['service_base_name'], + edge_conf['notebook_instance_name'], True) try: local("~/scripts/{}.py {}".format('common_create_security_group', params)) except: @@ -404,10 +408,11 @@ if __name__ == "__main__": logging.info('[CREATING SECURITY GROUPS FOR MASTER NODE]') print("[CREATING SECURITY GROUPS FOR MASTER NODE]") try: - params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} --infra_tag_value {} --force {}". \ - format(edge_conf['dataengine_master_security_group_name'], edge_conf['vpc2_id'], - json.dumps(private_sg_ingress), json.dumps(private_sg_egress), edge_conf['service_base_name'], - edge_conf['dataengine_instances_name'], True) + params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \ + "--infra_tag_value {} --force {}".format(edge_conf['dataengine_master_security_group_name'], + edge_conf['vpc2_id'], json.dumps(private_sg_ingress), + json.dumps(private_sg_egress), edge_conf['service_base_name'], + edge_conf['dataengine_instances_name'], True) try: local("~/scripts/{}.py {}".format('common_create_security_group', params)) except: @@ -425,10 +430,11 @@ if __name__ == "__main__": logging.info('[CREATING SECURITY GROUPS FOR SLAVE NODES]') print("[CREATING SECURITY GROUPS FOR SLAVE NODES]") try: - params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} --infra_tag_value {} --force {}". \ - format(edge_conf['dataengine_slave_security_group_name'], edge_conf['vpc2_id'], - json.dumps(private_sg_ingress), json.dumps(private_sg_egress), edge_conf['service_base_name'], - edge_conf['dataengine_instances_name'], True) + params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \ + "--infra_tag_value {} --force {}".format(edge_conf['dataengine_slave_security_group_name'], + edge_conf['vpc2_id'], json.dumps(private_sg_ingress), + json.dumps(private_sg_egress), edge_conf['service_base_name'], + edge_conf['dataengine_instances_name'], True) try: local("~/scripts/{}.py {}".format('common_create_security_group', params)) except: @@ -447,9 +453,9 @@ if __name__ == "__main__": try: logging.info('[CREATE BUCKETS]') print('[CREATE BUCKETS]') - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {}" \ + params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}" \ .format(edge_conf['bucket_name'], edge_conf['tag_name'], edge_conf['bucket_name'], - edge_conf['region']) + edge_conf['region'], edge_conf['bucket_name_tag']) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) except: @@ -468,10 +474,14 @@ if __name__ == "__main__": try: logging.info('[CREATING BUCKET POLICY FOR USER INSTANCES]') print('[CREATING BUCKET POLICY FOR USER INSTANCES]') - params = '--bucket_name {} --ssn_bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} --notebook_role_name {} --service_base_name {} --region {} --user_predefined_s3_policies "{}"'.format( - edge_conf['bucket_name'], edge_conf['ssn_bucket_name'], edge_conf['shared_bucket_name'], - os.environ['edge_user_name'], edge_conf['role_name'], edge_conf['notebook_dataengine_role_name'], - edge_conf['service_base_name'], edge_conf['region'], os.environ['aws_user_predefined_s3_policies']) + params = '--bucket_name {} --ssn_bucket_name {} --shared_bucket_name {} --username {} --edge_role_name {} ' \ + '--notebook_role_name {} --service_base_name {} --region {} ' \ + '--user_predefined_s3_policies "{}"'.format(edge_conf['bucket_name'], edge_conf['ssn_bucket_name'], + edge_conf['shared_bucket_name'], + os.environ['edge_user_name'], edge_conf['role_name'], + edge_conf['notebook_dataengine_role_name'], + edge_conf['service_base_name'], edge_conf['region'], + os.environ['aws_user_predefined_s3_policies']) try: local("~/scripts/{}.py {}".format('common_create_policy', params)) except: @@ -532,7 +542,8 @@ if __name__ == "__main__": print('Error: {0}'.format(err)) append_result("Failed to associate elastic ip.", str(err)) try: - edge_conf['edge_public_ip'] = get_instance_ip_address(edge_conf['tag_name'], edge_conf['instance_name']).get('Public') + edge_conf['edge_public_ip'] = get_instance_ip_address(edge_conf['tag_name'], + edge_conf['instance_name']).get('Public') edge_conf['allocation_id'] = get_allocation_id_by_elastic_ip(edge_conf['edge_public_ip']) except: print("No Elastic IPs to release!") diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py index 03ecdda..dcb4442 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py +++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py @@ -43,8 +43,10 @@ if __name__ == "__main__": role_name = service_base_name.lower().replace('-', '_') + '-ssn-Role' role_profile_name = service_base_name.lower().replace('-', '_') + '-ssn-Profile' policy_name = service_base_name.lower().replace('-', '_') + '-ssn-Policy' - user_bucket_name = (service_base_name + '-ssn-bucket').lower().replace('_', '-') - shared_bucket_name = (service_base_name + '-shared-bucket').lower().replace('_', '-') + ssn_bucket_name_tag = service_base_name + '-ssn-bucket' + shared_bucket_name_tag = service_base_name + '-shared-bucket' + ssn_bucket_name = ssn_bucket_name_tag.lower().replace('_', '-') + shared_bucket_name = shared_bucket_name_tag.lower().replace('_', '-') tag_name = service_base_name + '-Tag' tag2_name = service_base_name + '-secondary-Tag' instance_name = service_base_name + '-ssn' @@ -391,7 +393,7 @@ if __name__ == "__main__": print("Security IDs: {}".format(os.environ['aws_security_groups_ids'])) print("SSN instance shape: {}".format(os.environ['aws_ssn_instance_size'])) print("SSN AMI name: {}".format(ssn_image_name)) - print("SSN bucket name: {}".format(user_bucket_name)) + print("SSN bucket name: {}".format(ssn_bucket_name)) print("Shared bucket name: {}".format(shared_bucket_name)) print("Region: {}".format(region)) jenkins_url = "http://{}/jenkins".format(get_instance_hostname(tag_name, instance_name)) @@ -418,7 +420,7 @@ if __name__ == "__main__": "subnet_id": os.environ['aws_subnet_id'], "security_id": os.environ['aws_security_groups_ids'], "instance_shape": os.environ['aws_ssn_instance_size'], - "bucket_name": user_bucket_name, + "bucket_name": ssn_bucket_name, "shared_bucket_name": shared_bucket_name, "region": region, "action": "Create SSN instance"} diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py index 5712695..3fcda41 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py @@ -60,8 +60,10 @@ if __name__ == "__main__": role_name = service_base_name.lower().replace('-', '_') + '-ssn-Role' role_profile_name = service_base_name.lower().replace('-', '_') + '-ssn-Profile' policy_name = service_base_name.lower().replace('-', '_') + '-ssn-Policy' - user_bucket_name = (service_base_name + '-ssn-bucket').lower().replace('_', '-') - shared_bucket_name = (service_base_name + '-shared-bucket').lower().replace('_', '-') + ssn_bucket_name_tag = service_base_name + '-ssn-bucket' + shared_bucket_name_tag = service_base_name + '-shared-bucket' + ssn_bucket_name = ssn_bucket_name_tag.lower().replace('_', '-') + shared_bucket_name = shared_bucket_name_tag.lower().replace('_', '-') tag_name = service_base_name + '-Tag' tag2_name = service_base_name + '-secondary-Tag' instance_name = service_base_name + '-ssn' @@ -86,7 +88,8 @@ if __name__ == "__main__": pre_defined_vpc = True logging.info('[CREATE VPC AND ROUTE TABLE]') print('[CREATE VPC AND ROUTE TABLE]') - params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {}".format(vpc_cidr, region, tag_name, service_base_name) + params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {}".format( + vpc_cidr, region, tag_name, service_base_name) try: local("~/scripts/{}.py {}".format('ssn_create_vpc', params)) except: @@ -111,7 +114,8 @@ if __name__ == "__main__": pre_defined_vpc2 = True logging.info('[CREATE SECONDARY VPC AND ROUTE TABLE]') print('[CREATE SECONDARY VPC AND ROUTE TABLE]') - params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --secondary".format(vpc2_cidr, region, tag2_name, service_base_name) + params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --secondary".format( + vpc2_cidr, region, tag2_name, service_base_name) try: local("~/scripts/{}.py {}".format('ssn_create_vpc', params)) except: @@ -136,7 +140,8 @@ if __name__ == "__main__": pre_defined_subnet = True logging.info('[CREATE SUBNET]') print('[CREATE SUBNET]') - params = "--vpc_id {} --username {} --infra_tag_name {} --infra_tag_value {} --prefix {} --ssn {}".format(os.environ['aws_vpc_id'], 'ssn', tag_name, service_base_name, '20', True) + params = "--vpc_id {} --username {} --infra_tag_name {} --infra_tag_value {} --prefix {} " \ + "--ssn {}".format(os.environ['aws_vpc_id'], 'ssn', tag_name, service_base_name, '20', True) try: local("~/scripts/{}.py {}".format('common_create_subnet', params)) except: @@ -169,9 +174,11 @@ if __name__ == "__main__": try: logging.info('[CREATE PEERING CONNECTION]') print('[CREATE PEERING CONNECTION]') - os.environ['aws_peering_id'] = create_peering_connection(os.environ['aws_vpc_id'], os.environ['aws_vpc2_id'], service_base_name) + os.environ['aws_peering_id'] = create_peering_connection(os.environ['aws_vpc_id'], + os.environ['aws_vpc2_id'], service_base_name) print('PEERING CONNECTION ID:' + os.environ['aws_peering_id']) - create_route_by_id(os.environ['aws_subnet_id'], os.environ['aws_vpc_id'], os.environ['aws_peering_id'], get_cidr_by_vpc(os.environ['aws_vpc2_id'])) + create_route_by_id(os.environ['aws_subnet_id'], os.environ['aws_vpc_id'], os.environ['aws_peering_id'], + get_cidr_by_vpc(os.environ['aws_vpc2_id'])) except Exception as err: print('Error: {0}'.format(err)) append_result("Failed to create peering connection.", str(err)) @@ -236,8 +243,10 @@ if __name__ == "__main__": egress_sg_rules_template = format_sg([ {"IpProtocol": "-1", "IpRanges": [{"CidrIp": all_ip_cidr}], "UserIdGroupPairs": [], "PrefixListIds": []} ]) - params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} --infra_tag_value {} --force {} --ssn {}". \ - format(sg_name, os.environ['aws_vpc_id'], json.dumps(ingress_sg_rules_template), json.dumps(egress_sg_rules_template), service_base_name, tag_name, False, True) + params = "--name {} --vpc_id {} --security_group_rules '{}' --egress '{}' --infra_tag_name {} " \ + "--infra_tag_value {} --force {} --ssn {}". \ + format(sg_name, os.environ['aws_vpc_id'], json.dumps(ingress_sg_rules_template), + json.dumps(egress_sg_rules_template), service_base_name, tag_name, False, True) try: local("~/scripts/{}.py {}".format('common_create_security_group', params)) except: @@ -341,8 +350,8 @@ if __name__ == "__main__": try: logging.info('[CREATE BUCKETS]') print('[CREATE BUCKETS]') - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {}". \ - format(user_bucket_name, tag_name, user_bucket_name, region) + params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \ + format(ssn_bucket_name, tag_name, ssn_bucket_name, region, ssn_bucket_name_tag) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) @@ -350,8 +359,8 @@ if __name__ == "__main__": traceback.print_exc() raise Exception - params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {}". \ - format(shared_bucket_name, tag_name, shared_bucket_name, region) + params = "--bucket_name {} --infra_tag_name {} --infra_tag_value {} --region {} --bucket_name_tag {}". \ + format(shared_bucket_name, tag_name, shared_bucket_name, region, shared_bucket_name_tag) try: local("~/scripts/{}.py {}".format('common_create_bucket', params)) @@ -379,7 +388,8 @@ if __name__ == "__main__": try: logging.info('[CREATE SSN INSTANCE]') print('[CREATE SSN INSTANCE]') - params = "--node_name {} --ami_id {} --instance_type {} --key_name {} --security_group_ids {} --subnet_id {} --iam_profile {} --infra_tag_name {} --infra_tag_value {}".\ + params = "--node_name {} --ami_id {} --instance_type {} --key_name {} --security_group_ids {} --subnet_id {} " \ + "--iam_profile {} --infra_tag_name {} --infra_tag_value {}".\ format(instance_name, ssn_ami_id, os.environ['aws_ssn_instance_size'], os.environ['conf_key_name'], os.environ['aws_security_groups_ids'], os.environ['aws_subnet_id'], role_profile_name, tag_name, instance_name) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
