This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2674 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 8ac6956aa9049e4f1e3a7198d637c0d93d5011f9 Author: leonidfrolov <[email protected]> AuthorDate: Wed Mar 9 14:53:29 2022 +0200 [DATALAB-2674]: added lifecycle rules for gcp storage buckets --- infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++ .../src/general/conf/datalab.ini | 2 ++ .../src/general/lib/gcp/actions_lib.py | 5 ++++- .../src/general/scripts/gcp/common_create_bucket.py | 4 +++- .../src/general/scripts/gcp/project_prepare.py | 18 ++++++++++++------ 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py index 8f55428..2de4c5d 100644 --- a/infrastructure-provisioning/scripts/deploy_datalab.py +++ b/infrastructure-provisioning/scripts/deploy_datalab.py @@ -268,6 +268,8 @@ def build_parser(): gcp_parser.add_argument('--gcp_cmek_resource_name', type=str, default='', help='customer managed encryption key resource name ' 'e.g. projects/{project_name}/locations/{us}/keyRings/{keyring_name}/cryptoKeys/{key_name}') + gcp_parser.add_argument('--gcp_storage_lifecycle_rules', type=str, default='', + help='storage bucket lifecycle rules') gcp_parser.add_argument('--gcp_wrapped_csek', type=str, default='', help='customer supplied encryption key for disk/image encryption in RFC 4648 base64 ' 'encoded, RSA-wrapped 2048-bit format as rsaEncryptedKey') diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini index 336d7e5..fe84416 100644 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@ -236,6 +236,8 @@ block_project_ssh_keys = FALSE bucket_enable_versioning = false ### gcp customer managed encryption key to use # cmek_resource_name = +### gcp customer managed encryption key to use +# storage_lifecycle_rules = ### gcp customer supplied wrapped encryption key to use # wrapped_csek = ### GCP region name for whole DataLab provisioning diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py index 1ff8a4a..b37cdcd 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py @@ -222,7 +222,8 @@ class GCPActions: file=sys.stdout)})) traceback.print_exc(file=sys.stdout) - def add_bucket_labels_vers_cmek(self, bucket_name, tags, versioning_enabled='false', cmek_resource_name=''): + def add_bucket_labels_vers_cmek(self, bucket_name, tags, versioning_enabled='false', cmek_resource_name='', + lifecycle_rules=''): try: bucket = self.storage_client.get_bucket(bucket_name) labels = bucket.labels @@ -231,6 +232,8 @@ class GCPActions: bucket.versioning = {"enabled": versioning_enabled} if cmek_resource_name != '': bucket.encryption = {"defaultKmsKeyName": cmek_resource_name} + if lifecycle_rules != '': + bucket.lifecycle = lifecycle_rules bucket.patch() print('Updated labels on {}.'.format(bucket_name)) except Exception as err: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py index 5a49201..389ca10 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py @@ -33,6 +33,7 @@ parser.add_argument('--bucket_name', type=str, default='') parser.add_argument('--tags', type=str, default='') parser.add_argument('--versioning_enabled', type=str, default='false') parser.add_argument('--cmek_resource_name', type=str, default='') +parser.add_argument('--lifecycle_rules', type=str, default='') args = parser.parse_args() if __name__ == "__main__": @@ -42,7 +43,8 @@ if __name__ == "__main__": else: logging.info("Creating Bucket {}".format(args.bucket_name)) GCPActions().create_bucket(args.bucket_name) - GCPActions().add_bucket_labels_vers_cmek(args.bucket_name, json.loads(args.tags), args.versioning_enabled, args.cmek_resource_name) + GCPActions().add_bucket_labels_vers_cmek(args.bucket_name, json.loads(args.tags), args.versioning_enabled, + args.cmek_resource_name, json.loads(args.lifecycle_rules)) else: parser.print_help() sys.exit(2) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py index 15a7270..a8da0ec 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py @@ -126,6 +126,12 @@ if __name__ == "__main__": project_conf['gcp_cmek_resource_name'] = os.environ['gcp_cmek_resource_name'] else: project_conf['gcp_cmek_resource_name'] = '' + + if 'gcp_storage_lifecycle_rules' in os.environ: + project_conf['gcp_storage_lifecycle_rules'] = os.environ['gcp_storage_lifecycle_rules'] + else: + project_conf['gcp_storage_lifecycle_rules'] = '' + # FUSE in case of absence of user's key try: project_conf['user_key'] = os.environ['key'] @@ -404,9 +410,9 @@ if __name__ == "__main__": os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'], "sbn": project_conf['service_base_name'], "name": project_conf['shared_bucket_name']} - params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'], - json.dumps(project_conf['shared_bucket_tags']), - project_conf['gcp_bucket_enable_versioning']) + params = "--bucket_name {} --tags '{}' --versioning_enabled {} --lifecycle_rules '{}'".format( + project_conf['shared_bucket_name'], json.dumps(project_conf['shared_bucket_tags']), + project_conf['gcp_bucket_enable_versioning'], json.dumps(project_conf['gcp_storage_lifecycle_rules'])) if project_conf['gcp_cmek_resource_name'] != '': params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name']) @@ -424,9 +430,9 @@ if __name__ == "__main__": "sbn": project_conf['service_base_name'], "project_tag": project_conf['project_tag'], "name": project_conf['bucket_name']} - params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['bucket_name'], - json.dumps(project_conf['bucket_tags']), - project_conf['gcp_bucket_enable_versioning']) + params = "--bucket_name {} --tags '{}' --versioning_enabled {} --lifecycle_rules '{}'".format( + project_conf['bucket_name'], json.dumps(project_conf['bucket_tags']), + project_conf['gcp_bucket_enable_versioning'], json.dumps(project_conf['gcp_storage_lifecycle_rules'])) if project_conf['gcp_cmek_resource_name'] != '': params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name']) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
