This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2674 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 986f6086f3c1826aad9142dbb0905070a93cef9b Author: leonidfrolov <[email protected]> AuthorDate: Tue Feb 1 16:04:40 2022 +0200 [DATALAB-2674]: added optional parameter to enable bucket versioning --- infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++ infrastructure-provisioning/src/general/conf/datalab.ini | 4 ++++ .../src/general/lib/gcp/actions_lib.py | 10 ++++++++-- .../src/general/scripts/gcp/common_create_bucket.py | 3 ++- .../src/general/scripts/gcp/project_prepare.py | 12 ++++++++---- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py index 40b233d..edafc90 100644 --- a/infrastructure-provisioning/scripts/deploy_datalab.py +++ b/infrastructure-provisioning/scripts/deploy_datalab.py @@ -263,6 +263,8 @@ def build_parser(): help='TRUE if os login enabled for gcp instances') gcp_parser.add_argument('--gcp_block_project_ssh_keys', type=str, default='FALSE', help='TRUE to block project ssh keys for gcp instances') + gcp_parser.add_argument('--gcp_bucket_enable_versioning', type=str, default='false', + help='TRUE to enable versioning for gcp storage buckets') gcp_required_args = gcp_parser.add_argument_group('Required arguments') gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region') diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini index 73ed7ba..9ae94fa 100644 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@ -230,6 +230,10 @@ datalake_enable = false # firewall_rules = ### True if os login is enabled for instances # os_login_enabled = +### True if block_project_ssh_keys is enabled for instances +# block_project_ssh_keys = +### True if versioning is enabled for buckets +# bucket_enable_versioning = ### GCP region name for whole DataLab provisioning region = us-west1 ### GCP zone name for whole DataLab provisioning diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py index 69eb9e7..e92d835 100644 --- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py @@ -210,9 +210,15 @@ class GCPActions: traceback.print_exc(file=sys.stdout) - def create_bucket(self, bucket_name): + def create_bucket(self, bucket_name, versioning_enabled='false'): try: - bucket = self.storage_client.create_bucket(bucket_name) + bucket_params = { + "name": bucket_name, + "versioning": { + "enabled": "{}".format(versioning_enabled) + } + } + bucket = self.storage_client.create_bucket(project=self.project, body=bucket_params) print('Bucket {} created.'.format(bucket.name)) except Exception as err: logging.info( diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py index 34352e5..0c63b2d 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py @@ -31,6 +31,7 @@ from datalab.logger import logging parser = argparse.ArgumentParser() parser.add_argument('--bucket_name', type=str, default='') parser.add_argument('--tags', type=str, default='') +parser.add_argument('--versioning_enabled', type=str, default='false') args = parser.parse_args() if __name__ == "__main__": @@ -39,7 +40,7 @@ if __name__ == "__main__": logging.info("REQUESTED BUCKET {} ALREADY EXISTS".format(args.bucket_name)) else: logging.info("Creating Bucket {}".format(args.bucket_name)) - GCPActions().create_bucket(args.bucket_name) + GCPActions().create_bucket(args.bucket_name, args.versioning_enabled) GCPActions().add_bucket_labels(args.bucket_name, json.loads(args.tags)) else: parser.print_help() diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py index 028087d..0ce097b 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py @@ -121,6 +121,8 @@ if __name__ == "__main__": else: project_conf['user_subnets_range'] = '' + if os.environ['gcp_bucket_enable_versioning'] != 'false': + project_conf['gcp_bucket_enable_versioning'] = 'true' # FUSE in case of absence of user's key try: project_conf['user_key'] = os.environ['key'] @@ -399,8 +401,9 @@ if __name__ == "__main__": os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'], "sbn": project_conf['service_base_name'], "name": project_conf['shared_bucket_name']} - params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'], - json.dumps(project_conf['shared_bucket_tags'])) + params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'], + json.dumps(project_conf['shared_bucket_tags']), + project_conf['gcp_bucket_enable_versioning']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True) except: @@ -414,8 +417,9 @@ if __name__ == "__main__": "sbn": project_conf['service_base_name'], "project_tag": project_conf['project_tag'], "name": project_conf['bucket_name']} - params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'], - json.dumps(project_conf['bucket_tags'])) + params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['bucket_name'], + json.dumps(project_conf['bucket_tags']), + project_conf['gcp_bucket_enable_versioning']) try: subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
