This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DLAB-1539 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 96f901f4c97a4da9969de0a70ddcd4189c191c43 Author: leonidfrolov <[email protected]> AuthorDate: Thu Mar 5 11:52:52 2020 +0200 [DLAB-1539]: changed how additional labels are assigned --- ...common_notebook_configure_dataengine-service.py | 21 +++++++++-------- .../general/scripts/gcp/common_prepare_notebook.py | 21 ++++++++--------- .../scripts/gcp/dataengine-service_prepare.py | 21 +++++++++-------- .../src/general/scripts/gcp/dataengine_prepare.py | 26 +++++++++------------- 4 files changed, 40 insertions(+), 49 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py index 41817fc..0f5ab39 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py @@ -63,27 +63,26 @@ if __name__ == "__main__": application = 'jupyter' else: application = os.environ['application'] - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - if '@' in additional_tags['user_tag']: - notebook_config['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - notebook_config['user_tag'] = additional_tags['user_tag'] + additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "" ).replace("'}", "") notebook_config['cluster_labels'] = { os.environ['notebook_instance_name']: "configured", "name": notebook_config['cluster_name'], "sbn": notebook_config['service_base_name'], - "user": notebook_config['user_tag'], "notebook_name": os.environ['notebook_instance_name'], - "project_tag": notebook_config['project_tag'], - "endpoint_tag": notebook_config['endpoint_tag'], "product": "dlab", "computational_name": (os.environ['computational_name']).lower().replace('_', '-') } - if additional_tags['custom_tag']: - if additional_tags['custom_tag'] != '': - notebook_config['cluster_labels'].update({'custom_tag': additional_tags['custom_tag']}) + + for tag in additional_tags.split(','): + label_key = tag.split(':')[0] + label_value = tag.split(':')[1] + if label_key == 'user_tag': + if '@' in label_value: + label_value = label_value[:label_value.find('@')] + if label_value != '': + notebook_config['cluster_labels'].update({label_key: label_value}) try: logging.info('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]') diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py index f1707a1..96ea2da 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py @@ -132,25 +132,22 @@ if __name__ == "__main__": data = {"notebook_name": notebook_config['instance_name'], "error": ""} json.dump(data, f) - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - - if '@' in additional_tags['user_tag']: - notebook_config['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - notebook_config['user_tag'] = additional_tags['user_tag'] + additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "" ).replace("'}", "") print('Additional tags will be added: {}'.format(additional_tags)) notebook_config['labels'] = {"name": notebook_config['instance_name'], "sbn": notebook_config['service_base_name'], - "project_tag": notebook_config['project_tag'], - "endpoint_tag": notebook_config['endpoint_tag'], - "user": notebook_config['user_tag'], "product": "dlab" } - if additional_tags['custom_tag']: - if additional_tags['custom_tag'] != '': - notebook_config['labels'].update({'custom_tag': additional_tags['custom_tag']}) + for tag in additional_tags.split(','): + label_key = tag.split(':')[0] + label_value = tag.split(':')[1] + if label_key == 'user_tag': + if '@' in label_value: + label_value = label_value[:label_value.find('@')] + if label_value != '': + notebook_config['labels'].update({label_key: label_value}) # launching instance for notebook server try: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py index d6a3e11..4fc3028 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py @@ -76,27 +76,26 @@ if __name__ == "__main__": dataproc_conf['project_name'], dataproc_conf['endpoint_name']) dataproc_conf['release_label'] = os.environ['dataproc_version'] - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - if '@' in additional_tags['user_tag']: - dataproc_conf['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - dataproc_conf['user_tag'] = additional_tags['user_tag'] + additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "" ).replace("'}", "") dataproc_conf['cluster_labels'] = { os.environ['notebook_instance_name']: "not-configured", "name": dataproc_conf['cluster_name'], "sbn": dataproc_conf['service_base_name'], - "user": dataproc_conf['user_tag'], - "project_tag": dataproc_conf['project_tag'], - "endpoint_tag": dataproc_conf['endpoint_tag'], "notebook_name": os.environ['notebook_instance_name'], "product": "dlab", "computational_name": dataproc_conf['computational_name'] } - if additional_tags['custom_tag']: - if additional_tags['custom_tag'] != '': - dataproc_conf['cluster_labels'].update({'custom_tag': additional_tags['custom_tag']}) + + for tag in additional_tags.split(','): + label_key = tag.split(':')[0] + label_value = tag.split(':')[1] + if label_key == 'user_tag': + if '@' in label_value: + label_value = label_value[:label_value.find('@')] + if label_value != '': + dataproc_conf['cluster_labels'].update({label_key: label_value}) dataproc_conf['dataproc_service_account_name'] = '{0}-{1}-ps'.format(dataproc_conf['service_base_name'], dataproc_conf['project_name']) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py index 8febaed..5088de1 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py @@ -143,33 +143,29 @@ if __name__ == "__main__": data_engine['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type'] data_engine['network_tag'] = '{0}-{1}-ps'.format(data_engine['service_base_name'], data_engine['project_name']) - additional_tags = json.loads(os.environ['tags'].replace("': u'", "\": \"").replace("', u'", "\", \"").replace("{u'", "{\"" ).replace("'}", "\"}")) - if '@' in additional_tags['user_tag']: - data_engine['user_tag'] = additional_tags['user_tag'][:additional_tags['user_tag'].find('@')] - else: - data_engine['user_tag'] = additional_tags['user_tag'] + additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "" ).replace("'}", "") data_engine['slave_labels'] = {"name": data_engine['cluster_name'], "sbn": data_engine['service_base_name'], - "user": data_engine['user_tag'], - "project_tag": data_engine['project_tag'], - "endpoint_tag": data_engine['endpoint_tag'], "type": "slave", "notebook_name": data_engine['notebook_name'], "product": "dlab"} data_engine['master_labels'] = {"name": data_engine['cluster_name'], "sbn": data_engine['service_base_name'], - "user": data_engine['user_tag'], - "project_tag": data_engine['project_tag'], - "endpoint_tag": data_engine['endpoint_tag'], "type": "master", "notebook_name": data_engine['notebook_name'], "product": "dlab"} - if additional_tags['custom_tag']: - if additional_tags['custom_tag'] != '': - data_engine['slave_labels'].update({'custom_tag': additional_tags['custom_tag']}) - data_engine['master_labels'].update({'custom_tag': additional_tags['custom_tag']}) + + for tag in additional_tags.split(','): + label_key = tag.split(':')[0] + label_value = tag.split(':')[1] + if label_key == 'user_tag': + if '@' in label_value: + label_value = label_value[:label_value.find('@')] + if label_value != '': + data_engine['slave_labels'].update({label_key: label_value}) + data_engine['master_labels'].update({label_key: label_value}) try: logging.info('[CREATE MASTER NODE]') --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
