This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-1546
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1546 by this push:
new d43e92f [DLAB-1546]: added name convention changes
d43e92f is described below
commit d43e92f3409cd64bae287a9cdbb5e5efe1cb297f
Author: Oleh Martushevskyi <[email protected]>
AuthorDate: Tue Feb 25 14:11:15 2020 +0200
[DLAB-1546]: added name convention changes
---
.../scripts/azure/common_create_notebook_image.py | 45 ++++++-------
.../azure/common_notebook_configure_dataengine.py | 76 ++++++++++-----------
.../general/scripts/azure/common_start_notebook.py | 42 ++++++------
.../general/scripts/azure/common_stop_notebook.py | 33 +++++-----
.../scripts/azure/common_terminate_notebook.py | 31 ++++-----
.../azure/common_terminate_notebook_image.py | 12 ++--
.../general/scripts/azure/dataengine_prepare.py | 2 +-
.../src/general/scripts/azure/dataengine_start.py | 45 +++++++------
.../src/general/scripts/azure/dataengine_stop.py | 40 +++++------
.../general/scripts/azure/dataengine_terminate.py | 42 ++++++------
.../src/general/scripts/azure/edge_start.py | 26 ++++----
.../src/general/scripts/azure/edge_status.py | 22 ++++---
.../src/general/scripts/azure/edge_stop.py | 22 ++++---
.../src/general/scripts/azure/edge_terminate.py | 54 ++++++++-------
.../src/general/scripts/azure/ssn_terminate.py | 51 +++++++-------
...common_notebook_configure_dataengine-service.py | 56 +++++++++-------
.../gcp/common_notebook_configure_dataengine.py | 77 +++++++++++-----------
.../general/scripts/gcp/common_start_notebook.py | 23 ++++---
.../general/scripts/gcp/common_stop_notebook.py | 30 ++++-----
.../scripts/gcp/common_terminate_notebook.py | 30 ++++-----
.../general/scripts/gcp/dataengine_configure.py | 2 +-
.../src/general/scripts/gcp/dataengine_prepare.py | 2 +-
.../src/general/scripts/gcp/dataengine_start.py | 46 ++++++-------
.../src/general/scripts/gcp/dataengine_stop.py | 41 ++++++------
.../general/scripts/gcp/dataengine_terminate.py | 47 ++++++-------
.../src/general/scripts/gcp/edge_start.py | 25 ++++---
.../src/general/scripts/gcp/edge_stop.py | 24 +++----
.../src/general/scripts/gcp/edge_terminate.py | 45 +++++++------
.../src/general/scripts/gcp/ssn_configure.py | 2 +-
.../src/general/scripts/gcp/ssn_terminate.py | 28 ++++----
30 files changed, 538 insertions(+), 483 deletions(-)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index c3024c5..883ce95 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@ -21,9 +21,11 @@
#
#
******************************************************************************
-from dlab.actions_lib import *
-from dlab.meta_lib import *
-from dlab.fab import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import os
+import traceback
import sys
import json
@@ -33,19 +35,20 @@ if __name__ == "__main__":
image_conf = dict()
image_conf['service_base_name'] = os.environ['conf_service_base_name']
image_conf['resource_group_name'] =
os.environ['azure_resource_group_name']
- image_conf['user_name'] = os.environ['edge_user_name'].replace('_',
'-')
- image_conf['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- image_conf['project_tag'] = os.environ['project_name'].replace('_',
'-')
- image_conf['endpoint_name'] = os.environ['endpoint_name'].replace('_',
'-')
- image_conf['endpoint_tag'] = os.environ['endpoint_name'].replace('_',
'-')
+ image_conf['user_name'] = os.environ['edge_user_name']
+ image_conf['project_name'] = os.environ['project_name'].lower()
+ image_conf['project_tag'] = image_conf['project_name']
+ image_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
+ image_conf['endpoint_tag'] = image_conf['endpoint_name']
image_conf['instance_name'] = os.environ['notebook_instance_name']
image_conf['application'] = os.environ['application']
image_conf['dlab_ssh_user'] = os.environ['conf_os_user']
- image_conf['image_name'] =
os.environ['notebook_image_name'].lower().replace('_', '-')
- image_conf['full_image_name'] =
'{}-{}-{}-{}'.format(image_conf['service_base_name'],
-
image_conf['project_name'],
-
image_conf['application'],
-
image_conf['image_name']).lower()
+ image_conf['image_name'] = os.environ['notebook_image_name'].lower()
+ image_conf['full_image_name'] =
'{}-{}-{}-{}-{}'.format(image_conf['service_base_name'],
+
image_conf['project_name'],
+
image_conf['endpoint_name'],
+
image_conf['application'],
+
image_conf['image_name']).lower()
image_conf['tags'] = {"Name": image_conf['service_base_name'],
"SBN": image_conf['service_base_name'],
"User": image_conf['user_name'],
@@ -72,7 +75,7 @@ if __name__ == "__main__":
image = AzureMeta().get_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
if image == '':
print('Creating image from existing notebook.')
- prepare_vm_for_image(True, image_conf['dlab_ssh_user'],
instance_hostname, keyfile_name)
+ dlab.actions_lib.prepare_vm_for_image(True,
image_conf['dlab_ssh_user'], instance_hostname, keyfile_name)
AzureActions().create_image_from_instance(image_conf['resource_group_name'],
image_conf['instance_name'],
os.environ['azure_region'],
@@ -88,9 +91,9 @@ if __name__ == "__main__":
instance_running = True
instance_hostname =
AzureMeta().get_private_ip_address(image_conf['resource_group_name'],
image_conf['instance_name'])
- remount_azure_disk(True, image_conf['dlab_ssh_user'],
instance_hostname, keyfile_name)
- set_git_proxy(image_conf['dlab_ssh_user'], instance_hostname,
keyfile_name,
- 'http://{}:3128'.format(edge_instance_hostname))
+ dlab.actions_lib.remount_azure_disk(True,
image_conf['dlab_ssh_user'], instance_hostname, keyfile_name)
+ dlab.fab.set_git_proxy(image_conf['dlab_ssh_user'],
instance_hostname, keyfile_name,
+
'http://{}:3128'.format(edge_instance_hostname))
additional_config = {"proxy_host": edge_instance_hostname,
"proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {}
--additional_config '{}' --os_user {}" \
.format(instance_hostname, image_conf['instance_name'],
keyfile_name,
@@ -98,9 +101,8 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('common_configure_proxy',
params))
print("Image was successfully created. It's name is
{}".format(image_conf['full_image_name']))
except Exception as err:
- print('Error: {0}'.format(err))
AzureActions().remove_instance(image_conf['resource_group_name'],
image_conf['instance_name'])
- append_result("Failed to create instance from image.",
str(err))
+ dlab.fab.append_result("Failed to create instance from
image.", str(err))
sys.exit(1)
with open("/root/result.json", 'w') as result:
@@ -114,6 +116,5 @@ if __name__ == "__main__":
"Action": "Create image from notebook"}
result.write(json.dumps(res))
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to create image from notebook", str(err))
- sys.exit(1)
\ No newline at end of file
+ dlab.fab.append_result("Failed to create image from notebook",
str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
index b755c91..6b67ffe 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
@@ -24,13 +24,20 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
import uuid
+def clear_resources():
+ for i in range(notebook_config['instance_count'] - 1):
+ slave_name = notebook_config['slave_node_name'] + '{}'.format(i + 1)
+ AzureActions().remove_instance(notebook_config['resource_group_name'],
slave_name)
+ AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['master_node_name'])
+
+
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
os.environ['request_id'])
@@ -43,28 +50,30 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- try:
- notebook_config['exploratory_name'] =
os.environ['exploratory_name'].replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ notebook_config['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
notebook_config['exploratory_name'] = ''
- try:
- notebook_config['computational_name'] =
os.environ['computational_name'].replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ notebook_config['computational_name'] =
os.environ['computational_name'].lower()
+ else:
notebook_config['computational_name'] = ''
notebook_config['service_base_name'] =
os.environ['conf_service_base_name']
notebook_config['resource_group_name'] =
os.environ['azure_resource_group_name']
notebook_config['region'] = os.environ['azure_region']
- notebook_config['user_name'] =
os.environ['edge_user_name'].replace('_', '-')
- notebook_config['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- notebook_config['project_tag'] =
os.environ['project_name'].replace('_', '-')
- notebook_config['endpoint_tag'] =
os.environ['endpoint_name'].replace('_', '-')
- notebook_config['cluster_name'] = notebook_config['service_base_name']
+ '-' + notebook_config['project_name'] + \
- '-de-' +
notebook_config['exploratory_name'] + '-' + \
- notebook_config['computational_name']
+ notebook_config['user_name'] = os.environ['edge_user_name']
+ notebook_config['project_name'] = os.environ['project_name'].lower()
+ notebook_config['project_tag'] = notebook_config['project_name']
+ notebook_config['endpoint_name'] = os.environ['endpoint_name'].lower()
+ notebook_config['endpoint_tag'] = notebook_config['endpoint_name']
+ notebook_config['cluster_name'] =
'{}-{}-{}-de-{}'.format(notebook_config['service_base_name'],
+
notebook_config['project_name'],
+
notebook_config['endpoint_name'],
+
notebook_config['exploratory_name'])
notebook_config['master_node_name'] = notebook_config['cluster_name']
+ '-m'
notebook_config['slave_node_name'] = notebook_config['cluster_name'] +
'-s'
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
- notebook_config['key_path'] = os.environ['conf_key_dir'] + '/' +
os.environ['conf_key_name'] + '.pem'
+ notebook_config['key_path'] =
'{}/{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['instance_count'] =
int(os.environ['dataengine_instance_count'])
try:
@@ -73,16 +82,14 @@ if __name__ == "__main__":
notebook_config['notebook_ip'] =
AzureMeta().get_private_ip_address(
notebook_config['resource_group_name'],
notebook_config['notebook_name'])
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to get instance IP address",
str(err))
+ clear_resources()
sys.exit(1)
notebook_config['spark_master_url'] =
'spark://{}:7077'.format(notebook_config['spark_master_ip'])
except Exception as err:
- for i in range(notebook_config['instance_count'] - 1):
- slave_name = notebook_config['slave_node_name'] + '{}'.format(i+1)
-
AzureActions().remove_instance(notebook_config['resource_group_name'],
slave_name)
- AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['master_node_name'])
- append_result("Failed to generate infrastructure names", str(err))
+ clear_resources()
+ dlab.fab.append_result("Failed to generate infrastructure names",
str(err))
sys.exit(1)
try:
@@ -100,12 +107,8 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- for i in range(notebook_config['instance_count'] - 1):
- slave_name = notebook_config['slave_node_name'] + '{}'.format(i+1)
-
AzureActions().remove_instance(notebook_config['resource_group_name'],
slave_name)
- AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['master_node_name'])
- append_result("Failed installing Dataengine kernels.", str(err))
+ clear_resources()
+ dlab.fab.append_result("Failed installing Dataengine kernels.",
str(err))
sys.exit(1)
try:
@@ -125,12 +128,8 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- for i in range(notebook_config['instance_count'] - 1):
- slave_name = notebook_config['slave_node_name'] + '{}'.format(i+1)
-
AzureActions().remove_instance(notebook_config['resource_group_name'],
slave_name)
- AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['master_node_name'])
- append_result("Failed to configure Spark.", str(err))
+ clear_resources()
+ dlab.fab.append_result("Failed to configure Spark.", str(err))
sys.exit(1)
try:
@@ -139,6 +138,7 @@ if __name__ == "__main__":
"Action": "Configure notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ clear_resources()
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
index 5dcbf3e..a47d883 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
@@ -24,9 +24,10 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
import argparse
@@ -53,11 +54,11 @@ if __name__ == "__main__":
print("Starting notebook")
AzureActions().start_instance(notebook_config['resource_group_name'],
notebook_config['notebook_name'])
print("Instance {} has been
started".format(notebook_config['notebook_name']))
- except Exception as err:
+ except:
traceback.print_exc()
- append_result("Failed to start notebook.", str(err))
raise Exception
- except:
+ except Exception as err:
+ dlab.fab.append_result("Failed to start notebook.", str(err))
sys.exit(1)
try:
@@ -70,11 +71,11 @@ if __name__ == "__main__":
.format(os.environ['conf_os_user'],
notebook_config['notebook_ip'], notebook_config['keyfile'])
try:
local("~/scripts/{}.py {}".format('manage_git_creds', params))
- except Exception as err:
+ except:
traceback.print_exc()
- append_result("Failed to setup git credentials.", str(err))
raise Exception
- except:
+ except Exception as err:
+ dlab.fab.append_result("Failed to setup git credentials.", str(err))
sys.exit(1)
if os.environ['azure_datalake_enable'] == 'true':
@@ -90,13 +91,14 @@ if __name__ == "__main__":
params = '--refresh_token
{}'.format(os.environ['azure_user_refresh_token'])
try:
put('~/scripts/common_notebook_update_refresh_token.py',
'/tmp/common_notebook_update_refresh_token.py')
- sudo('mv /tmp/common_notebook_update_refresh_token.py
/usr/local/bin/common_notebook_update_refresh_token.py')
+ sudo('mv /tmp/common_notebook_update_refresh_token.py '
+ '/usr/local/bin/common_notebook_update_refresh_token.py')
sudo("/usr/bin/python /usr/local/bin/{}.py
{}".format('common_notebook_update_refresh_token', params))
- except Exception as err:
+ except:
traceback.print_exc()
- append_result("Failed to update storage credentials.",
str(err))
raise Exception
- except:
+ except Exception as err:
+ dlab.fab.append_result("Failed to update storage credentials.",
str(err))
sys.exit(1)
try:
@@ -106,16 +108,16 @@ if __name__ == "__main__":
.format(os.environ['conf_os_user'],
notebook_config['notebook_ip'], notebook_config['keyfile'])
try:
local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
- except Exception as err:
+ except:
traceback.print_exc()
- append_result("Failed to update last activity time.", str(err))
raise Exception
- except:
+ except Exception as err:
+ dlab.fab.append_result("Failed to update last activity time.",
str(err))
sys.exit(1)
try:
ip_address =
AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
-
notebook_config['notebook_name'])
+
notebook_config['notebook_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['notebook_name']))
@@ -126,8 +128,8 @@ if __name__ == "__main__":
"Action": "Start up notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
index 4c4ba17..1534b0f 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
@@ -24,9 +24,9 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
import uuid
import argparse
@@ -45,7 +45,7 @@ def stop_notebook(resource_group_name, notebook_name):
AzureActions().stop_instance(resource_group_name, vm.name)
print("Instance {} has been stopped".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to stop clusters", str(err))
sys.exit(1)
print("Stopping notebook")
@@ -56,7 +56,7 @@ def stop_notebook(resource_group_name, notebook_name):
AzureActions().stop_instance(resource_group_name, vm.name)
print("Instance {} has been stopped".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to stop instance", str(err))
sys.exit(1)
@@ -71,13 +71,13 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- try:
- notebook_config['exploratory_name'] =
os.environ['exploratory_name'].replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ notebook_config['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
notebook_config['exploratory_name'] = ''
- try:
- notebook_config['computational_name'] =
os.environ['computational_name'].replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ notebook_config['computational_name'] =
os.environ['computational_name'].lower()
+ else:
notebook_config['computational_name'] = ''
notebook_config['resource_group_name'] =
os.environ['azure_resource_group_name']
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
@@ -87,18 +87,15 @@ if __name__ == "__main__":
try:
stop_notebook(notebook_config['resource_group_name'],
notebook_config['notebook_name'])
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to stop notebook.", str(err))
+ dlab.fab.append_result("Failed to stop notebook.", str(err))
sys.exit(1)
-
try:
with open("/root/result.json", 'w') as result:
res = {"notebook_name": notebook_config['notebook_name'],
"Action": "Stop notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
-
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
index e08130d..d25f40c 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
@@ -24,11 +24,12 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
import uuid
+import traceback
def terminate_nb(resource_group_name, notebook_name):
@@ -40,7 +41,7 @@ def terminate_nb(resource_group_name, notebook_name):
AzureActions().remove_instance(resource_group_name,
vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate clusters", str(err))
sys.exit(1)
print("Terminating notebook")
@@ -51,7 +52,7 @@ def terminate_nb(resource_group_name, notebook_name):
AzureActions().remove_instance(resource_group_name,
vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate instance", str(err))
sys.exit(1)
@@ -65,13 +66,13 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- try:
- notebook_config['exploratory_name'] =
os.environ['exploratory_name'].replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ notebook_config['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
notebook_config['exploratory_name'] = ''
- try:
- notebook_config['computational_name'] =
os.environ['computational_name'].replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ notebook_config['computational_name'] =
os.environ['computational_name'].lower()
+ else:
notebook_config['computational_name'] = ''
notebook_config['resource_group_name'] =
os.environ['azure_resource_group_name']
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
@@ -83,7 +84,7 @@ if __name__ == "__main__":
terminate_nb(notebook_config['resource_group_name'],
notebook_config['notebook_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to terminate notebook.", str(err))
+ dlab.fab.append_result("Failed to terminate notebook.", str(err))
raise Exception
except:
sys.exit(1)
@@ -94,6 +95,6 @@ if __name__ == "__main__":
"Action": "Terminate notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
\ No newline at end of file
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
index ff186ac..cab6fc4 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
@@ -21,11 +21,12 @@
#
#
******************************************************************************
-from dlab.actions_lib import *
-from dlab.meta_lib import *
-from dlab.fab import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import sys
import json
+import os
if __name__ == "__main__":
@@ -45,6 +46,5 @@ if __name__ == "__main__":
"Action": "Delete existing notebook image"}
result.write(json.dumps(res))
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to delete existing notebook image", str(err))
- sys.exit(1)
\ No newline at end of file
+ dlab.fab.append_result("Failed to delete existing notebook image",
str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
index 26a9a57..58fc584 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
@@ -130,7 +130,7 @@ if __name__ == "__main__":
data_engine['image_name'] =
os.environ['azure_{}_image_name'.format(os.environ['conf_os_family'])]
print('No pre-configured image found. Using default one:
{}'.format(data_engine['image_name']))
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.
Exception:" + str(err))
+ dlab.fab.append_result("Failed to generate variables dictionary",
str(err))
sys.exit(1)
try:
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
index cf2a613..a058c85 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
@@ -24,10 +24,11 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
+import traceback
import uuid
@@ -40,7 +41,7 @@ def start_data_engine(resource_group_name, cluster_name):
AzureActions().start_instance(resource_group_name, vm.name)
print("Instance {} has been started".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to start dataengine", str(err))
sys.exit(1)
@@ -54,22 +55,23 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
data_engine = dict()
- try:
- data_engine['exploratory_name'] =
os.environ['exploratory_name'].replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
data_engine['exploratory_name'] = ''
- try:
- data_engine['computational_name'] =
os.environ['computational_name'].replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ data_engine['computational_name'] =
os.environ['computational_name'].lower()
+ else:
data_engine['computational_name'] = ''
data_engine['service_base_name'] = os.environ['conf_service_base_name']
data_engine['resource_group_name'] =
os.environ['azure_resource_group_name']
data_engine['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- data_engine['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- data_engine['cluster_name'] =
'{}-{}-de-{}-{}'.format(data_engine['service_base_name'],
+ data_engine['project_name'] = os.environ['project_name'].lower()
+ data_engine['endpoint_name'] = os.environ['endpoint_name'].lower()
+ data_engine['cluster_name'] =
'{}-{}-{}-de-{}'.format(data_engine['service_base_name'],
data_engine['project_name'],
-
data_engine['exploratory_name'],
-
data_engine['computational_name'])
+
data_engine['endpoint_name'],
+
data_engine['exploratory_name'])
try:
logging.info('[STARTING DATA ENGINE]')
print('[STARTING DATA ENGINE]')
@@ -86,8 +88,10 @@ if __name__ == "__main__":
logging.info('[UPDATE LAST ACTIVITY TIME]')
print('[UPDATE LAST ACTIVITY TIME]')
data_engine['computational_id'] = data_engine['cluster_name'] + '-m'
- data_engine['notebook_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
os.environ['notebook_instance_name'])
- data_engine['computational_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
data_engine['computational_id'])
+ data_engine['notebook_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
+
os.environ['notebook_instance_name'])
+ data_engine['computational_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
+
data_engine['computational_id'])
data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
params = '--os_user {0} --notebook_ip {1} --keyfile "{2}" --cluster_ip
{3}' \
.format(os.environ['conf_os_user'], data_engine['notebook_ip'],
data_engine['keyfile'],
@@ -96,18 +100,17 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
except Exception as err:
traceback.print_exc()
- append_result("Failed to update last activity time.", str(err))
+ dlab.fab.append_result("Failed to update last activity time.",
str(err))
raise Exception
except:
sys.exit(1)
-
try:
with open("/root/result.json", 'w') as result:
res = {"service_base_name": data_engine['service_base_name'],
"Action": "Start Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
\ No newline at end of file
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
index ef1521f..a6d28d1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
@@ -24,10 +24,11 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
+import traceback
import uuid
@@ -40,7 +41,7 @@ def stop_data_engine(resource_group_name, cluster_name):
AzureActions().stop_instance(resource_group_name, vm.name)
print("Instance {} has been stopped".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to stop dataengine", str(err))
sys.exit(1)
@@ -54,22 +55,23 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
data_engine = dict()
- try:
- data_engine['exploratory_name'] =
os.environ['exploratory_name'].replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
data_engine['exploratory_name'] = ''
- try:
- data_engine['computational_name'] =
os.environ['computational_name'].replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ data_engine['computational_name'] =
os.environ['computational_name'].lower()
+ else:
data_engine['computational_name'] = ''
data_engine['service_base_name'] = os.environ['conf_service_base_name']
- data_engine['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- data_engine['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
+ data_engine['user_name'] = os.environ['edge_user_name'].lower()
+ data_engine['project_name'] = os.environ['project_name'].lower()
+ data_engine['endpoint_name'] = os.environ['endpoint_name'].lower()
data_engine['resource_group_name'] =
os.environ['azure_resource_group_name']
- data_engine['cluster_name'] =
'{}-{}-de-{}-{}'.format(data_engine['service_base_name'],
+ data_engine['cluster_name'] =
'{}-{}-{}-de-{}'.format(data_engine['service_base_name'],
data_engine['project_name'],
-
data_engine['exploratory_name'],
-
data_engine['computational_name'])
+
data_engine['endpoint_name'],
+
data_engine['exploratory_name'])
try:
logging.info('[STOPPING DATA ENGINE]')
print('[STOPPING DATA ENGINE]')
@@ -77,7 +79,7 @@ if __name__ == "__main__":
stop_data_engine(data_engine['resource_group_name'],
data_engine['cluster_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to stop Data Engine.", str(err))
+ dlab.fab.append_result("Failed to stop Data Engine.", str(err))
raise Exception
except:
sys.exit(1)
@@ -88,6 +90,6 @@ if __name__ == "__main__":
"Action": "Stop Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
index 974fc3e..a47ecdc 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
@@ -24,10 +24,11 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
+import traceback
import uuid
@@ -40,14 +41,14 @@ def terminate_data_engine(resource_group_name,
notebook_name, os_user, key_path,
AzureActions().remove_instance(resource_group_name,
vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate dataengine", str(err))
sys.exit(1)
print("Removing Data Engine kernels from notebook")
try:
AzureActions().remove_dataengine_kernels(resource_group_name,
notebook_name, os_user, key_path, cluster_name)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove dataengine kernels from
notebook", str(err))
sys.exit(1)
@@ -61,22 +62,23 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
data_engine = dict()
- try:
- data_engine['exploratory_name'] =
os.environ['exploratory_name'].replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
data_engine['exploratory_name'] = ''
- try:
- data_engine['computational_name'] =
os.environ['computational_name'].replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ data_engine['computational_name'] =
os.environ['computational_name'].lower()
+ else:
data_engine['computational_name'] = ''
data_engine['service_base_name'] = os.environ['conf_service_base_name']
data_engine['resource_group_name'] =
os.environ['azure_resource_group_name']
- data_engine['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- data_engine['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- data_engine['cluster_name'] =
'{}-{}-de-{}-{}'.format(data_engine['service_base_name'],
+ data_engine['user_name'] = os.environ['edge_user_name'].lower()
+ data_engine['project_name'] = os.environ['project_name'].lower()
+ data_engine['endpoint_name'] = os.environ['endpoint_name'].lower()
+ data_engine['cluster_name'] =
'{}-{}-{}-de-{}'.format(data_engine['service_base_name'],
data_engine['project_name'],
-
data_engine['exploratory_name'],
-
data_engine['computational_name'])
+
data_engine['endpoint_name'],
+
data_engine['exploratory_name'])
data_engine['notebook_name'] = os.environ['notebook_instance_name']
data_engine['key_path'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
@@ -89,7 +91,7 @@ if __name__ == "__main__":
os.environ['conf_os_user'],
data_engine['key_path'], data_engine['cluster_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to terminate Data Engine.", str(err))
+ dlab.fab.append_result("Failed to terminate Data Engine.",
str(err))
raise Exception
except:
sys.exit(1)
@@ -100,6 +102,6 @@ if __name__ == "__main__":
"Action": "Terminate Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
index 445f48d..1174c4a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
@@ -21,9 +21,13 @@
#
#
******************************************************************************
-from dlab.fab import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import os
+import logging
import sys
+import json
if __name__ == "__main__":
@@ -38,26 +42,26 @@ if __name__ == "__main__":
edge_conf = dict()
edge_conf['service_base_name'] = os.environ['conf_service_base_name']
edge_conf['resource_group_name'] = os.environ['azure_resource_group_name']
- edge_conf['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- edge_conf['endpoint_name'] =
os.environ['endpoint_name'].lower().replace('_', '-')
+ edge_conf['project_name'] = os.environ['project_name'].lower()
+ edge_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
edge_conf['instance_name'] =
'{0}-{1}-{2}-edge'.format(edge_conf['service_base_name'],
edge_conf['project_name'], edge_conf['endpoint_name'])
- edge_conf['instance_dns_name'] = 'host-' + edge_conf['instance_name'] +
'.' + os.environ['azure_region'] + '.cloudapp.azure.com'
+ edge_conf['instance_dns_name'] =
'host-{}.{}.cloudapp.azure.com'.format(edge_conf['instance_name'],
+
os.environ['azure_region'])
logging.info('[START EDGE]')
print('[START EDGE]')
try:
AzureActions().start_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to start edge.", str(err))
+ dlab.fab.append_result("Failed to start edge.", str(err))
sys.exit(1)
try:
public_ip_address =
AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'],
edge_conf['instance_name'])
private_ip_address =
AzureMeta().get_private_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
+
edge_conf['instance_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(edge_conf['instance_name']))
@@ -72,7 +76,7 @@ if __name__ == "__main__":
"Action": "Start up notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
index 8c16d12..969eba8 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
@@ -23,14 +23,19 @@
import json
-from dlab.fab import *
-from dlab.meta_lib import *
-import sys, time, os
-from dlab.actions_lib import *
+import sys
+import time
+import os
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import logging
+import traceback
if __name__ == "__main__":
- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
+ os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
@@ -44,13 +49,12 @@ if __name__ == "__main__":
logging.info('[COLLECT DATA]')
print('[COLLECTING DATA]')
params = '--resource_group_name {} --list_resources
"{}"'.format(edge_conf['resource_group_name'],
-
os.environ['edge_list_resources'])
+
os.environ['edge_list_resources'])
try:
local("~/scripts/{}.py {}".format('common_collect_data', params))
except:
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to collect necessary information.", str(err))
- sys.exit(1)
\ No newline at end of file
+ dlab.fab.append_result("Failed to collect necessary information.",
str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
index 1bb319b..6724739 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
@@ -20,10 +20,13 @@
# under the License.
#
#
******************************************************************************
-
-from dlab.fab import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import logging
+import os
import sys
+import json
if __name__ == "__main__":
@@ -38,8 +41,8 @@ if __name__ == "__main__":
edge_conf = dict()
edge_conf['service_base_name'] = os.environ['conf_service_base_name']
edge_conf['resource_group_name'] = os.environ['azure_resource_group_name']
- edge_conf['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- edge_conf['endpoint_name'] =
os.environ['endpoint_name'].lower().replace('_', '-')
+ edge_conf['project_name'] = os.environ['project_name'].lower()
+ edge_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
edge_conf['instance_name'] =
'{0}-{1}-{2}-edge'.format(edge_conf['service_base_name'],
edge_conf['project_name'], edge_conf['endpoint_name'])
@@ -48,8 +51,7 @@ if __name__ == "__main__":
try:
AzureActions().stop_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to stop edge.", str(err))
+ dlab.fab.append_result("Failed to stop edge.", str(err))
sys.exit(1)
try:
@@ -58,7 +60,7 @@ if __name__ == "__main__":
"Action": "Stop edge server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
index d785f23..6c2c038 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
@@ -22,10 +22,14 @@
#
******************************************************************************
import json
-import sys, time, os
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import sys
+import time
+import os
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import logging
+import traceback
def terminate_edge_node(resource_group_name, service_base_name, project_tag,
subnet_name, vpc_name):
@@ -39,7 +43,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate instance", str(err))
sys.exit(1)
print("Removing network interfaces")
@@ -52,7 +56,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove network interfaces", str(err))
sys.exit(1)
print("Removing static public IPs")
@@ -65,7 +69,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove static IPs", str(err))
sys.exit(1)
print("Removing disks")
@@ -78,7 +82,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove disks", str(err))
sys.exit(1)
print("Removing storage account")
@@ -91,7 +95,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove storage accounts", str(err))
sys.exit(1)
print("Deleting Data Lake Store directory")
@@ -104,7 +108,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove Data Lake", str(err))
sys.exit(1)
print("Removing security groups")
@@ -117,7 +121,7 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove security groups", str(err))
sys.exit(1)
print("Removing private subnet")
@@ -125,12 +129,13 @@ def terminate_edge_node(resource_group_name,
service_base_name, project_tag, sub
AzureActions().remove_subnet(resource_group_name, vpc_name,
subnet_name)
print("Private subnet {} has been terminated".format(subnet_name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove subnet", str(err))
sys.exit(1)
if __name__ == "__main__":
- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
+ os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
@@ -140,10 +145,12 @@ if __name__ == "__main__":
edge_conf = dict()
edge_conf['service_base_name'] = os.environ['conf_service_base_name']
edge_conf['resource_group_name'] = os.environ['azure_resource_group_name']
- edge_conf['user_name'] = os.environ['edge_user_name'].replace('_', '-')
- edge_conf['project_name'] = os.environ['project_name'].replace('_', '-')
- edge_conf['project_tag'] = os.environ['project_name'].replace('_', '-')
- edge_conf['private_subnet_name'] = edge_conf['service_base_name'] + "-" +
edge_conf['project_name'] + '-subnet'
+ edge_conf['user_name'] = os.environ['edge_user_name'].lower()
+ edge_conf['project_name'] = os.environ['project_name'].lower()
+ edge_conf['project_tag'] = edge_conf['project_name']
+ edge_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
+ edge_conf['private_subnet_name'] =
"{}-{}-{}-subnet".format(edge_conf['service_base_name'],
+
edge_conf['project_name'], edge_conf['endpoint_name'])
edge_conf['vpc_name'] = os.environ['azure_vpc_name']
@@ -153,10 +160,11 @@ if __name__ == "__main__":
try:
terminate_edge_node(edge_conf['resource_group_name'],
edge_conf['service_base_name'],
edge_conf['project_tag'],
edge_conf['private_subnet_name'], edge_conf['vpc_name'])
- except Exception as err:
+ except:
traceback.print_exc()
- append_result("Failed to terminate edge.", str(err))
- except:
+ raise Exception
+ except Exception as err:
+ dlab.fab.append_result("Failed to terminate edge.", str(err))
sys.exit(1)
try:
@@ -166,6 +174,6 @@ if __name__ == "__main__":
"Action": "Terminate edge node"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
index bf2f91e..1bb24f3 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
@@ -21,12 +21,16 @@
#
#
******************************************************************************
-from dlab.fab import *
-from dlab.actions_lib import *
-from dlab.meta_lib import *
-import sys, os
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import dlab.ssn_lib
+import sys
+import os
from fabric.api import *
-from dlab.ssn_lib import *
+import logging
+import traceback
+import json
def terminate_ssn_node(resource_group_name, service_base_name, vpc_name,
region):
@@ -37,7 +41,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_instance(resource_group_name, vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate instances", str(err))
sys.exit(1)
print("Removing network interfaces")
@@ -47,7 +51,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().delete_network_if(resource_group_name,
network_interface.name)
print("Network interface {} has been
removed".format(network_interface.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove network interfaces", str(err))
sys.exit(1)
print("Removing static public IPs")
@@ -57,7 +61,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().delete_static_public_ip(resource_group_name,
static_public_ip.name)
print("Static public IP {} has been
removed".format(static_public_ip.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove static IPs", str(err))
sys.exit(1)
print("Removing disks")
@@ -67,7 +71,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_disk(resource_group_name, disk.name)
print("Disk {} has been removed".format(disk.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove disks", str(err))
sys.exit(1)
print("Removing storage accounts")
@@ -77,7 +81,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_storage_account(resource_group_name,
storage_account.name)
print("Storage account {} has been
terminated".format(storage_account.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove storage accounts", str(err))
sys.exit(1)
print("Removing Data Lake Store")
@@ -87,7 +91,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().delete_datalake_store(resource_group_name,
datalake.name)
print("Data Lake Store {} has been
terminated".format(datalake.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove Data Lake", str(err))
sys.exit(1)
print("Removing images")
@@ -97,7 +101,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_image(resource_group_name, image.name)
print("Image {} has been removed".format(image.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove images", str(err))
sys.exit(1)
print("Removing security groups")
@@ -107,7 +111,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_security_group(resource_group_name,
sg.name)
print("Security group {} has been terminated".format(sg.name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove security groups", str(err))
sys.exit(1)
print("Removing VPC")
@@ -116,7 +120,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_vpc(resource_group_name, vpc_name)
print("VPC {} has been terminated".format(vpc_name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove VPC", str(err))
sys.exit(1)
print("Removing Resource Group")
@@ -125,7 +129,7 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
AzureActions().remove_resource_group(service_base_name, region)
print("Resource group {} has been terminated".format(vpc_name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove resource group", str(err))
sys.exit(1)
@@ -138,10 +142,10 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
ssn_conf = dict()
- ssn_conf['service_base_name'] = replace_multi_symbols(
- os.environ['conf_service_base_name'].replace('_', '-')[:12], '-', True)
- ssn_conf['resource_group_name'] = replace_multi_symbols(
- os.environ['azure_resource_group_name'].replace('_', '-')[:12], '-',
True)
+ ssn_conf['service_base_name'] =
dlab.fab.replace_multi_symbols(os.environ['conf_service_base_name'].lower()[:20],
+ '-', True)
+ ssn_conf['resource_group_name'] = dlab.fab.replace_multi_symbols(
+ os.environ['azure_resource_group_name'].lower()[:20], '-', True)
ssn_conf['region'] = os.environ['azure_region']
ssn_conf['vpc_name'] = os.environ['azure_vpc_name']
@@ -155,8 +159,7 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to terminate ssn.", str(err))
+ dlab.fab.append_result("Failed to terminate ssn.", str(err))
sys.exit(1)
try:
@@ -165,6 +168,6 @@ if __name__ == "__main__":
"Action": "Terminate ssn with all service_base_name
environment"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
index 311a805..a8cd1c8 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
@@ -24,13 +24,21 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
+import traceback
import uuid
+def clear_resources():
+
actions_lib.GCPActions().delete_dataproc_cluster(notebook_config['cluster_name'],
os.environ['gcp_region'])
+ actions_lib.GCPActions().remove_kernels(notebook_config['notebook_name'],
notebook_config['cluster_name'],
+ os.environ['dataproc_version'],
os.environ['conf_os_user'],
+ notebook_config['key_path'])
+
+
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
os.environ['request_id'])
@@ -42,13 +50,13 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
+ notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
- notebook_config['edge_user_name'] =
(os.environ['edge_user_name']).lower().replace('_', '-')
- notebook_config['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
- notebook_config['project_tag'] =
(os.environ['project_name']).lower().replace('_', '-')
- notebook_config['endpoint_tag'] =
(os.environ['endpoint_name']).lower().replace('_', '-')
- notebook_config['endpoint_name'] =
(os.environ['endpoint_name']).lower().replace('_', '-')
+ notebook_config['edge_user_name'] = (os.environ['edge_user_name']).lower()
+ notebook_config['project_name'] = (os.environ['project_name']).lower()
+ notebook_config['project_tag'] = notebook_config['project_name']
+ notebook_config['endpoint_name'] = (os.environ['endpoint_name']).lower()
+ notebook_config['endpoint_tag'] = notebook_config['endpoint_name']
notebook_config['tag_name'] = notebook_config['service_base_name'] + '-Tag'
notebook_config['bucket_name'] =
'{0}-{1}-{2}-bucket'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
@@ -78,11 +86,14 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]')
print('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]')
- params = "--bucket {} --cluster_name {} --dataproc_version {}
--keyfile {} --notebook_ip {} --region {} --edge_user_name {} --project_name {}
--os_user {} --edge_hostname {} --proxy_port {} --scala_version {}
--application {} --pip_mirror {}" \
+ params = "--bucket {} --cluster_name {} --dataproc_version {}
--keyfile {} --notebook_ip {} --region {} " \
+ "--edge_user_name {} --project_name {} --os_user {}
--edge_hostname {} --proxy_port {} " \
+ "--scala_version {} --application {} --pip_mirror {}" \
.format(notebook_config['bucket_name'],
notebook_config['cluster_name'], os.environ['dataproc_version'],
notebook_config['key_path'],
notebook_config['notebook_ip'], os.environ['gcp_region'],
- notebook_config['edge_user_name'],
notebook_config['project_name'], os.environ['conf_os_user'],
edge_instance_hostname, '3128',
- os.environ['notebook_scala_version'],
os.environ['application'], os.environ['conf_pypi_mirror'])
+ notebook_config['edge_user_name'],
notebook_config['project_name'], os.environ['conf_os_user'],
+ edge_instance_hostname, '3128',
os.environ['notebook_scala_version'], os.environ['application'],
+ os.environ['conf_pypi_mirror'])
try:
local("~/scripts/{}_{}.py {}".format(application,
'install_dataengine-service_kernels', params))
actions_lib.GCPActions().update_dataproc_cluster(notebook_config['cluster_name'],
@@ -91,11 +102,8 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed installing Dataproc kernels.", str(err))
-
actions_lib.GCPActions().delete_dataproc_cluster(notebook_config['cluster_name'],
os.environ['gcp_region'])
-
actions_lib.GCPActions().remove_kernels(notebook_config['notebook_name'],
notebook_config['cluster_name'],
-
os.environ['dataproc_version'], os.environ['conf_os_user'],
notebook_config['key_path'])
+ clear_resources()
+ dlab.fab.append_result("Failed installing Dataproc kernels.", str(err))
sys.exit(1)
try:
@@ -113,11 +121,8 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to configure Spark.", str(err))
-
actions_lib.GCPActions().delete_dataproc_cluster(notebook_config['cluster_name'],
os.environ['gcp_region'])
-
actions_lib.GCPActions().remove_kernels(notebook_config['notebook_name'],
notebook_config['cluster_name'],
-
os.environ['dataproc_version'], os.environ['conf_os_user'],
notebook_config['key_path'])
+ dlab.fab.append_result("Failed to configure Spark.", str(err))
+ clear_resources()
sys.exit(1)
try:
@@ -127,6 +132,7 @@ if __name__ == "__main__":
"Action": "Configure notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ clear_resources()
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
index adf1f0b..6b03c90 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
@@ -24,13 +24,21 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
+def clear_resources():
+ for i in range(notebook_config['instance_count'] - 1):
+ slave_name = notebook_config['slave_node_name'] + '{}'.format(i + 1)
+ GCPActions().remove_instance(slave_name, notebook_config['zone'])
+ GCPActions().remove_instance(notebook_config['master_node_name'],
notebook_config['zone'])
+
+
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
os.environ['request_id'])
@@ -43,50 +51,48 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- try:
- notebook_config['exploratory_name'] =
os.environ['exploratory_name'].lower().replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ notebook_config['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
notebook_config['exploratory_name'] = ''
- try:
- notebook_config['computational_name'] =
os.environ['computational_name'].lower().replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ notebook_config['computational_name'] =
os.environ['computational_name'].lower()
+ else:
notebook_config['computational_name'] = ''
notebook_config['service_base_name'] =
os.environ['conf_service_base_name']
notebook_config['region'] = os.environ['gcp_region']
notebook_config['zone'] = os.environ['gcp_zone']
- notebook_config['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- notebook_config['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- notebook_config['cluster_name'] = notebook_config['service_base_name']
+ '-' + notebook_config['project_name'] + \
- '-de-' +
notebook_config['exploratory_name'] + '-' + \
- notebook_config['computational_name']
+ notebook_config['user_name'] = os.environ['edge_user_name'].lower()
+ notebook_config['project_name'] = os.environ['project_name'].lower()
+ notebook_config['endpoint_name'] = os.environ['endpoint_name'].lower()
+ notebook_config['cluster_name'] =
"{}-{}-{}-de-{}".format(notebook_config['service_base_name'],
+
notebook_config['project_name'],
+
notebook_config['endpoint_name'],
+
notebook_config['exploratory_name'])
notebook_config['master_node_name'] = notebook_config['cluster_name']
+ '-m'
notebook_config['slave_node_name'] = notebook_config['cluster_name'] +
'-s'
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
- notebook_config['key_path'] = os.environ['conf_key_dir'] + '/' +
os.environ['conf_key_name'] + '.pem'
+ notebook_config['key_path'] =
'{}/{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['instance_count'] =
int(os.environ['dataengine_instance_count'])
try:
notebook_config['spark_master_ip'] =
GCPMeta().get_private_ip_address(notebook_config['master_node_name'])
notebook_config['notebook_ip'] =
GCPMeta().get_private_ip_address(notebook_config['notebook_name'])
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to get instance IP address",
str(err))
sys.exit(1)
notebook_config['spark_master_url'] =
'spark://{}:7077'.format(notebook_config['spark_master_ip'])
except Exception as err:
- print('Error: {0}'.format(err))
- for i in range(notebook_config['instance_count'] - 1):
- slave_name = notebook_config['slave_node_name'] + '{}'.format(i+1)
- GCPActions().remove_instance(slave_name, notebook_config['zone'])
- GCPActions().remove_instance(notebook_config['master_node_name'],
notebook_config['zone'])
- append_result("Failed to generate infrastructure names", str(err))
+ clear_resources()
+ dlab.fab.append_result("Failed to generate infrastructure names",
str(err))
sys.exit(1)
try:
logging.info('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]')
print('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]')
- params = "--cluster_name {0} --spark_version {1} --hadoop_version {2}
--os_user {3} --spark_master {4} --keyfile {5}" \
- " --notebook_ip {6} --spark_master_ip {7}".\
+ params = "--cluster_name {0} --spark_version {1} --hadoop_version {2}
--os_user {3} --spark_master {4}" \
+ " --keyfile {5} --notebook_ip {6} --spark_master_ip {7}".\
format(notebook_config['cluster_name'],
os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'],
notebook_config['dlab_ssh_user'],
notebook_config['spark_master_url'],
notebook_config['key_path'],
@@ -97,12 +103,8 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- for i in range(notebook_config['instance_count'] - 1):
- slave_name = notebook_config['slave_node_name'] + '{}'.format(i+1)
- GCPActions().remove_instance(slave_name, notebook_config['zone'])
- GCPActions().remove_instance(notebook_config['master_node_name'],
notebook_config['zone'])
- append_result("Failed installing Dataengine kernels.", str(err))
+ clear_resources()
+ dlab.fab.append_result("Failed installing Dataengine kernels.",
str(err))
sys.exit(1)
try:
@@ -122,12 +124,8 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- for i in range(notebook_config['instance_count'] - 1):
- slave_name = notebook_config['slave_node_name'] + '{}'.format(i+1)
- GCPActions().remove_instance(slave_name, notebook_config['zone'])
- GCPActions().remove_instance(notebook_config['master_node_name'],
notebook_config['zone'])
- append_result("Failed to configure Spark.", str(err))
+ clear_resources()
+ dlab.fab.append_result("Failed to configure Spark.", str(err))
sys.exit(1)
try:
@@ -136,6 +134,7 @@ if __name__ == "__main__":
"Action": "Configure notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ clear_resources()
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
index b9c8a08..b684dc2 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
@@ -24,9 +24,10 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
import argparse
@@ -42,7 +43,7 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
+ notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -54,7 +55,7 @@ if __name__ == "__main__":
GCPActions().start_instance(notebook_config['notebook_name'],
notebook_config['zone'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to start notebook.", str(err))
+ dlab.fab.append_result("Failed to start notebook.", str(err))
raise Exception
except:
sys.exit(1)
@@ -70,7 +71,7 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('manage_git_creds', params))
except Exception as err:
traceback.print_exc()
- append_result("Failed to setup git credentials.", str(err))
+ dlab.fab.append_result("Failed to setup git credentials.",
str(err))
raise Exception
except:
sys.exit(1)
@@ -84,7 +85,7 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
except Exception as err:
traceback.print_exc()
- append_result("Failed to update last activity time.", str(err))
+ dlab.fab.append_result("Failed to update last activity time.",
str(err))
raise Exception
except:
sys.exit(1)
@@ -101,8 +102,6 @@ if __name__ == "__main__":
"Action": "Start up notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
-
-
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
b/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
index f336a0b..72db577 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
@@ -24,9 +24,9 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import os
import uuid
import argparse
@@ -54,7 +54,7 @@ def stop_notebook(instance_name, bucket_name, region, zone,
ssh_user, key_path,
else:
print("There are no Dataproc clusters to terminate.")
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate dataproc", str(err))
sys.exit(1)
print("Stopping data engine cluster")
@@ -71,15 +71,14 @@ def stop_notebook(instance_name, bucket_name, region, zone,
ssh_user, key_path,
print("There are no data engine clusters to terminate.")
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to stop dataengine cluster", str(err))
sys.exit(1)
print("Stopping notebook")
try:
GCPActions().stop_instance(instance_name, zone)
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to stop notebook.", str(err))
+ dlab.fab.append_result("Failed to stop instance", str(err))
sys.exit(1)
@@ -94,10 +93,10 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
- notebook_config['edge_user_name'] =
(os.environ['edge_user_name']).lower().replace('_', '-')
- notebook_config['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
- notebook_config['endpoint_name'] =
(os.environ['endpoint_name']).lower().replace('_', '-')
+ notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
+ notebook_config['edge_user_name'] = (os.environ['edge_user_name']).lower()
+ notebook_config['project_name'] = (os.environ['project_name']).lower()
+ notebook_config['endpoint_name'] = (os.environ['endpoint_name']).lower()
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
notebook_config['bucket_name'] =
'{0}-{1}-{2}-bucket'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
@@ -115,7 +114,7 @@ if __name__ == "__main__":
notebook_config['project_name'])
except Exception as err:
print('Error: {0}'.format(err))
- append_result("Failed to stop notebook.", str(err))
+ dlab.fab.append_result("Failed to stop notebook.", str(err))
sys.exit(1)
try:
@@ -124,7 +123,6 @@ if __name__ == "__main__":
"Action": "Stop notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
-
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
index 4b243a0..e6b6d02 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
@@ -24,9 +24,10 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
@@ -47,7 +48,7 @@ def terminate_nb(instance_name, bucket_name, region, zone,
user_name):
else:
print("There are no Dataproc clusters to terminate.")
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate dataproc", str(err))
sys.exit(1)
print("Terminating data engine cluster")
@@ -64,15 +65,14 @@ def terminate_nb(instance_name, bucket_name, region, zone,
user_name):
print("There are no data engine clusters to terminate.")
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate dataengine", str(err))
sys.exit(1)
print("Terminating notebook")
try:
GCPActions().remove_instance(instance_name, zone)
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to terminate notebook.", str(err))
+ dlab.fab.append_result("Failed to terminate instance", str(err))
sys.exit(1)
@@ -86,10 +86,10 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
notebook_config = dict()
- notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
- notebook_config['edge_user_name'] =
(os.environ['edge_user_name']).lower().replace('_', '-')
- notebook_config['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
- notebook_config['endpoint_name'] =
(os.environ['endpoint_name']).lower().replace('_', '-')
+ notebook_config['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
+ notebook_config['edge_user_name'] = (os.environ['edge_user_name']).lower()
+ notebook_config['project_name'] = (os.environ['project_name']).lower()
+ notebook_config['endpoint_name'] = (os.environ['endpoint_name']).lower()
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
notebook_config['bucket_name'] =
'{0}-{1}-{2}-bucket'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
@@ -106,7 +106,7 @@ if __name__ == "__main__":
notebook_config['project_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to terminate notebook.", str(err))
+ dlab.fab.append_result("Failed to terminate notebook.", str(err))
raise Exception
except:
sys.exit(1)
@@ -117,6 +117,6 @@ if __name__ == "__main__":
"Action": "Terminate notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
index 07757c8..c8ea420 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
@@ -158,7 +158,7 @@ if __name__ == "__main__":
else:
data_engine['vpc_name'] = os.environ['gcp_vpc_name']
except KeyError:
- data_engine['vpc_name'] =
'{}-ssn-vpc'.format(data_engine['service_base_name'])
+ data_engine['vpc_name'] =
'{}-vpc'.format(data_engine['service_base_name'])
if 'exploratory_name' in os.environ:
data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
else:
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index cfea534..342b943 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -69,7 +69,7 @@ if __name__ == "__main__":
else:
data_engine['vpc_name'] = os.environ['gcp_vpc_name']
except KeyError:
- data_engine['vpc_name'] =
'{}-ssn-vpc'.format(data_engine['service_base_name'])
+ data_engine['vpc_name'] =
'{}-vpc'.format(data_engine['service_base_name'])
if 'exploratory_name' in os.environ:
data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
else:
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
index 0e40ed9..d82633e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
@@ -24,9 +24,10 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
@@ -39,7 +40,7 @@ def start_data_engine(zone, cluster_name):
for i in instances['items']:
GCPActions().start_instance(i['name'], zone)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to start dataengine", str(err))
sys.exit(1)
@@ -53,21 +54,23 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
data_engine = dict()
- try:
- data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower().replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
data_engine['exploratory_name'] = ''
- try:
- data_engine['computational_name'] =
os.environ['computational_name'].lower().replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ data_engine['computational_name'] =
os.environ['computational_name'].lower()
+ else:
data_engine['computational_name'] = ''
data_engine['service_base_name'] = os.environ['conf_service_base_name']
data_engine['zone'] = os.environ['gcp_zone']
- data_engine['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- data_engine['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- data_engine['cluster_name'] = \
- data_engine['service_base_name'] + '-' + data_engine['project_name'] +
'-de-' + \
- data_engine['exploratory_name'] + '-' +
data_engine['computational_name']
+ data_engine['user_name'] = os.environ['edge_user_name'].lower()
+ data_engine['project_name'] = os.environ['project_name'].lower()
+ data_engine['endpoint_name'] = os.environ['endpoint_name'].lower()
+ data_engine['cluster_name'] =
"{}-{}-{}-de-{}".format(data_engine['service_base_name'],
+
data_engine['project_name'],
+
data_engine['endpoint_name'],
+
data_engine['exploratory_name'])
try:
logging.info('[STARTING DATA ENGINE]')
print('[STARTING DATA ENGINE]')
@@ -75,7 +78,7 @@ if __name__ == "__main__":
start_data_engine(data_engine['zone'], data_engine['cluster_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to start Data Engine.", str(err))
+ dlab.fab.append_result("Failed to start Data Engine.", str(err))
raise Exception
except:
sys.exit(1)
@@ -84,7 +87,7 @@ if __name__ == "__main__":
logging.info('[UPDATE LAST ACTIVITY TIME]')
print('[UPDATE LAST ACTIVITY TIME]')
data_engine['computational_id'] = data_engine['cluster_name'] + '-m'
- data_engine['tag_name'] = data_engine['service_base_name'] + '-Tag'
+ data_engine['tag_name'] = data_engine['service_base_name'] + '-tag'
data_engine['notebook_ip'] =
GCPMeta().get_private_ip_address(os.environ['notebook_instance_name'])
data_engine['computational_ip'] =
GCPMeta().get_private_ip_address(data_engine['computational_id'])
data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
@@ -95,18 +98,17 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('update_inactivity_on_start',
params))
except Exception as err:
traceback.print_exc()
- append_result("Failed to update last activity time.", str(err))
+ dlab.fab.append_result("Failed to update last activity time.",
str(err))
raise Exception
except:
sys.exit(1)
-
try:
with open("/root/result.json", 'w') as result:
res = {"service_base_name": data_engine['service_base_name'],
"Action": "Start Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
\ No newline at end of file
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
index 2396600..abeb335 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
@@ -24,9 +24,10 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
@@ -39,7 +40,7 @@ def stop_data_engine(zone, cluster_name):
for i in instances['items']:
GCPActions().stop_instance(i['name'], zone)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to stop dataengine", str(err))
sys.exit(1)
@@ -53,21 +54,23 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
data_engine = dict()
- try:
- data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower().replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
data_engine['exploratory_name'] = ''
- try:
- data_engine['computational_name'] =
os.environ['computational_name'].lower().replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ data_engine['computational_name'] =
os.environ['computational_name'].lower()
+ else:
data_engine['computational_name'] = ''
data_engine['service_base_name'] = os.environ['conf_service_base_name']
data_engine['zone'] = os.environ['gcp_zone']
- data_engine['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- data_engine['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- data_engine['cluster_name'] = \
- data_engine['service_base_name'] + '-' + data_engine['project_name'] +
'-de-' + \
- data_engine['exploratory_name'] + '-' +
data_engine['computational_name']
+ data_engine['user_name'] = os.environ['edge_user_name'].lower()
+ data_engine['project_name'] = os.environ['project_name'].lower()
+ data_engine['endpoint_name'] = os.environ['endpoint_name'].lower()
+ data_engine['cluster_name'] =
"{}-{}-{}-de-{}".format(data_engine['service_base_name'],
+
data_engine['project_name'],
+
data_engine['endpoint_name'],
+
data_engine['exploratory_name'])
try:
logging.info('[STOPPING DATA ENGINE]')
print('[STOPPING DATA ENGINE]')
@@ -75,7 +78,7 @@ if __name__ == "__main__":
stop_data_engine(data_engine['zone'], data_engine['cluster_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to stop Data Engine.", str(err))
+ dlab.fab.append_result("Failed to stop Data Engine.", str(err))
raise Exception
except:
sys.exit(1)
@@ -85,6 +88,6 @@ if __name__ == "__main__":
"Action": "Stop Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
index f50ffb2..2cada80 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
@@ -24,9 +24,10 @@
import logging
import json
import sys
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import traceback
import os
import uuid
@@ -39,14 +40,14 @@ def terminate_data_engine(zone, notebook_name, os_user,
key_path, cluster_name):
for i in instances['items']:
GCPActions().remove_instance(i['name'], zone)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate dataengine", str(err))
sys.exit(1)
print("Removing Data Engine kernels from notebook")
try:
- remove_dataengine_kernels(notebook_name, os_user, key_path,
cluster_name)
+ dlab.actions_lib.remove_dataengine_kernels(notebook_name, os_user,
key_path, cluster_name)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove dataengine kernels from
notebook", str(err))
sys.exit(1)
@@ -60,23 +61,25 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
data_engine = dict()
- try:
- data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower().replace('_', '-')
- except:
+ if 'exploratory_name' in os.environ:
+ data_engine['exploratory_name'] =
os.environ['exploratory_name'].lower()
+ else:
data_engine['exploratory_name'] = ''
- try:
- data_engine['computational_name'] =
os.environ['computational_name'].lower().replace('_', '-')
- except:
+ if 'computational_name' in os.environ:
+ data_engine['computational_name'] =
os.environ['computational_name'].lower()
+ else:
data_engine['computational_name'] = ''
data_engine['service_base_name'] = os.environ['conf_service_base_name']
data_engine['zone'] = os.environ['gcp_zone']
- data_engine['user_name'] =
os.environ['edge_user_name'].lower().replace('_', '-')
- data_engine['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- data_engine['cluster_name'] = \
- data_engine['service_base_name'] + '-' + data_engine['project_name'] +
'-de-' + \
- data_engine['exploratory_name'] + '-' +
data_engine['computational_name']
+ data_engine['user_name'] = os.environ['edge_user_name'].lower()
+ data_engine['project_name'] = os.environ['project_name'].lower()
+ data_engine['endpoint_name'] = os.environ['endpoint_name'].lower()
+ data_engine['cluster_name'] =
"{}-{}-{}-de-{}".format(data_engine['service_base_name'],
+
data_engine['project_name'],
+
data_engine['endpoint_name'],
+
data_engine['exploratory_name'])
data_engine['notebook_name'] = os.environ['notebook_instance_name']
- data_engine['key_path'] = os.environ['conf_key_dir'] + '/' +
os.environ['conf_key_name'] + '.pem'
+ data_engine['key_path'] = '{}/{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
try:
@@ -87,7 +90,7 @@ if __name__ == "__main__":
data_engine['key_path'],
data_engine['cluster_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to terminate Data Engine.", str(err))
+ dlab.fab.append_result("Failed to terminate Data Engine.",
str(err))
raise Exception
except:
sys.exit(1)
@@ -98,6 +101,6 @@ if __name__ == "__main__":
"Action": "Terminate Data Engine"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
b/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
index 06085dd..fedfe33 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
@@ -21,9 +21,13 @@
#
#
******************************************************************************
-from dlab.fab import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import logging
+import os
import sys
+import json
if __name__ == "__main__":
@@ -37,22 +41,23 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
edge_conf = dict()
- edge_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
- edge_conf['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
- edge_conf['endpoint_name'] =
(os.environ['endpoint_name']).lower().replace('_', '-')
+ edge_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
+ edge_conf['project_name'] = (os.environ['project_name']).lower()
+ edge_conf['endpoint_name'] = (os.environ['endpoint_name']).lower()
edge_conf['instance_name'] =
'{0}-{1}-{2}-edge'.format(edge_conf['service_base_name'],
edge_conf['project_name'], edge_conf['endpoint_name'])
edge_conf['region'] = os.environ['gcp_region']
edge_conf['zone'] = os.environ['gcp_zone']
- edge_conf['static_address_name'] =
'{0}-{1}-ip'.format(edge_conf['service_base_name'], edge_conf['project_name'])
+ edge_conf['static_address_name'] =
'{0}-{1}-{2}-ip'.format(edge_conf['service_base_name'],
+
edge_conf['project_name'],
+
edge_conf['endpoint_name'])
logging.info('[START EDGE]')
print('[START EDGE]')
try:
GCPActions().start_instance(edge_conf['instance_name'],
edge_conf['zone'])
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to start edge.", str(err))
+ dlab.fab.append_result("Failed to start edge.", str(err))
sys.exit(1)
try:
@@ -74,7 +79,7 @@ if __name__ == "__main__":
"Action": "Start up notebook server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
b/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
index 5342d8a..adff595 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
@@ -21,9 +21,13 @@
#
#
******************************************************************************
-from dlab.fab import *
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import os
+import logging
import sys
+import json
if __name__ == "__main__":
@@ -36,10 +40,10 @@ if __name__ == "__main__":
print('Generating infrastructure names and tags')
edge_conf = dict()
- edge_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
+ edge_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
edge_conf['zone'] = os.environ['gcp_zone']
- edge_conf['project_name'] =
(os.environ['project_name']).lower().replace('_', '-')
- edge_conf['endpoint_name'] =
os.environ['endpoint_name'].lower().replace('_', '-')
+ edge_conf['project_name'] = (os.environ['project_name']).lower()
+ edge_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
edge_conf['instance_name'] =
'{0}-{1}-{2}-edge'.format(edge_conf['service_base_name'],
edge_conf['project_name'], edge_conf['endpoint_name'])
@@ -48,8 +52,7 @@ if __name__ == "__main__":
try:
GCPActions().stop_instance(edge_conf['instance_name'],
edge_conf['zone'])
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to stop edge.", str(err))
+ dlab.fab.append_result("Failed to stop edge.", str(err))
sys.exit(1)
try:
@@ -58,7 +61,6 @@ if __name__ == "__main__":
"Action": "Stop edge server"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
-
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
b/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
index 6385ffd..dbe9c34 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
@@ -22,10 +22,14 @@
#
******************************************************************************
import json
-from dlab.fab import *
-from dlab.meta_lib import *
-import sys, time, os
-from dlab.actions_lib import *
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import sys
+import time
+import os
+import traceback
+import logging
def terminate_edge_node(user_name, service_base_name, region, zone):
@@ -43,7 +47,7 @@ def terminate_edge_node(user_name, service_base_name, region,
zone):
else:
print("There are no Dataproc clusters to terminate.")
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate dataproc", str(err))
sys.exit(1)
print("Terminating EDGE and notebook instances")
@@ -57,7 +61,7 @@ def terminate_edge_node(user_name, service_base_name, region,
zone):
if 'user' in i['labels'] and user_name == i['labels']['user']:
GCPActions().remove_instance(i['name'], zone)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate instances", str(err))
sys.exit(1)
print("Removing static addresses")
@@ -68,7 +72,7 @@ def terminate_edge_node(user_name, service_base_name, region,
zone):
if bool(set(targets) & set([i['name']])):
GCPActions().remove_static_address(i['name'], region)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove static IPs", str(err))
sys.exit(1)
print("Removing storage bucket")
@@ -79,7 +83,7 @@ def terminate_edge_node(user_name, service_base_name, region,
zone):
if bool(set(targets) & set([i['name']])):
GCPActions().remove_bucket(i['name'])
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove buckets", str(err))
sys.exit(1)
print("Removing firewalls")
@@ -90,7 +94,7 @@ def terminate_edge_node(user_name, service_base_name, region,
zone):
if bool(set(targets) & set(i['targetTags'])):
GCPActions().remove_firewall(i['name'])
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove security groups", str(err))
sys.exit(1)
print("Removing Service accounts and roles")
@@ -104,7 +108,7 @@ def terminate_edge_node(user_name, service_base_name,
region, zone):
if role.startswith(service_base_name):
GCPActions().remove_role(role)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove service accounts and roles",
str(err))
sys.exit(1)
print("Removing subnets")
@@ -118,12 +122,13 @@ def terminate_edge_node(user_name, service_base_name,
region, zone):
if bool(set(targets) & set([i['name']])):
GCPActions().remove_subnet(i['name'], region)
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove subnets", str(err))
sys.exit(1)
if __name__ == "__main__":
- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
+ os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
@@ -132,8 +137,8 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
edge_conf = dict()
- edge_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower().replace('_', '-')
- edge_conf['edge_user_name'] =
(os.environ['edge_user_name']).lower().replace('_', '-')
+ edge_conf['service_base_name'] =
(os.environ['conf_service_base_name']).lower()
+ edge_conf['edge_user_name'] = (os.environ['edge_user_name']).lower()
edge_conf['region'] = os.environ['gcp_region']
edge_conf['zone'] = os.environ['gcp_zone']
@@ -145,9 +150,9 @@ if __name__ == "__main__":
edge_conf['region'], edge_conf['zone'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to terminate edge.", str(err))
- except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate edge.", str(err))
+ raise Exception
+ except:
sys.exit(1)
try:
@@ -157,6 +162,6 @@ if __name__ == "__main__":
"Action": "Terminate edge node"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index 66163d9..98c3d10 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -81,7 +81,7 @@ if __name__ == "__main__":
ssn_conf['pre_defined_vpc'] = True
ssn_conf['vpc_name'] = os.environ['gcp_vpc_name']
except KeyError:
- ssn_conf['vpc_name'] =
'{}-ssn-vpc'.format(ssn_conf['service_base_name'])
+ ssn_conf['vpc_name'] =
'{}-vpc'.format(ssn_conf['service_base_name'])
try:
if os.environ['gcp_subnet_name'] == '':
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
index c033429..9a59258 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
@@ -21,11 +21,16 @@
#
#
******************************************************************************
-from dlab.fab import *
-from dlab.actions_lib import *
-import sys, os
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
+import dlab.ssn_lib
+import sys
+import os
+import logging
+import json
+import traceback
from fabric.api import *
-from dlab.ssn_lib import *
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'],
os.environ['request_id'])
@@ -36,8 +41,8 @@ if __name__ == "__main__":
# generating variables dictionary
print('Generating infrastructure names and tags')
ssn_conf = dict()
- ssn_conf['service_base_name'] = replace_multi_symbols(
- os.environ['conf_service_base_name'].lower().replace('_', '-')[:12],
'-', True)
+ ssn_conf['service_base_name'] = dlab.fab.replace_multi_symbols(
+ os.environ['conf_service_base_name'].lower()[:20], '-', True)
ssn_conf['region'] = os.environ['gcp_region']
ssn_conf['zone'] = os.environ['gcp_zone']
pre_defined_vpc = False
@@ -48,7 +53,7 @@ if __name__ == "__main__":
pre_defined_vpc = True
ssn_conf['vpc_name'] = os.environ['gcp_vpc_name']
except KeyError:
- ssn_conf['vpc_name'] =
'{}-ssn-vpc'.format(ssn_conf['service_base_name'])
+ ssn_conf['vpc_name'] = '{}-vpc'.format(ssn_conf['service_base_name'])
try:
logging.info('[TERMINATE SSN]')
@@ -61,8 +66,7 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- print('Error: {0}'.format(err))
- append_result("Failed to terminate ssn.", str(err))
+ dlab.fab.append_result("Failed to terminate ssn.", str(err))
sys.exit(1)
try:
@@ -71,6 +75,6 @@ if __name__ == "__main__":
"Action": "Terminate ssn with all service_base_name
environment"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
\ No newline at end of file
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]