This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-1546
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1546 by this push:
new 52b6642 [DLAB-1546]: added name convention changes
52b6642 is described below
commit 52b6642cff6c1eeceebe0c6389a36fab2143c7ee
Author: Oleh Martushevskyi <[email protected]>
AuthorDate: Thu Feb 27 16:31:18 2020 +0200
[DLAB-1546]: added name convention changes
---
.../scripts/azure/common_create_notebook_image.py | 32 ++--
.../azure/common_notebook_configure_dataengine.py | 10 +-
.../scripts/azure/common_prepare_notebook.py | 16 +-
.../general/scripts/azure/common_start_notebook.py | 12 +-
.../general/scripts/azure/common_stop_notebook.py | 10 +-
.../scripts/azure/common_terminate_notebook.py | 10 +-
.../azure/common_terminate_notebook_image.py | 6 +-
.../general/scripts/azure/dataengine_configure.py | 36 ++--
.../general/scripts/azure/dataengine_prepare.py | 28 +--
.../src/general/scripts/azure/dataengine_start.py | 16 +-
.../src/general/scripts/azure/dataengine_stop.py | 6 +-
.../general/scripts/azure/dataengine_terminate.py | 8 +-
.../src/general/scripts/azure/edge_configure.py | 66 +++----
.../src/general/scripts/azure/edge_start.py | 12 +-
.../src/general/scripts/azure/edge_stop.py | 4 +-
.../src/general/scripts/azure/project_prepare.py | 206 +++++++++++----------
.../src/general/scripts/azure/project_terminate.py | 93 ++++++----
.../src/general/scripts/azure/ssn_configure.py | 44 ++---
.../src/general/scripts/azure/ssn_prepare.py | 2 +-
.../src/general/scripts/azure/ssn_terminate.py | 42 +++--
20 files changed, 355 insertions(+), 304 deletions(-)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index 883ce95..10e6fb9 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@ -32,6 +32,8 @@ import json
if __name__ == "__main__":
try:
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
image_conf = dict()
image_conf['service_base_name'] = os.environ['conf_service_base_name']
image_conf['resource_group_name'] =
os.environ['azure_resource_group_name']
@@ -58,39 +60,39 @@ if __name__ == "__main__":
"FIN": image_conf['full_image_name'],
os.environ['conf_billing_tag_key']:
os.environ['conf_billing_tag_value']}
- instance_hostname =
AzureMeta().get_private_ip_address(image_conf['resource_group_name'],
-
image_conf['instance_name'])
+ instance_hostname =
AzureMeta.get_private_ip_address(image_conf['resource_group_name'],
+
image_conf['instance_name'])
edge_instance_name =
'{0}-{1}-{2}-edge'.format(image_conf['service_base_name'],
image_conf['project_name'],
image_conf['endpoint_name'])
- edge_instance_hostname =
AzureMeta().get_private_ip_address(image_conf['resource_group_name'],
+ edge_instance_hostname =
AzureMeta.get_private_ip_address(image_conf['resource_group_name'],
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
- instance = AzureMeta().get_instance(image_conf['resource_group_name'],
image_conf['instance_name'])
+ instance = AzureMeta.get_instance(image_conf['resource_group_name'],
image_conf['instance_name'])
os.environ['azure_notebook_instance_size'] =
instance.hardware_profile.vm_size
os.environ['exploratory_name'] = instance.tags['Exploratory']
os.environ['notebook_image_name'] = image_conf['image_name']
- image = AzureMeta().get_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
+ image = AzureMeta.get_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
if image == '':
print('Creating image from existing notebook.')
dlab.actions_lib.prepare_vm_for_image(True,
image_conf['dlab_ssh_user'], instance_hostname, keyfile_name)
-
AzureActions().create_image_from_instance(image_conf['resource_group_name'],
-
image_conf['instance_name'],
-
os.environ['azure_region'],
-
image_conf['full_image_name'],
-
json.dumps(image_conf['tags']))
+
AzureActions.create_image_from_instance(image_conf['resource_group_name'],
+
image_conf['instance_name'],
+ os.environ['azure_region'],
+
image_conf['full_image_name'],
+
json.dumps(image_conf['tags']))
print("Image was successfully created.")
try:
local("~/scripts/{}.py".format('common_prepare_notebook'))
instance_running = False
while not instance_running:
- if
AzureMeta().get_instance_status(image_conf['resource_group_name'],
-
image_conf['instance_name']) == 'running':
+ if
AzureMeta.get_instance_status(image_conf['resource_group_name'],
+
image_conf['instance_name']) == 'running':
instance_running = True
- instance_hostname =
AzureMeta().get_private_ip_address(image_conf['resource_group_name'],
-
image_conf['instance_name'])
+ instance_hostname =
AzureMeta.get_private_ip_address(image_conf['resource_group_name'],
+
image_conf['instance_name'])
dlab.actions_lib.remount_azure_disk(True,
image_conf['dlab_ssh_user'], instance_hostname, keyfile_name)
dlab.fab.set_git_proxy(image_conf['dlab_ssh_user'],
instance_hostname, keyfile_name,
'http://{}:3128'.format(edge_instance_hostname))
@@ -101,7 +103,7 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('common_configure_proxy',
params))
print("Image was successfully created. It's name is
{}".format(image_conf['full_image_name']))
except Exception as err:
-
AzureActions().remove_instance(image_conf['resource_group_name'],
image_conf['instance_name'])
+
AzureActions.remove_instance(image_conf['resource_group_name'],
image_conf['instance_name'])
dlab.fab.append_result("Failed to create instance from
image.", str(err))
sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
index 6b67ffe..f28b66c 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
@@ -34,8 +34,8 @@ import uuid
def clear_resources():
for i in range(notebook_config['instance_count'] - 1):
slave_name = notebook_config['slave_node_name'] + '{}'.format(i + 1)
- AzureActions().remove_instance(notebook_config['resource_group_name'],
slave_name)
- AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['master_node_name'])
+ AzureActions.remove_instance(notebook_config['resource_group_name'],
slave_name)
+ AzureActions.remove_instance(notebook_config['resource_group_name'],
notebook_config['master_node_name'])
if __name__ == "__main__":
@@ -48,6 +48,8 @@ if __name__ == "__main__":
try:
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
notebook_config = dict()
if 'exploratory_name' in os.environ:
@@ -77,9 +79,9 @@ if __name__ == "__main__":
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['instance_count'] =
int(os.environ['dataengine_instance_count'])
try:
- notebook_config['spark_master_ip'] =
AzureMeta().get_private_ip_address(
+ notebook_config['spark_master_ip'] =
AzureMeta.get_private_ip_address(
notebook_config['resource_group_name'],
notebook_config['master_node_name'])
- notebook_config['notebook_ip'] =
AzureMeta().get_private_ip_address(
+ notebook_config['notebook_ip'] = AzureMeta.get_private_ip_address(
notebook_config['resource_group_name'],
notebook_config['notebook_name'])
except Exception as err:
dlab.fab.append_result("Failed to get instance IP address",
str(err))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
index be276e0..4b15d64 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
@@ -43,6 +43,8 @@ if __name__ == "__main__":
# generating variables dictionary
try:
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
notebook_config = dict()
notebook_config['user_name'] = os.environ['edge_user_name'].lower()
notebook_config['project_name'] = os.environ['project_name'].lower()
@@ -112,7 +114,7 @@ if __name__ == "__main__":
else
notebook_config['expected_image_name'])(str(os.environ.get('notebook_image_name')))
print('Searching pre-configured images')
notebook_config['image_name'] =
os.environ['azure_{}_image_name'.format(os.environ['conf_os_family'])]
- if AzureMeta().get_image(notebook_config['resource_group_name'],
notebook_config['notebook_image_name']):
+ if AzureMeta.get_image(notebook_config['resource_group_name'],
notebook_config['notebook_image_name']):
notebook_config['image_name'] =
notebook_config['notebook_image_name']
notebook_config['image_type'] = 'pre-configured'
print('Pre-configured image found. Using:
{}'.format(notebook_config['notebook_image_name']))
@@ -125,15 +127,15 @@ if __name__ == "__main__":
sys.exit(1)
try:
- edge_status =
AzureMeta().get_instance_status(notebook_config['resource_group_name'],
-
'{0}-{1}-{2}-edge'.format(os.environ['conf_service_base_name'],
-
notebook_config['project_name'],
-
notebook_config['endpoint_name']))
+ edge_status =
AzureMeta.get_instance_status(notebook_config['resource_group_name'],
+
'{0}-{1}-{2}-edge'.format(os.environ['conf_service_base_name'],
+
notebook_config['project_name'],
+
notebook_config['endpoint_name']))
if edge_status != 'running':
logging.info('ERROR: Edge node is unavailable! Aborting...')
print('ERROR: Edge node is unavailable! Aborting...')
- ssn_hostname =
AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
+ ssn_hostname =
AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
os.environ['conf_service_base_name'] + '-ssn')
dlab.fab.put_resource_status('edge', 'Unavailable',
os.environ['ssn_dlab_path'], os.environ['conf_os_user'],
ssn_hostname)
@@ -171,7 +173,7 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
try:
-
AzureActions().remove_instance(notebook_config['resource_group_name'],
notebook_config['instance_name'])
+
AzureActions.remove_instance(notebook_config['resource_group_name'],
notebook_config['instance_name'])
except:
print("The instance hasn't been created.")
dlab.fab.append_result("Failed to create instance.", str(err))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
index a47d883..7c391f6 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
@@ -41,6 +41,8 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
notebook_config = dict()
notebook_config['service_base_name'] = os.environ['conf_service_base_name']
@@ -52,7 +54,7 @@ if __name__ == "__main__":
print('[START NOTEBOOK]')
try:
print("Starting notebook")
-
AzureActions().start_instance(notebook_config['resource_group_name'],
notebook_config['notebook_name'])
+
AzureActions.start_instance(notebook_config['resource_group_name'],
notebook_config['notebook_name'])
print("Instance {} has been
started".format(notebook_config['notebook_name']))
except:
traceback.print_exc()
@@ -64,7 +66,7 @@ if __name__ == "__main__":
try:
logging.info('[SETUP USER GIT CREDENTIALS]')
print('[SETUP USER GIT CREDENTIALS]')
- notebook_config['notebook_ip'] = AzureMeta().get_private_ip_address(
+ notebook_config['notebook_ip'] = AzureMeta.get_private_ip_address(
notebook_config['resource_group_name'],
notebook_config['notebook_name'])
notebook_config['keyfile'] =
'{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
@@ -82,7 +84,7 @@ if __name__ == "__main__":
try:
logging.info('[UPDATE STORAGE CREDENTIALS]')
print('[UPDATE STORAGE CREDENTIALS]')
- notebook_config['notebook_ip'] =
AzureMeta().get_private_ip_address(
+ notebook_config['notebook_ip'] = AzureMeta.get_private_ip_address(
notebook_config['resource_group_name'],
notebook_config['notebook_name'])
env.hosts = "{}".format(notebook_config['notebook_ip'])
env.user = os.environ['conf_os_user']
@@ -116,8 +118,8 @@ if __name__ == "__main__":
sys.exit(1)
try:
- ip_address =
AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
-
notebook_config['notebook_name'])
+ ip_address =
AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
+
notebook_config['notebook_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['notebook_name']))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
index 1534b0f..073b5f6 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
@@ -37,12 +37,12 @@ def stop_notebook(resource_group_name, notebook_name):
print("Stopping data engine cluster")
cluster_list = []
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "notebook_name" in vm.tags:
if notebook_name == vm.tags['notebook_name']:
if 'master' == vm.tags["Type"]:
cluster_list.append(vm.tags["Name"])
- AzureActions().stop_instance(resource_group_name, vm.name)
+ AzureActions.stop_instance(resource_group_name, vm.name)
print("Instance {} has been stopped".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to stop clusters", str(err))
@@ -50,10 +50,10 @@ def stop_notebook(resource_group_name, notebook_name):
print("Stopping notebook")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "Name" in vm.tags:
if notebook_name == vm.tags["Name"]:
- AzureActions().stop_instance(resource_group_name, vm.name)
+ AzureActions.stop_instance(resource_group_name, vm.name)
print("Instance {} has been stopped".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to stop instance", str(err))
@@ -69,6 +69,8 @@ if __name__ == "__main__":
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
notebook_config = dict()
if 'exploratory_name' in os.environ:
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
index d25f40c..4fc6d9f 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
@@ -35,10 +35,10 @@ import traceback
def terminate_nb(resource_group_name, notebook_name):
print("Terminating data engine cluster")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "notebook_name" in vm.tags:
if notebook_name == vm.tags['notebook_name']:
- AzureActions().remove_instance(resource_group_name,
vm.name)
+ AzureActions.remove_instance(resource_group_name, vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to terminate clusters", str(err))
@@ -46,10 +46,10 @@ def terminate_nb(resource_group_name, notebook_name):
print("Terminating notebook")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "Name" in vm.tags:
if notebook_name == vm.tags["Name"]:
- AzureActions().remove_instance(resource_group_name,
vm.name)
+ AzureActions.remove_instance(resource_group_name, vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to terminate instance", str(err))
@@ -64,6 +64,8 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
notebook_config = dict()
if 'exploratory_name' in os.environ:
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
index cab6fc4..c74abfe 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
@@ -31,14 +31,16 @@ import os
if __name__ == "__main__":
try:
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
image_conf = dict()
image_conf['service_base_name'] = os.environ['conf_service_base_name']
image_conf['resource_group_name'] =
os.environ['azure_resource_group_name']
image_conf['full_image_name'] = os.environ['notebook_image_name']
- image = AzureMeta().get_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
+ image = AzureMeta.get_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
if image != '':
- AzureActions().remove_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
+ AzureActions.remove_image(image_conf['resource_group_name'],
image_conf['full_image_name'])
with open("/root/result.json", 'w') as result:
res = {"notebook_image_name": image_conf['full_image_name'],
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index 557cc08..c9ee8fa 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -144,8 +144,8 @@ def configure_slave(slave_number, data_engine):
def clear_resources():
for i in range(data_engine['instance_count'] - 1):
slave_name = data_engine['slave_node_name'] + '{}'.format(i + 1)
- AzureActions().remove_instance(data_engine['resource_group_name'],
slave_name)
- AzureActions().remove_instance(data_engine['resource_group_name'],
data_engine['master_node_name'])
+ AzureActions.remove_instance(data_engine['resource_group_name'],
slave_name)
+ AzureActions.remove_instance(data_engine['resource_group_name'],
data_engine['master_node_name'])
if __name__ == "__main__":
@@ -157,6 +157,8 @@ if __name__ == "__main__":
filename=local_log_filepath)
try:
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
data_engine = dict()
if 'exploratory_name' in os.environ:
@@ -180,9 +182,9 @@ if __name__ == "__main__":
data_engine['private_subnet_name'] =
'{}-{}-{}-subnet'.format(data_engine['service_base_name'],
data_engine['project_name'],
data_engine['endpoint_name'])
- data_engine['private_subnet_cidr'] =
AzureMeta().get_subnet(data_engine['resource_group_name'],
-
data_engine['vpc_name'],
-
data_engine['private_subnet_name']).address_prefix
+ data_engine['private_subnet_cidr'] =
AzureMeta.get_subnet(data_engine['resource_group_name'],
+
data_engine['vpc_name'],
+
data_engine['private_subnet_name']).address_prefix
data_engine['master_security_group_name'] =
'{}-{}-{}-de-master-sg'.format(
data_engine['service_base_name'], data_engine['project_name'],
data_engine['endpoint_name'])
data_engine['slave_security_group_name'] =
'{}-{}-{}-de-slave-sg'.format(
@@ -199,19 +201,19 @@ if __name__ == "__main__":
data_engine['slave_size'] = os.environ['azure_dataengine_slave_size']
data_engine['dlab_ssh_user'] = os.environ['conf_os_user']
data_engine['notebook_name'] = os.environ['notebook_instance_name']
- master_node_hostname =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
data_engine['master_node_name'])
+ master_node_hostname =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
data_engine['master_node_name'])
edge_instance_name =
'{0}-{1}-{2}-edge'.format(data_engine['service_base_name'],
data_engine['project_name'],
data_engine['endpoint_name'])
- edge_instance_private_hostname =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
edge_instance_name)
+ edge_instance_private_hostname =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
edge_instance_name)
if os.environ['conf_network_type'] == 'private':
- edge_instance_hostname =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
edge_instance_name)
+ edge_instance_hostname =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
edge_instance_name)
else:
- edge_instance_hostname =
AzureMeta().get_instance_public_ip_address(data_engine['resource_group_name'],
-
edge_instance_name)
+ edge_instance_hostname =
AzureMeta.get_instance_public_ip_address(data_engine['resource_group_name'],
+
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
key = RSA.importKey(open(keyfile_name, 'rb').read())
data_engine['public_ssh_key'] = key.publickey().exportKey("OpenSSH")
@@ -348,8 +350,8 @@ if __name__ == "__main__":
try:
print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
- notebook_instance_ip =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
data_engine['notebook_name'])
+ notebook_instance_ip =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
data_engine['notebook_name'])
additional_info = {
"computational_name": data_engine['computational_name'],
"master_node_hostname": master_node_hostname,
@@ -382,8 +384,8 @@ if __name__ == "__main__":
sys.exit(1)
try:
- ip_address =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
data_engine['master_node_name'])
+ ip_address =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
data_engine['master_node_name'])
spark_master_url = "http://" + ip_address + ":8080"
spark_master_access_url = "https://" + edge_instance_hostname +
"/{}/".format(
data_engine['exploratory_name'] + '_' +
data_engine['computational_name'])
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
index df943a3..bf69d5b 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
@@ -44,6 +44,8 @@ if __name__ == "__main__":
level=logging.INFO,
filename=local_log_filepath)
try:
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
data_engine = dict()
data_engine['user_name'] = os.environ['edge_user_name'].lower()
data_engine['project_name'] = os.environ['project_name'].lower()
@@ -67,9 +69,9 @@ if __name__ == "__main__":
data_engine['private_subnet_name'] =
'{}-{}-{}-subnet'.format(data_engine['service_base_name'],
data_engine['project_name'],
data_engine['endpoint_name'])
- data_engine['private_subnet_cidr'] =
AzureMeta().get_subnet(data_engine['resource_group_name'],
-
data_engine['vpc_name'],
-
data_engine['private_subnet_name']).address_prefix
+ data_engine['private_subnet_cidr'] =
AzureMeta.get_subnet(data_engine['resource_group_name'],
+
data_engine['vpc_name'],
+
data_engine['private_subnet_name']).address_prefix
data_engine['master_security_group_name'] =
'{}-{}-{}-de-master-sg'.format(
data_engine['service_base_name'], data_engine['project_name'],
data_engine['endpoint_name'])
data_engine['slave_security_group_name'] =
'{}-{}-{}-de-slave-sg'.format(
@@ -121,7 +123,7 @@ if __name__ == "__main__":
else
data_engine['expected_image_name'])(str(os.environ.get('notebook_image_name')))
print('Searching pre-configured images')
- if AzureMeta().get_image(data_engine['resource_group_name'],
data_engine['notebook_image_name']) and \
+ if AzureMeta.get_image(data_engine['resource_group_name'],
data_engine['notebook_image_name']) and \
os.environ['application'] in
os.environ['dataengine_image_notebooks'].split(','):
data_engine['image_name'] = data_engine['notebook_image_name']
data_engine['image_type'] = 'pre-configured'
@@ -134,15 +136,15 @@ if __name__ == "__main__":
sys.exit(1)
try:
- edge_status =
AzureMeta().get_instance_status(data_engine['resource_group_name'],
-
'{0}-{1}-{2}-edge'.format(data_engine['service_base_name'],
-
data_engine['project_name'],
-
data_engine['endpoint_name']))
+ edge_status =
AzureMeta.get_instance_status(data_engine['resource_group_name'],
+
'{0}-{1}-{2}-edge'.format(data_engine['service_base_name'],
+
data_engine['project_name'],
+
data_engine['endpoint_name']))
if edge_status != 'running':
logging.info('ERROR: Edge node is unavailable! Aborting...')
print('ERROR: Edge node is unavailable! Aborting...')
- ssn_hostname =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
data_engine['service_base_name'] + '-ssn')
+ ssn_hostname =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
data_engine['service_base_name'] + '-ssn')
dlab.fab.put_resource_status('edge', 'Unavailable',
os.environ['ssn_dlab_path'], os.environ['conf_os_user'],
ssn_hostname)
dlab.fab.append_result("Edge node is unavailable")
@@ -184,7 +186,7 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
try:
- AzureActions().remove_instance(data_engine['resource_group_name'],
data_engine['master_node_name'])
+ AzureActions.remove_instance(data_engine['resource_group_name'],
data_engine['master_node_name'])
except:
print("The instance hasn't been created.")
dlab.fab.append_result("Failed to create master instance.", str(err))
@@ -220,9 +222,9 @@ if __name__ == "__main__":
for i in range(data_engine['instance_count'] - 1):
slave_name = data_engine['slave_node_name'] + '{}'.format(i+1)
try:
-
AzureActions().remove_instance(data_engine['resource_group_name'], slave_name)
+
AzureActions.remove_instance(data_engine['resource_group_name'], slave_name)
except:
print("The slave instance {} hasn't been
created.".format(slave_name))
- AzureActions().remove_instance(data_engine['resource_group_name'],
data_engine['master_node_name'])
+ AzureActions.remove_instance(data_engine['resource_group_name'],
data_engine['master_node_name'])
dlab.fab.append_result("Failed to create slave instances.", str(err))
sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
index a058c85..6e27e96 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
@@ -35,10 +35,10 @@ import uuid
def start_data_engine(resource_group_name, cluster_name):
print("Starting data engine cluster")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "Name" in vm.tags:
if cluster_name == vm.tags["Name"]:
- AzureActions().start_instance(resource_group_name, vm.name)
+ AzureActions.start_instance(resource_group_name, vm.name)
print("Instance {} has been started".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to start dataengine", str(err))
@@ -53,6 +53,8 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
data_engine = dict()
if 'exploratory_name' in os.environ:
@@ -88,10 +90,10 @@ if __name__ == "__main__":
logging.info('[UPDATE LAST ACTIVITY TIME]')
print('[UPDATE LAST ACTIVITY TIME]')
data_engine['computational_id'] = data_engine['cluster_name'] + '-m'
- data_engine['notebook_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
os.environ['notebook_instance_name'])
- data_engine['computational_ip'] =
AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
-
data_engine['computational_id'])
+ data_engine['notebook_ip'] =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
os.environ['notebook_instance_name'])
+ data_engine['computational_ip'] =
AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
+
data_engine['computational_id'])
data_engine['keyfile'] = '{}{}.pem'.format(os.environ['conf_key_dir'],
os.environ['conf_key_name'])
params = '--os_user {0} --notebook_ip {1} --keyfile "{2}" --cluster_ip
{3}' \
.format(os.environ['conf_os_user'], data_engine['notebook_ip'],
data_engine['keyfile'],
@@ -113,4 +115,4 @@ if __name__ == "__main__":
result.write(json.dumps(res))
except Exception as err:
dlab.fab.append_result("Error with writing results", str(err))
- sys.exit(1)
\ No newline at end of file
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
index a6d28d1..8298fc3 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
@@ -35,10 +35,10 @@ import uuid
def stop_data_engine(resource_group_name, cluster_name):
print("Stopping data engine cluster")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "Name" in vm.tags:
if cluster_name == vm.tags["Name"]:
- AzureActions().stop_instance(resource_group_name, vm.name)
+ AzureActions.stop_instance(resource_group_name, vm.name)
print("Instance {} has been stopped".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to stop dataengine", str(err))
@@ -53,6 +53,8 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
data_engine = dict()
if 'exploratory_name' in os.environ:
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
index a47ecdc..da6363a 100644
---
a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
+++
b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
@@ -35,10 +35,10 @@ import uuid
def terminate_data_engine(resource_group_name, notebook_name, os_user,
key_path, cluster_name):
print("Terminating data engine cluster")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if "Name" in vm.tags:
if cluster_name == vm.tags["Name"]:
- AzureActions().remove_instance(resource_group_name,
vm.name)
+ AzureActions.remove_instance(resource_group_name, vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to terminate dataengine", str(err))
@@ -46,7 +46,7 @@ def terminate_data_engine(resource_group_name, notebook_name,
os_user, key_path,
print("Removing Data Engine kernels from notebook")
try:
- AzureActions().remove_dataengine_kernels(resource_group_name,
notebook_name, os_user, key_path, cluster_name)
+ AzureActions.remove_dataengine_kernels(resource_group_name,
notebook_name, os_user, key_path, cluster_name)
except Exception as err:
dlab.fab.append_result("Failed to remove dataengine kernels from
notebook", str(err))
sys.exit(1)
@@ -60,6 +60,8 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
data_engine = dict()
if 'exploratory_name' in os.environ:
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
index 5a16b20..ab1945c 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
@@ -41,26 +41,28 @@ if __name__ == "__main__":
filename=local_log_filepath)
def clear_resources():
- AzureActions().remove_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
- AzureActions().remove_subnet(edge_conf['resource_group_name'],
edge_conf['vpc_name'],
- edge_conf['private_subnet_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['edge_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'],
-
edge_conf['notebook_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'],
-
edge_conf['master_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'],
-
edge_conf['slave_security_group_name'])
- for storage_account in
AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
+ AzureActions.remove_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
+ AzureActions.remove_subnet(edge_conf['resource_group_name'],
edge_conf['vpc_name'],
+ edge_conf['private_subnet_name'])
+ AzureActions.remove_security_group(edge_conf['resource_group_name'],
edge_conf['edge_security_group_name'])
+ AzureActions.remove_security_group(edge_conf['resource_group_name'],
+
edge_conf['notebook_security_group_name'])
+ AzureActions.remove_security_group(edge_conf['resource_group_name'],
+
edge_conf['master_security_group_name'])
+ AzureActions.remove_security_group(edge_conf['resource_group_name'],
+
edge_conf['slave_security_group_name'])
+ for storage_account in
AzureMeta.list_storage_accounts(edge_conf['resource_group_name']):
if edge_conf['user_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(edge_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(edge_conf['resource_group_name'],
storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
- for datalake in
AzureMeta().list_datalakes(edge_conf['resource_group_name']):
+ for datalake in
AzureMeta.list_datalakes(edge_conf['resource_group_name']):
if edge_conf['datalake_store_name'] == datalake.tags["Name"]:
- AzureActions().remove_datalake_directory(datalake.name,
edge_conf['datalake_user_directory_name'])
+ AzureActions.remove_datalake_directory(datalake.name,
edge_conf['datalake_user_directory_name'])
try:
print('Generating infrastructure names and tags')
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
edge_conf = dict()
edge_conf['service_base_name'] = os.environ['conf_service_base_name']
= dlab.fab.replace_multi_symbols(
os.environ['conf_service_base_name'].lower()[:20], '-', True)
@@ -104,30 +106,30 @@ if __name__ == "__main__":
edge_conf['endpoint_name'])
edge_conf['dlab_ssh_user'] = os.environ['conf_os_user']
edge_conf['keyfile_name'] =
"{}{}.pem".format(os.environ['conf_key_dir'], edge_conf['key_name'])
- edge_conf['private_subnet_cidr'] =
AzureMeta().get_subnet(edge_conf['resource_group_name'],
-
edge_conf['vpc_name'],
-
edge_conf['private_subnet_name']).address_prefix
+ edge_conf['private_subnet_cidr'] =
AzureMeta.get_subnet(edge_conf['resource_group_name'],
+
edge_conf['vpc_name'],
+
edge_conf['private_subnet_name']).address_prefix
if os.environ['conf_network_type'] == 'private':
- edge_conf['edge_private_ip'] =
AzureMeta().get_private_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
+ edge_conf['edge_private_ip'] =
AzureMeta.get_private_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name'])
edge_conf['edge_public_ip'] = edge_conf['edge_private_ip']
else:
- edge_conf['edge_public_ip'] =
AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
- edge_conf['edge_private_ip'] =
AzureMeta().get_private_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
- edge_conf['instance_hostname'] =
AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
- edge_conf['vpc_cidrs'] =
AzureMeta().get_vpc(edge_conf['resource_group_name'],
-
edge_conf['vpc_name']).address_space.address_prefixes
+ edge_conf['edge_public_ip'] =
AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name'])
+ edge_conf['edge_private_ip'] =
AzureMeta.get_private_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name'])
+ edge_conf['instance_hostname'] =
AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name'])
+ edge_conf['vpc_cidrs'] =
AzureMeta.get_vpc(edge_conf['resource_group_name'],
+
edge_conf['vpc_name']).address_space.address_prefixes
if os.environ['conf_stepcerts_enabled'] == 'true':
- edge_conf['step_cert_sans'] = ' --san {0} --san {1}
'.format(AzureMeta().get_private_ip_address(
+ edge_conf['step_cert_sans'] = ' --san {0} --san {1}
'.format(AzureMeta.get_private_ip_address(
edge_conf['resource_group_name'], edge_conf['instance_name']),
edge_conf['instance_dns_name'])
if os.environ['conf_network_type'] == 'public':
edge_conf['step_cert_sans'] += ' --san {0}'.format(
-
AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name']))
+
AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name']))
else:
edge_conf['step_cert_sans'] = ''
@@ -254,7 +256,7 @@ if __name__ == "__main__":
sys.exit(1)
try:
- for storage_account in
AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
+ for storage_account in
AzureMeta.list_storage_accounts(edge_conf['resource_group_name']):
if edge_conf['shared_storage_account_name'] ==
storage_account.tags["Name"]:
edge_conf['shared_storage_account_name'] = storage_account.name
if edge_conf['user_storage_account_name'] ==
storage_account.tags["Name"]:
@@ -270,7 +272,7 @@ if __name__ == "__main__":
print("User storage account name:
{}".format(edge_conf['user_storage_account_name']))
print("User container name:
{}".format(edge_conf['user_container_name']))
if os.environ['azure_datalake_enable'] == 'true':
- for datalake in
AzureMeta().list_datalakes(edge_conf['resource_group_name']):
+ for datalake in
AzureMeta.list_datalakes(edge_conf['resource_group_name']):
if edge_conf['datalake_store_name'] == datalake.tags["Name"]:
edge_conf['datalake_id'] = datalake.name
print("Data Lake name: {}".format(edge_conf['datalake_id']))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
index 1174c4a..d0ba983 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
@@ -39,6 +39,8 @@ if __name__ == "__main__":
filename=local_log_filepath)
print('Generating infrastructure names and tags')
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
edge_conf = dict()
edge_conf['service_base_name'] = os.environ['conf_service_base_name']
edge_conf['resource_group_name'] = os.environ['azure_resource_group_name']
@@ -52,16 +54,16 @@ if __name__ == "__main__":
logging.info('[START EDGE]')
print('[START EDGE]')
try:
- AzureActions().start_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
+ AzureActions.start_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
except Exception as err:
dlab.fab.append_result("Failed to start edge.", str(err))
sys.exit(1)
try:
- public_ip_address =
AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
- private_ip_address =
AzureMeta().get_private_ip_address(edge_conf['resource_group_name'],
-
edge_conf['instance_name'])
+ public_ip_address =
AzureMeta.get_instance_public_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name'])
+ private_ip_address =
AzureMeta.get_private_ip_address(edge_conf['resource_group_name'],
+
edge_conf['instance_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(edge_conf['instance_name']))
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
index 6724739..02308e0 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
@@ -38,6 +38,8 @@ if __name__ == "__main__":
filename=local_log_filepath)
print('Generating infrastructure names and tags')
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
edge_conf = dict()
edge_conf['service_base_name'] = os.environ['conf_service_base_name']
edge_conf['resource_group_name'] = os.environ['azure_resource_group_name']
@@ -49,7 +51,7 @@ if __name__ == "__main__":
logging.info('[STOP EDGE]')
print('[STOP EDGE]')
try:
- AzureActions().stop_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
+ AzureActions.stop_instance(edge_conf['resource_group_name'],
edge_conf['instance_name'])
except Exception as err:
dlab.fab.append_result("Failed to stop edge.", str(err))
sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index 010e3df..b48625e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -44,6 +44,8 @@ if __name__ == "__main__":
try:
print('Generating infrastructure names and tags')
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
project_conf = dict()
project_conf['service_base_name'] =
os.environ['conf_service_base_name'] = dlab.fab.replace_multi_symbols(
os.environ['conf_service_base_name'].lower()[:20], '-', True)
@@ -160,16 +162,16 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
try:
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
except:
print("Subnet hasn't been created.")
dlab.fab.append_result("Failed to create subnet.", str(err))
sys.exit(1)
- project_conf['private_subnet_cidr'] =
AzureMeta().get_subnet(project_conf['resource_group_name'],
-
project_conf['vpc_name'],
-
project_conf['private_subnet_name']).address_prefix
+ project_conf['private_subnet_cidr'] =
AzureMeta.get_subnet(project_conf['resource_group_name'],
+
project_conf['vpc_name'],
+
project_conf['private_subnet_name']).address_prefix
print('NEW SUBNET CIDR CREATED:
{}'.format(project_conf['private_subnet_cidr']))
try:
@@ -447,11 +449,11 @@ if __name__ == "__main__":
try:
local("~/scripts/{}.py {}".format('common_create_security_group',
params))
except Exception as err:
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
try:
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
except:
print("Edge Security group hasn't been created.")
traceback.print_exc()
@@ -480,9 +482,9 @@ if __name__ == "__main__":
"protocol": "*",
"source_port_range": "*",
"destination_port_range": "*",
- "source_address_prefix":
AzureMeta().get_subnet(project_conf['resource_group_name'],
-
project_conf['vpc_name'],
-
project_conf['subnet_name']).address_prefix,
+ "source_address_prefix":
AzureMeta.get_subnet(project_conf['resource_group_name'],
+
project_conf['vpc_name'],
+
project_conf['subnet_name']).address_prefix,
"destination_address_prefix": "*",
"access": "Allow",
"priority": 110,
@@ -516,9 +518,9 @@ if __name__ == "__main__":
"source_port_range": "*",
"destination_port_range": "*",
"source_address_prefix": "*",
- "destination_address_prefix":
AzureMeta().get_subnet(project_conf['resource_group_name'],
-
project_conf['vpc_name'],
-
project_conf['subnet_name']).address_prefix,
+ "destination_address_prefix":
AzureMeta.get_subnet(project_conf['resource_group_name'],
+
project_conf['vpc_name'],
+
project_conf['subnet_name']).address_prefix,
"access": "Allow",
"priority": 110,
"direction": "Outbound"
@@ -556,13 +558,13 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
dlab.fab.append_result("Failed creating security group for private
subnet.", str(err))
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
try:
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
except:
print("Notebook Security group hasn't been created.")
sys.exit(1)
@@ -587,9 +589,9 @@ if __name__ == "__main__":
"protocol": "*",
"source_port_range": "*",
"destination_port_range": "*",
- "source_address_prefix":
AzureMeta().get_subnet(project_conf['resource_group_name'],
-
project_conf['vpc_name'],
-
project_conf['subnet_name']).address_prefix,
+ "source_address_prefix":
AzureMeta.get_subnet(project_conf['resource_group_name'],
+
project_conf['vpc_name'],
+
project_conf['subnet_name']).address_prefix,
"destination_address_prefix": "*",
"access": "Allow",
"priority": 110,
@@ -623,9 +625,9 @@ if __name__ == "__main__":
"source_port_range": "*",
"destination_port_range": "*",
"source_address_prefix": "*",
- "destination_address_prefix":
AzureMeta().get_subnet(project_conf['resource_group_name'],
-
project_conf['vpc_name'],
-
project_conf['subnet_name']).address_prefix,
+ "destination_address_prefix":
AzureMeta.get_subnet(project_conf['resource_group_name'],
+
project_conf['vpc_name'],
+
project_conf['subnet_name']).address_prefix,
"access": "Allow",
"priority": 110,
"direction": "Outbound"
@@ -662,15 +664,15 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
try:
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['master_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['master_security_group_name'])
except:
print("Master Security group hasn't been created.")
dlab.fab.append_result("Failed to create Security groups. Exception:"
+ str(err))
@@ -688,17 +690,17 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
except Exception as err:
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['master_security_group_name'])
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['master_security_group_name'])
try:
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['slave_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['slave_security_group_name'])
except:
print("Slave Security group hasn't been created.")
dlab.fab.append_result("Failed to create Security groups. Exception:"
+ str(err))
@@ -713,19 +715,19 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('common_create_storage_account',
params))
except Exception as err:
dlab.fab.append_result("Failed to create storage account.", str(err))
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['master_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['slave_security_group_name'])
- for storage_account in
AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['master_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['slave_security_group_name'])
+ for storage_account in
AzureMeta.list_storage_accounts(project_conf['resource_group_name']):
if project_conf['shared_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
sys.exit(1)
try:
@@ -742,21 +744,21 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
dlab.fab.append_result("Failed to create storage account.", str(err))
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['master_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['slave_security_group_name'])
- for storage_account in
AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['master_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['slave_security_group_name'])
+ for storage_account in
AzureMeta.list_storage_accounts(project_conf['resource_group_name']):
if project_conf['edge_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
if project_conf['shared_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
sys.exit(1)
if os.environ['azure_datalake_enable'] == 'true':
@@ -776,25 +778,25 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
dlab.fab.append_result("Failed to create Data Lake Store
directory.", str(err))
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['master_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['slave_security_group_name'])
- for storage_account in
AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['master_security_group_name'])
+
AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['slave_security_group_name'])
+ for storage_account in
AzureMeta.list_storage_accounts(project_conf['resource_group_name']):
if project_conf['edge_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
if project_conf['shared_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
try:
- for datalake in
AzureMeta().list_datalakes(project_conf['resource_group_name']):
+ for datalake in
AzureMeta.list_datalakes(project_conf['resource_group_name']):
if project_conf['datalake_store_name'] ==
datalake.tags["Name"]:
- AzureActions().remove_datalake_directory(datalake.name,
+ AzureActions.remove_datalake_directory(datalake.name,
project_conf['datalake_user_directory_name'])
except:
print("Data Lake Store directory hasn't been created.")
@@ -829,28 +831,28 @@ if __name__ == "__main__":
raise Exception
except Exception as err:
try:
-
AzureActions().remove_instance(project_conf['resource_group_name'],
project_conf['instance_name'])
+ AzureActions.remove_instance(project_conf['resource_group_name'],
project_conf['instance_name'])
except:
print("The instance hasn't been created.")
- AzureActions().remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
- project_conf['private_subnet_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['edge_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['notebook_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['master_security_group_name'])
-
AzureActions().remove_security_group(project_conf['resource_group_name'],
-
project_conf['slave_security_group_name'])
- for storage_account in
AzureMeta().list_storage_accounts(project_conf['resource_group_name']):
+ AzureActions.remove_subnet(project_conf['resource_group_name'],
project_conf['vpc_name'],
+ project_conf['private_subnet_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['edge_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['notebook_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['master_security_group_name'])
+ AzureActions.remove_security_group(project_conf['resource_group_name'],
+
project_conf['slave_security_group_name'])
+ for storage_account in
AzureMeta.list_storage_accounts(project_conf['resource_group_name']):
if project_conf['edge_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
if project_conf['shared_storage_account_name'] ==
storage_account.tags["Name"]:
-
AzureActions().remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
+
AzureActions.remove_storage_account(project_conf['resource_group_name'],
storage_account.name)
if os.environ['azure_datalake_enable'] == 'true':
- for datalake in
AzureMeta().list_datalakes(project_conf['resource_group_name']):
+ for datalake in
AzureMeta.list_datalakes(project_conf['resource_group_name']):
if project_conf['datalake_store_name'] ==
datalake.tags["Name"]:
- AzureActions().remove_datalake_directory(datalake.name,
-
project_conf['datalake_user_directory_name'])
+ AzureActions.remove_datalake_directory(datalake.name,
+
project_conf['datalake_user_directory_name'])
dlab.fab.append_result("Failed to create instance. Exception:" +
str(err))
sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
index 202a68b..71c1116 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
@@ -22,128 +22,138 @@
#
******************************************************************************
import json
-import sys, time, os
-from dlab.fab import *
-from dlab.meta_lib import *
-from dlab.actions_lib import *
+import logging
+import sys
+import time
+import os
+import dlab.fab
+import dlab.actions_lib
+import dlab.meta_lib
import requests
+import traceback
def terminate_edge_node(resource_group_name, service_base_name, project_tag,
subnet_name, vpc_name):
print("Terminating EDGE, notebook and dataengine virtual machines")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
try:
if project_tag == vm.tags["project_tag"]:
- AzureActions().remove_instance(resource_group_name,
vm.name)
+ AzureActions.remove_instance(resource_group_name, vm.name)
print("Instance {} has been terminated".format(vm.name))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to terminate edge instance.", str(err))
sys.exit(1)
print("Removing network interfaces")
try:
- for network_interface in
AzureMeta().list_network_interfaces(resource_group_name):
+ for network_interface in
AzureMeta.list_network_interfaces(resource_group_name):
try:
if project_tag == network_interface.tags["project_name"]:
- AzureActions().delete_network_if(resource_group_name,
network_interface.name)
+ AzureActions.delete_network_if(resource_group_name,
network_interface.name)
print("Network interface {} has been
removed".format(network_interface.name))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove network interfaces.",
str(err))
sys.exit(1)
print("Removing static public IPs")
try:
- for static_public_ip in
AzureMeta().list_static_ips(resource_group_name):
+ for static_public_ip in AzureMeta.list_static_ips(resource_group_name):
try:
if project_tag in static_public_ip.tags["project_tag"]:
-
AzureActions().delete_static_public_ip(resource_group_name,
static_public_ip.name)
+ AzureActions.delete_static_public_ip(resource_group_name,
static_public_ip.name)
print("Static public IP {} has been
removed".format(static_public_ip.name))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove static IP addresses.",
str(err))
sys.exit(1)
print("Removing disks")
try:
- for disk in AzureMeta().list_disks(resource_group_name):
+ for disk in AzureMeta.list_disks(resource_group_name):
try:
if project_tag in disk.tags["project_tag"]:
- AzureActions().remove_disk(resource_group_name, disk.name)
+ AzureActions.remove_disk(resource_group_name, disk.name)
print("Disk {} has been removed".format(disk.name))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove volumes.", str(err))
sys.exit(1)
print("Removing storage account")
try:
- for storage_account in
AzureMeta().list_storage_accounts(resource_group_name):
+ for storage_account in
AzureMeta.list_storage_accounts(resource_group_name):
try:
if project_tag == storage_account.tags["project_tag"]:
- AzureActions().remove_storage_account(resource_group_name,
storage_account.name)
+ AzureActions.remove_storage_account(resource_group_name,
storage_account.name)
print("Storage account {} has been
terminated".format(storage_account.name))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove storage accounts.", str(err))
sys.exit(1)
print("Deleting Data Lake Store directory")
try:
- for datalake in AzureMeta().list_datalakes(resource_group_name):
+ for datalake in AzureMeta.list_datalakes(resource_group_name):
try:
if service_base_name == datalake.tags["SBN"]:
- AzureActions().remove_datalake_directory(datalake.name,
project_tag + '-folder')
+ AzureActions.remove_datalake_directory(datalake.name,
project_tag + '-folder')
print("Data Lake Store directory {} has been
deleted".format(project_tag + '-folder'))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove Data Lake.", str(err))
sys.exit(1)
print("Removing security groups")
try:
- for sg in
AzureMeta().network_client.network_security_groups.list(resource_group_name):
+ for sg in
AzureMeta.network_client.network_security_groups.list(resource_group_name):
try:
if project_tag == sg.tags["project_tag"]:
- AzureActions().remove_security_group(resource_group_name,
sg.name)
+ AzureActions.remove_security_group(resource_group_name,
sg.name)
print("Security group {} has been
terminated".format(sg.name))
except:
pass
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove security groups.", str(err))
sys.exit(1)
print("Removing private subnet")
try:
- AzureActions().remove_subnet(resource_group_name, vpc_name,
subnet_name)
+ AzureActions.remove_subnet(resource_group_name, vpc_name, subnet_name)
print("Private subnet {} has been terminated".format(subnet_name))
except Exception as err:
- print('Error: {0}'.format(err))
+ dlab.fab.append_result("Failed to remove subnets.", str(err))
sys.exit(1)
if __name__ == "__main__":
- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'], os.environ['request_id'])
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
+ os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)
print('Generating infrastructure names and tags')
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
project_conf = dict()
project_conf['service_base_name'] = os.environ['conf_service_base_name']
project_conf['resource_group_name'] =
os.environ['azure_resource_group_name']
- project_conf['project_name'] =
os.environ['project_name'].lower().replace('_', '-')
- project_conf['project_tag'] =
os.environ['project_name'].lower().replace('_', '-')
- project_conf['private_subnet_name'] = project_conf['service_base_name'] +
"-" + project_conf['project_name'] + '-subnet'
+ project_conf['project_name'] = os.environ['project_name'].lower()
+ project_conf['project_tag'] = project_conf['project_name']
+ project_conf['endpoint_name'] = os.environ['endpoint_name'].lower()
+ project_conf['private_subnet_name'] =
'{}-{}-{}-subnet'.format(project_conf['service_base_name'],
+
project_conf['project_name'],
+
project_conf['endpoint_name'])
project_conf['vpc_name'] = os.environ['azure_vpc_name']
@@ -152,18 +162,22 @@ if __name__ == "__main__":
print('[TERMINATE EDGE]')
try:
terminate_edge_node(project_conf['resource_group_name'],
project_conf['service_base_name'],
- project_conf['project_tag'],
project_conf['private_subnet_name'], project_conf['vpc_name'])
+ project_conf['project_tag'],
project_conf['private_subnet_name'],
+ project_conf['vpc_name'])
except Exception as err:
traceback.print_exc()
- append_result("Failed to terminate edge.", str(err))
+ dlab.fab.append_result("Failed to terminate edge.", str(err))
+ raise Exception
except:
sys.exit(1)
try:
print('[KEYCLOAK PROJECT CLIENT DELETE]')
logging.info('[KEYCLOAK PROJECT CLIENT DELETE]')
- keycloak_auth_server_url =
'{}/realms/master/protocol/openid-connect/token'.format(os.environ['keycloak_auth_server_url'])
- keycloak_client_url =
'{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'],
os.environ['keycloak_realm_name'])
+ keycloak_auth_server_url =
'{}/realms/master/protocol/openid-connect/token'.format(
+ os.environ['keycloak_auth_server_url'])
+ keycloak_client_url =
'{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'],
+
os.environ['keycloak_realm_name'])
keycloak_auth_data = {
"username": os.environ['keycloak_user'],
@@ -173,7 +187,8 @@ if __name__ == "__main__":
}
client_params = {
- "clientId": project_conf['service_base_name'] + '-' +
os.environ['project_name'] + '-' + os.environ['endpoint_name'],
+ "clientId": "{}-{}-{}".format(project_conf['service_base_name'],
project_conf['project_name'],
+ project_conf['endpoint_name'])
}
keycloak_token = requests.post(keycloak_auth_server_url,
data=keycloak_auth_data).json()
@@ -201,6 +216,6 @@ if __name__ == "__main__":
"Action": "Terminate edge node"}
print(json.dumps(res))
result.write(json.dumps(res))
- except:
- print("Failed writing results.")
- sys.exit(0)
+ except Exception as err:
+ dlab.fab.append_result("Error with writing results", str(err))
+ sys.exit(1)
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
index f6b21cd..6905eb7 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
@@ -41,20 +41,22 @@ if __name__ == "__main__":
def clear_resources():
if 'azure_resource_group_name' not in os.environ:
-
AzureActions().remove_resource_group(ssn_conf['service_base_name'],
ssn_conf['region'])
+ AzureActions.remove_resource_group(ssn_conf['service_base_name'],
ssn_conf['region'])
if 'azure_vpc_name' not in os.environ:
- AzureActions().remove_vpc(ssn_conf['resource_group_name'],
ssn_conf['vpc_name'])
+ AzureActions.remove_vpc(ssn_conf['resource_group_name'],
ssn_conf['vpc_name'])
if 'azure_subnet_name' not in os.environ:
- AzureActions().remove_subnet(ssn_conf['resource_group_name'],
ssn_conf['vpc_name'],
- ssn_conf['subnet_name'])
+ AzureActions.remove_subnet(ssn_conf['resource_group_name'],
ssn_conf['vpc_name'],
+ ssn_conf['subnet_name'])
if 'azure_security_group_name' not in os.environ:
-
AzureActions().remove_security_group(ssn_conf['resource_group_name'],
ssn_conf['security_group_name'])
- for datalake in
AzureMeta().list_datalakes(ssn_conf['resource_group_name']):
+
AzureActions.remove_security_group(ssn_conf['resource_group_name'],
ssn_conf['security_group_name'])
+ for datalake in
AzureMeta.list_datalakes(ssn_conf['resource_group_name']):
if ssn_conf['datalake_store_name'] == datalake.tags["Name"]:
-
AzureActions().delete_datalake_store(ssn_conf['resource_group_name'],
datalake.name)
- AzureActions().remove_instance(ssn_conf['resource_group_name'],
ssn_conf['instance_name'])
+
AzureActions.delete_datalake_store(ssn_conf['resource_group_name'],
datalake.name)
+ AzureActions.remove_instance(ssn_conf['resource_group_name'],
ssn_conf['instance_name'])
try:
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
ssn_conf = dict()
ssn_conf['instance'] = 'ssn'
@@ -81,21 +83,21 @@ if __name__ == "__main__":
ssn_conf['ssh_key_path'] =
'{}{}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
ssn_conf['dlab_ssh_user'] = os.environ['conf_os_user']
if os.environ['conf_network_type'] == 'private':
- ssn_conf['instnace_ip'] =
AzureMeta().get_private_ip_address(ssn_conf['resource_group_name'],
-
ssn_conf['instance_name'])
+ ssn_conf['instnace_ip'] =
AzureMeta.get_private_ip_address(ssn_conf['resource_group_name'],
+
ssn_conf['instance_name'])
else:
- ssn_conf['instnace_ip'] =
AzureMeta().get_instance_public_ip_address(ssn_conf['resource_group_name'],
-
ssn_conf['instance_name'])
+ ssn_conf['instnace_ip'] =
AzureMeta.get_instance_public_ip_address(ssn_conf['resource_group_name'],
+
ssn_conf['instance_name'])
ssn_conf['instance_dns_name'] =
'host-{}.{}.cloudapp.azure.com'.format(ssn_conf['instance_name'],
ssn_conf['region'])
if os.environ['conf_stepcerts_enabled'] == 'true':
- ssn_conf['step_cert_sans'] = ' --san {0} --san {1}
'.format(AzureMeta().get_private_ip_address(
+ ssn_conf['step_cert_sans'] = ' --san {0} --san {1}
'.format(AzureMeta.get_private_ip_address(
ssn_conf['resource_group_name'], ssn_conf['instance_name']),
ssn_conf['instance_dns_name'])
if os.environ['conf_network_type'] == 'public':
ssn_conf['step_cert_sans'] += ' --san {0}'.format(
-
AzureMeta().get_instance_public_ip_address(ssn_conf['resource_group_name'],
-
ssn_conf['instance_name']))
+
AzureMeta.get_instance_public_ip_address(ssn_conf['resource_group_name'],
+
ssn_conf['instance_name']))
else:
ssn_conf['step_cert_sans'] = ''
@@ -396,8 +398,8 @@ if __name__ == "__main__":
})
if os.environ['azure_oauth2_enabled'] == 'false':
ssn_conf['ldap_login'] = 'true'
- ssn_conf['tenant_id'] =
json.dumps(AzureMeta().sp_creds['tenantId']).replace('"', '')
- ssn_conf['subscription_id'] =
json.dumps(AzureMeta().sp_creds['subscriptionId']).replace('"', '')
+ ssn_conf['tenant_id'] =
json.dumps(AzureMeta.sp_creds['tenantId']).replace('"', '')
+ ssn_conf['subscription_id'] =
json.dumps(AzureMeta.sp_creds['subscriptionId']).replace('"', '')
ssn_conf['datalake_application_id'] =
os.environ['azure_application_id']
ssn_conf['datalake_store_name'] = None
else:
@@ -411,10 +413,10 @@ if __name__ == "__main__":
'key': 'AZURE_CLIENT_ID',
'value': os.environ['azure_application_id']
})
- ssn_conf['tenant_id'] =
json.dumps(AzureMeta().sp_creds['tenantId']).replace('"', '')
- ssn_conf['subscription_id'] =
json.dumps(AzureMeta().sp_creds['subscriptionId']).replace('"', '')
+ ssn_conf['tenant_id'] =
json.dumps(AzureMeta.sp_creds['tenantId']).replace('"', '')
+ ssn_conf['subscription_id'] =
json.dumps(AzureMeta.sp_creds['subscriptionId']).replace('"', '')
ssn_conf['datalake_application_id'] =
os.environ['azure_application_id']
- for datalake in
AzureMeta().list_datalakes(ssn_conf['resource_group_name']):
+ for datalake in
AzureMeta.list_datalakes(ssn_conf['resource_group_name']):
if ssn_conf['datalake_store_name'] == datalake.tags["Name"]:
ssn_conf['datalake_store_name'] = datalake.name
params = "--hostname {} --keyfile {} --dlab_path {} --os_user {}
--os_family {} --request_id {} " \
@@ -455,7 +457,7 @@ if __name__ == "__main__":
print("SSN instance size:
{}".format(os.environ['azure_ssn_instance_size']))
ssn_conf['datalake_store_full_name'] = 'None'
if os.environ['azure_datalake_enable'] == 'true':
- for datalake in
AzureMeta().list_datalakes(ssn_conf['resource_group_name']):
+ for datalake in
AzureMeta.list_datalakes(ssn_conf['resource_group_name']):
if ssn_conf['datalake_store_name'] == datalake.tags["Name"]:
ssn_conf['datalake_store_full_name'] = datalake.name
print("DataLake store name:
{}".format(ssn_conf['datalake_store_full_name']))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
index e1606b1..e15cddf 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
@@ -248,7 +248,7 @@ if __name__ == "__main__":
AzureActions.remove_vpc(ssn_conf['resource_group_name'],
ssn_conf['vpc_name'])
if 'azure_subnet_name' not in os.environ:
AzureActions.remove_subnet(ssn_conf['resource_group_name'],
ssn_conf['vpc_name'],
- ssn_conf['subnet_name'])
+ ssn_conf['subnet_name'])
except Exception as err:
print("Resources hasn't been removed: " + str(err))
dlab.fab.append_result("Resources hasn't been removed.", str(err))
diff --git
a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
index 1bb24f3..14e1ea0 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
@@ -36,9 +36,9 @@ import json
def terminate_ssn_node(resource_group_name, service_base_name, vpc_name,
region):
print("Terminating instances")
try:
- for vm in
AzureMeta().compute_client.virtual_machines.list(resource_group_name):
+ for vm in
AzureMeta.compute_client.virtual_machines.list(resource_group_name):
if service_base_name == vm.tags["SBN"]:
- AzureActions().remove_instance(resource_group_name, vm.name)
+ AzureActions.remove_instance(resource_group_name, vm.name)
print("Instance {} has been terminated".format(vm.name))
except Exception as err:
dlab.fab.append_result("Failed to terminate instances", str(err))
@@ -46,9 +46,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing network interfaces")
try:
- for network_interface in
AzureMeta().list_network_interfaces(resource_group_name):
+ for network_interface in
AzureMeta.list_network_interfaces(resource_group_name):
if service_base_name == network_interface.tags["SBN"]:
- AzureActions().delete_network_if(resource_group_name,
network_interface.name)
+ AzureActions.delete_network_if(resource_group_name,
network_interface.name)
print("Network interface {} has been
removed".format(network_interface.name))
except Exception as err:
dlab.fab.append_result("Failed to remove network interfaces", str(err))
@@ -56,9 +56,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing static public IPs")
try:
- for static_public_ip in
AzureMeta().list_static_ips(resource_group_name):
+ for static_public_ip in AzureMeta.list_static_ips(resource_group_name):
if service_base_name == static_public_ip.tags["SBN"]:
- AzureActions().delete_static_public_ip(resource_group_name,
static_public_ip.name)
+ AzureActions.delete_static_public_ip(resource_group_name,
static_public_ip.name)
print("Static public IP {} has been
removed".format(static_public_ip.name))
except Exception as err:
dlab.fab.append_result("Failed to remove static IPs", str(err))
@@ -66,9 +66,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing disks")
try:
- for disk in AzureMeta().list_disks(resource_group_name):
+ for disk in AzureMeta.list_disks(resource_group_name):
if service_base_name == disk.tags["SBN"]:
- AzureActions().remove_disk(resource_group_name, disk.name)
+ AzureActions.remove_disk(resource_group_name, disk.name)
print("Disk {} has been removed".format(disk.name))
except Exception as err:
dlab.fab.append_result("Failed to remove disks", str(err))
@@ -76,9 +76,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing storage accounts")
try:
- for storage_account in
AzureMeta().list_storage_accounts(resource_group_name):
+ for storage_account in
AzureMeta.list_storage_accounts(resource_group_name):
if service_base_name == storage_account.tags["SBN"]:
- AzureActions().remove_storage_account(resource_group_name,
storage_account.name)
+ AzureActions.remove_storage_account(resource_group_name,
storage_account.name)
print("Storage account {} has been
terminated".format(storage_account.name))
except Exception as err:
dlab.fab.append_result("Failed to remove storage accounts", str(err))
@@ -86,9 +86,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing Data Lake Store")
try:
- for datalake in AzureMeta().list_datalakes(resource_group_name):
+ for datalake in AzureMeta.list_datalakes(resource_group_name):
if service_base_name == datalake.tags["SBN"]:
- AzureActions().delete_datalake_store(resource_group_name,
datalake.name)
+ AzureActions.delete_datalake_store(resource_group_name,
datalake.name)
print("Data Lake Store {} has been
terminated".format(datalake.name))
except Exception as err:
dlab.fab.append_result("Failed to remove Data Lake", str(err))
@@ -96,9 +96,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing images")
try:
- for image in AzureMeta().list_images():
+ for image in AzureMeta.list_images():
if service_base_name == image.tags["SBN"]:
- AzureActions().remove_image(resource_group_name, image.name)
+ AzureActions.remove_image(resource_group_name, image.name)
print("Image {} has been removed".format(image.name))
except Exception as err:
dlab.fab.append_result("Failed to remove images", str(err))
@@ -106,9 +106,9 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing security groups")
try:
- for sg in
AzureMeta().network_client.network_security_groups.list(resource_group_name):
+ for sg in
AzureMeta.network_client.network_security_groups.list(resource_group_name):
if service_base_name == sg.tags["SBN"]:
- AzureActions().remove_security_group(resource_group_name,
sg.name)
+ AzureActions.remove_security_group(resource_group_name,
sg.name)
print("Security group {} has been terminated".format(sg.name))
except Exception as err:
dlab.fab.append_result("Failed to remove security groups", str(err))
@@ -116,8 +116,8 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing VPC")
try:
- if AzureMeta().get_vpc(resource_group_name, service_base_name +
'-vpc'):
- AzureActions().remove_vpc(resource_group_name, vpc_name)
+ if AzureMeta.get_vpc(resource_group_name, service_base_name + '-vpc'):
+ AzureActions.remove_vpc(resource_group_name, vpc_name)
print("VPC {} has been terminated".format(vpc_name))
except Exception as err:
dlab.fab.append_result("Failed to remove VPC", str(err))
@@ -125,8 +125,8 @@ def terminate_ssn_node(resource_group_name,
service_base_name, vpc_name, region)
print("Removing Resource Group")
try:
- if AzureMeta().get_resource_group(service_base_name):
- AzureActions().remove_resource_group(service_base_name, region)
+ if AzureMeta.get_resource_group(service_base_name):
+ AzureActions.remove_resource_group(service_base_name, region)
print("Resource group {} has been terminated".format(vpc_name))
except Exception as err:
dlab.fab.append_result("Failed to remove resource group", str(err))
@@ -140,6 +140,8 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
+ AzureMeta = dlab.meta_lib.AzureMeta()
+ AzureActions = dlab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
ssn_conf = dict()
ssn_conf['service_base_name'] =
dlab.fab.replace_multi_symbols(os.environ['conf_service_base_name'].lower()[:20],
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]