This is an automated email from the ASF dual-hosted git repository. omartushevskyi pushed a commit to branch DLAB-252 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 9c6f4977bc208505cbc53a1d0d70bc71a2985905 Author: Oleh Martushevskyi <[email protected]> AuthorDate: Thu Mar 21 13:30:26 2019 +0200 added reverse proxy to azure --- .../src/general/files/azure/base_Dockerfile | 1 + .../general/scripts/azure/dataengine_configure.py | 53 ++++++++++++++++- .../scripts/azure/deeplearning_configure.py | 66 +++++++++++++++++---- .../src/general/scripts/azure/edge_configure.py | 67 ++++++++++++++++++---- .../src/general/scripts/azure/edge_prepare.py | 22 +++++++ .../src/general/scripts/azure/jupyter_configure.py | 44 +++++++++++++- .../src/general/scripts/azure/rstudio_configure.py | 44 +++++++++++++- .../src/general/scripts/azure/tensor_configure.py | 52 +++++++++++++++-- .../general/scripts/azure/zeppelin_configure.py | 44 +++++++++++++- 9 files changed, 358 insertions(+), 35 deletions(-) diff --git a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile index e3c47b2..aeb7b62 100644 --- a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile +++ b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile @@ -68,6 +68,7 @@ COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/dlab/commo COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/dlab/fab.py COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/ COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/ +COPY ${SRC_PATH}edge/templates/locations/ /root/locations/ RUN chmod a+x /root/*.py && \ chmod a+x /root/scripts/* && \ diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py index b0ae221..064bed1 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py @@ -210,6 +210,7 @@ if __name__ == "__main__": data_engine['instance_count'] = int(os.environ['dataengine_instance_count']) data_engine['slave_size'] = os.environ['azure_dataengine_slave_size'] data_engine['dlab_ssh_user'] = os.environ['conf_os_user'] + data_engine['notebook_name'] = os.environ['notebook_instance_name'] master_node_hostname = AzureMeta().get_private_ip_address(data_engine['resource_group_name'], data_engine['master_node_name']) edge_instance_name = '{}-{}-edge'.format(data_engine['service_base_name'], data_engine['user_name']) @@ -378,8 +379,51 @@ if __name__ == "__main__": AzureActions().remove_instance(data_engine['resource_group_name'], data_engine['master_node_name']) sys.exit(1) + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + notebook_instance_ip = AzureMeta().get_private_ip_address(data_engine['resource_group_name'], + data_engine['notebook_name']) + additional_info = { + "computational_name": data_engine['computational_name'], + "master_node_hostname": master_node_hostname, + "notebook_instance_ip": notebook_instance_ip, + "instance_count": data_engine['instance_count'], + "master_node_name": data_engine['master_node_name'], + "slave_node_name": data_engine['slave_node_name'], + "tensor": False + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + keyfile_name, + data_engine['dlab_ssh_user'], + 'spark', + data_engine['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + for i in range(data_engine['instance_count'] - 1): + slave_name = data_engine['slave_node_name'] + '{}'.format(i + 1) + AzureActions().remove_instance(data_engine['resource_group_name'], slave_name) + AzureActions().remove_instance(data_engine['resource_group_name'], data_engine['master_node_name']) + sys.exit(1) try: + ip_address = AzureMeta().get_private_ip_address(data_engine['resource_group_name'], + data_engine['master_node_name']) + spark_master_url = "http://"; + ip_address + ":8080" + spark_master_acces_url = "http://"; + edge_instance_hostname + "/{}/".format( + data_engine['exploratory_name'] + '_' + data_engine['computational_name']) logging.info('[SUMMARY]') print('[SUMMARY]') print("Service base name: {}".format(data_engine['service_base_name'])) @@ -392,7 +436,14 @@ if __name__ == "__main__": res = {"hostname": data_engine['cluster_name'], "instance_id": data_engine['master_node_name'], "key_name": data_engine['key_name'], - "Action": "Create new Data Engine"} + "Action": "Create new Data Engine", + "computational_url": [ + {"description": "Apache Spark Master", + "url": spark_master_acces_url}, + # {"description": "Apache Spark Master (via tunnel)", + # "url": spark_master_url} + ] + } print(json.dumps(res)) result.write(json.dumps(res)) except: diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py index a34b8dd..ab14a5b 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py @@ -252,12 +252,49 @@ if __name__ == "__main__": append_result("Failed creating image.", str(err)) AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) sys.exit(1) + + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + additional_info = { + 'instance_hostname': instance_hostname, + 'tensor': True + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + keyfile_name, + notebook_config['dlab_ssh_user'], + 'jupyter', + notebook_config['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to set edge reverse proxy template.", str(err)) + AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) + sys.exit(1) + try: ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'], notebook_config['instance_name']) - tensor_board_url = 'http://' + ip_address + ':6006' - jupyter_url = 'http://' + ip_address + ':8888/{}/'.format(notebook_config['exploratory_name']) + tensorboard_ip_url = 'http://' + ip_address + ':6006' + jupyter_ip_url = 'http://' + ip_address + ':8888/{}/'.format(notebook_config['exploratory_name']) ungit_ip_url = "http://"; + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) + jupyter_notebook_acces_url = "http://"; + edge_instance_hostname + "/{}/".format( + notebook_config['exploratory_name']) + jupyter_ungit_acces_url = "http://"; + edge_instance_hostname + "/{}-ungit/".format( + notebook_config['exploratory_name']) + tensorboard_acces_url = "http://"; + edge_instance_hostname + "/{}-tensor/".format( + notebook_config['exploratory_name']) print('[SUMMARY]') logging.info('[SUMMARY]') print("Instance name: {}".format(notebook_config['instance_name'])) @@ -269,9 +306,8 @@ if __name__ == "__main__": print("Jupyter URL: {}".format(jupyter_url)) print("Tensor Board URL: {}".format(tensor_board_url)) print("Ungit URL: {}".format(ungit_ip_url)) - print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(notebook_config['key_name'], - notebook_config['dlab_ssh_user'], - ip_address)) + print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format( + notebook_config['key_name'], notebook_config['dlab_ssh_user'], ip_address)) with open("/root/result.json", 'w') as result: res = {"ip": ip_address, @@ -281,13 +317,19 @@ if __name__ == "__main__": "notebook_image_name": notebook_config['notebook_image_name'], "Action": "Create new notebook server", "exploratory_url": [ - {"description": "TensorBoard", - "url": tensor_board_url}, - {"description": "Jupyter", - "url": jupyter_url}, - {"description": "Ungit", - "url": ungit_ip_url} - ]} + {"description": "Jupyter", + "url": jupyter_notebook_acces_url}, + {"description": "TensorBoard", + "url": tensorboard_acces_url}, + {"description": "Ungit", + "url": jupyter_ungit_acces_url}, + {"description": "Jupyter (via tunnel)", + "url": jupyter_ip_url}, + {"description": "TensorBoard (via tunnel)", + "url": tensorboard_ip_url}, + {"description": "Ungit (via tunnel)", + "url": ungit_ip_url} + ]} result.write(json.dumps(res)) except Exception as err: print('Error: {0}'.format(err)) diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py index bedf3f1..7a8b5aa 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py @@ -48,18 +48,23 @@ if __name__ == "__main__": edge_conf['user_keyname'] = os.environ['edge_user_name'] edge_conf['private_subnet_name'] = edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + '-subnet' edge_conf['instance_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge' - edge_conf['network_interface_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge-nif' - edge_conf['static_public_ip_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge-ip' + edge_conf['network_interface_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + \ + '-edge-nif' + edge_conf['static_public_ip_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + \ + '-edge-ip' edge_conf['primary_disk_name'] = edge_conf['instance_name'] + '-disk0' - edge_conf['instance_dns_name'] = 'host-' + edge_conf['instance_name'] + '.' + edge_conf['region'] + '.cloudapp.azure.com' + edge_conf['instance_dns_name'] = 'host-' + edge_conf['instance_name'] + '.' + edge_conf['region'] + \ + '.cloudapp.azure.com' edge_conf['user_storage_account_name'] = edge_conf['service_base_name'] + '-' + edge_conf[ 'user_name'] + '-storage' - edge_conf['user_container_name'] = (edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + '-container').lower() + edge_conf['user_container_name'] = (edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + + '-container').lower() edge_conf['shared_storage_account_name'] = edge_conf['service_base_name'] + '-shared-storage' edge_conf['shared_container_name'] = (edge_conf['service_base_name'] + '-shared-container').lower() edge_conf['datalake_store_name'] = edge_conf['service_base_name'] + '-ssn-datalake' edge_conf['datalake_shared_directory_name'] = edge_conf['service_base_name'] + '-shared-folder' - edge_conf['datalake_user_directory_name'] = '{0}-{1}-folder'.format(edge_conf['service_base_name'], edge_conf['user_name']) + edge_conf['datalake_user_directory_name'] = '{0}-{1}-folder'.format(edge_conf['service_base_name'], + edge_conf['user_name']) edge_conf['edge_security_group_name'] = edge_conf['instance_name'] + '-sg' edge_conf['notebook_security_group_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + \ '-nb-sg' @@ -69,11 +74,12 @@ if __name__ == "__main__": + edge_conf['user_name'] + '-dataengine-slave-sg' edge_conf['dlab_ssh_user'] = os.environ['conf_os_user'] keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], edge_conf['key_name']) - edge_conf['private_subnet_cidr'] = AzureMeta().get_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], + edge_conf['private_subnet_cidr'] = AzureMeta().get_subnet(edge_conf['resource_group_name'], + edge_conf['vpc_name'], edge_conf['private_subnet_name']).address_prefix if os.environ['conf_network_type'] == 'private': edge_conf['edge_private_ip'] = AzureMeta().get_private_ip_address(edge_conf['resource_group_name'], - edge_conf['instance_name']) + edge_conf['instance_name']) edge_conf['edge_public_ip'] = edge_conf['edge_private_ip'] else: edge_conf['edge_public_ip'] = AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'], @@ -91,7 +97,8 @@ if __name__ == "__main__": AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], edge_conf['private_subnet_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name']) - AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['notebook_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['master_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], @@ -131,7 +138,8 @@ if __name__ == "__main__": AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], edge_conf['private_subnet_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name']) - AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['notebook_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['master_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], @@ -162,7 +170,8 @@ if __name__ == "__main__": AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], edge_conf['private_subnet_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name']) - AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['notebook_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['master_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], @@ -202,7 +211,8 @@ if __name__ == "__main__": AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], edge_conf['private_subnet_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name']) - AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['notebook_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['master_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], @@ -236,7 +246,40 @@ if __name__ == "__main__": AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], edge_conf['private_subnet_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name']) - AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['notebook_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['master_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['slave_security_group_name']) + for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']): + if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]: + AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name) + if os.environ['azure_datalake_enable'] == 'true': + for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']): + if edge_conf['datalake_store_name'] == datalake.tags["Name"]: + AzureActions().remove_datalake_directory(datalake.name, edge_conf['datalake_user_directory_name']) + sys.exit(1) + + try: + print('[INSTALLING NGINX REVERSE PROXY]') + logging.info('[INSTALLING NGINX REVERSE PROXY]') + params = "--hostname {} --keyfile {} --user {}" \ + .format(instance_hostname, keyfile_name, edge_conf['dlab_ssh_user']) + try: + local("~/scripts/{}.py {}".format('configure_nginx_reverse_proxy', params)) + except: + traceback.print_exc() + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed installing Nginx reverse proxy. Excpeption: " + str(err)) + AzureActions().remove_instance(edge_conf['resource_group_name'], edge_conf['instance_name']) + AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'], + edge_conf['private_subnet_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name']) + AzureActions().remove_security_group(edge_conf['resource_group_name'], + edge_conf['notebook_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['master_security_group_name']) AzureActions().remove_security_group(edge_conf['resource_group_name'], diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py index 0654ac2..1976482 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py +++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py @@ -164,6 +164,17 @@ if __name__ == "__main__": "direction": "Inbound" }, { + "name": "in-4", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "80", + "source_address_prefix": "*", + "destination_address_prefix": "*", + "access": "Allow", + "priority": 120, + "direction": "Inbound" + }, + { "name": "out-1", "protocol": "Tcp", "source_port_range": "*", @@ -341,6 +352,17 @@ if __name__ == "__main__": }, { "name": "out-17", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "8042", + "source_address_prefix": "*", + "destination_address_prefix": edge_conf['private_subnet_cidr'], + "access": "Allow", + "priority": 190, + "direction": "Outbound" + }, + { + "name": "out-18", "protocol": "*", "source_port_range": "*", "destination_port_range": "*", diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py index 66a979e..065f741 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py @@ -253,12 +253,47 @@ if __name__ == "__main__": append_result("Failed creating image from notebook.", str(err)) AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) sys.exit(1) + + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + additional_info = { + 'instance_hostname': instance_hostname, + 'tensor': False + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + keyfile_name, + notebook_config['dlab_ssh_user'], + 'jupyter', + notebook_config['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to set edge reverse proxy template.", str(err)) + AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) + sys.exit(1) + # generating output information try: ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'], notebook_config['instance_name']) jupyter_ip_url = "http://"; + ip_address + ":8888/{}/".format(notebook_config['exploratory_name']) ungit_ip_url = "http://"; + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) + jupyter_notebook_acces_url = "http://"; + edge_instance_hostname + "/{}/".format( + notebook_config['exploratory_name']) + jupyter_ungit_acces_url = "http://"; + edge_instance_hostname + "/{}-ungit/".format( + notebook_config['exploratory_name']) print('[SUMMARY]') logging.info('[SUMMARY]') print("Instance name: {}".format(notebook_config['instance_name'])) @@ -281,9 +316,14 @@ if __name__ == "__main__": "notebook_image_name": notebook_config['notebook_image_name'], "exploratory_url": [ {"description": "Jupyter", - "url": jupyter_ip_url}, + "url": jupyter_notebook_acces_url}, {"description": "Ungit", - "url": ungit_ip_url}]} + "url": jupyter_ungit_acces_url}, + {"description": "Jupyter (via tunnel)", + "url": jupyter_ip_url}, + {"description": "Ungit (via tunnel)", + "url": ungit_ip_url} + ]} result.write(json.dumps(res)) except Exception as err: append_result("Failed to generate output information", str(err)) diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py index 0b0e293..61e1680 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py @@ -257,12 +257,47 @@ if __name__ == "__main__": append_result("Failed creating image.", str(err)) AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) sys.exit(1) + + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + additional_info = { + 'instance_hostname': instance_hostname, + 'tensor': False + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + keyfile_name, + notebook_config['dlab_ssh_user'], + 'rstudio', + notebook_config['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to set edge reverse proxy template.", str(err)) + AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) + sys.exit(1) + try: # generating output information ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'], notebook_config['instance_name']) rstudio_ip_url = "http://"; + ip_address + ":8787/" ungit_ip_url = "http://"; + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) + rstudio_notebook_acces_url = "http://"; + edge_instance_hostname + "/{}/".format( + notebook_config['exploratory_name']) + rstudio_ungit_acces_url = "http://"; + edge_instance_hostname + "/{}-ungit/".format( + notebook_config['exploratory_name']) print('[SUMMARY]') logging.info('[SUMMARY]') print("Instance name: {}".format(notebook_config['instance_name'])) @@ -287,9 +322,14 @@ if __name__ == "__main__": "Action": "Create new notebook server", "exploratory_url": [ {"description": "RStudio", - "url": rstudio_ip_url}, + "url": rstudio_notebook_acces_url}, {"description": "Ungit", - "url": ungit_ip_url}], + "url": rstudio_ungit_acces_url}, + {"description": "RStudio (via tunnel)", + "url": rstudio_ip_url}, + {"description": "Ungit (via tunnel)", + "url": ungit_ip_url} + ], "exploratory_user": notebook_config['dlab_ssh_user'], "exploratory_pass": notebook_config['rstudio_pass']} result.write(json.dumps(res)) diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py index 094abd8..b502766 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py @@ -249,6 +249,37 @@ if __name__ == "__main__": append_result("Failed creating image.", str(err)) AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) sys.exit(1) + + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + additional_info = { + 'instance_hostname': instance_hostname, + 'tensor': True + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + keyfile_name, + notebook_config['dlab_ssh_user'], + 'jupyter', + notebook_config['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to set edge reverse proxy template.", str(err)) + AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) + sys.exit(1) + # generating output information try: ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'], @@ -256,6 +287,12 @@ if __name__ == "__main__": tensorboard_url = "http://"; + ip_address + ":6006/" jupyter_ip_url = "http://"; + ip_address + ":8888/{}/".format(notebook_config['exploratory_name']) ungit_ip_url = "http://"; + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) + jupyter_notebook_acces_url = "http://"; + edge_instance_hostname + "/{}/".format( + notebook_config['exploratory_name']) + tensorboard_acces_url = "http://"; + edge_instance_hostname + "/{}-tensor/".format( + notebook_config['exploratory_name']) + jupyter_ungit_acces_url = "http://"; + edge_instance_hostname + "/{}-ungit/".format( + notebook_config['exploratory_name']) print('[SUMMARY]') logging.info('[SUMMARY]') print("Instance name: {}".format(notebook_config['instance_name'])) @@ -280,12 +317,19 @@ if __name__ == "__main__": "instance_id": notebook_config['instance_name'], "Action": "Create new notebook server", "exploratory_url": [ - {"description": "TensorBoard", - "url": tensorboard_url}, {"description": "Jupyter", - "url": jupyter_ip_url}, + "url": jupyter_notebook_acces_url}, + {"description": "TensorBoard", + "url": tensorboard_acces_url}, {"description": "Ungit", - "url": ungit_ip_url}]} + "url": jupyter_ungit_acces_url}, + {"description": "Jupyter (via tunnel)", + "url": jupyter_ip_url}, + {"description": "TensorBoard (via tunnel)", + "url": tensorboard_url}, + {"description": "Ungit (via tunnel)", + "url": ungit_ip_url} + ]} result.write(json.dumps(res)) except Exception as err: print('Error: {0}'.format(err)) diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py index 527d4aa..5ca4b77 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py @@ -264,12 +264,47 @@ if __name__ == "__main__": append_result("Failed creating image.", str(err)) AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) sys.exit(1) + + try: + print('[SETUP EDGE REVERSE PROXY TEMPLATE]') + logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') + additional_info = { + 'instance_hostname': instance_hostname, + 'tensor': False + } + params = "--edge_hostname {} " \ + "--keyfile {} " \ + "--os_user {} " \ + "--type {} " \ + "--exploratory_name {} " \ + "--additional_info '{}'"\ + .format(edge_instance_hostname, + keyfile_name, + notebook_config['dlab_ssh_user'], + 'zeppelin', + notebook_config['exploratory_name'], + json.dumps(additional_info)) + try: + local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params)) + except: + append_result("Failed edge reverse proxy template") + raise Exception + except Exception as err: + print('Error: {0}'.format(err)) + append_result("Failed to set edge reverse proxy template.", str(err)) + AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name']) + sys.exit(1) + # generating output information try: ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'], notebook_config['instance_name']) zeppelin_ip_url = "http://"; + ip_address + ":8080/" ungit_ip_url = "http://"; + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) + zeppelin_notebook_acces_url = "http://"; + edge_instance_hostname + "/{}/".format( + notebook_config['exploratory_name']) + zeppelin_ungit_acces_url = "http://"; + edge_instance_hostname + "/{}-ungit/".format( + notebook_config['exploratory_name']) print('[SUMMARY]') logging.info('[SUMMARY]') print("Instance name: {}".format(notebook_config['instance_name'])) @@ -292,9 +327,14 @@ if __name__ == "__main__": "Action": "Create new notebook server", "exploratory_url": [ {"description": "Apache Zeppelin", - "url": zeppelin_ip_url}, + "url": zeppelin_notebook_acces_url}, {"description": "Ungit", - "url": ungit_ip_url}]} + "url": zeppelin_ungit_acces_url}, + {"description": "Apache Zeppelin (via tunnel)", + "url": zeppelin_ip_url}, + {"description": "Ungit (via tunnel)", + "url": ungit_ip_url} + ]} result.write(json.dumps(res)) except Exception as err: print('Error: {0}'.format(err)) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
