This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-1837 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 22f8380b4cf38aa17dd795ba1c64c1ff91f27c52 Author: leonidfrolov <[email protected]> AuthorDate: Fri Oct 8 12:42:28 2021 +0300 [DATALAB-1837]: added additional disk mount checks for azure notebooks --- .../src/general/lib/azure/actions_lib.py | 23 ++++++++++++++-------- .../scripts/azure/common_create_notebook_image.py | 2 ++ .../scripts/azure/deeplearning_configure.py | 6 +++++- .../src/general/scripts/azure/jupyter_configure.py | 6 +++++- .../general/scripts/azure/jupyterlab_configure.py | 6 +++++- .../src/general/scripts/azure/rstudio_configure.py | 6 +++++- .../src/general/scripts/azure/tensor_configure.py | 6 +++++- .../general/scripts/azure/zeppelin_configure.py | 6 +++++- 8 files changed, 47 insertions(+), 14 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py index b83aba7..09ec650 100644 --- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py @@ -1232,15 +1232,22 @@ def remount_azure_disk(creds=False, os_user='', hostname='', keyfile=''): if creds: conn.close() -def ensure_right_mount_paths(): - opt_disk = datalab.fab.conn.sudo("cat /etc/fstab | grep /opt/ | awk '{print $1}'").stdout.split('\n')[0].split('/')[2] - if opt_disk not in datalab.fab.conn.sudo("lsblk | grep /opt", warn=True).stdout or opt_disk in datalab.fab.conn.sudo("fdisk -l | grep 'BIOS boot'").stdout: - disk_names = datalab.fab.conn.sudo("lsblk | grep disk | awk '{print $1}' | sort").stdout.split('\n') +def ensure_right_mount_paths(creds=False, os_user='', hostname='', keyfile=''): + if creds: + global conn + conn = datalab.fab.init_datalab_connection(hostname, os_user, keyfile) + else: + conn = datalab.fab.conn + opt_disk = conn.sudo("cat /etc/fstab | grep /opt/ | awk '{print $1}'").stdout.split('\n')[0].split('/')[2] + if opt_disk not in conn.sudo("lsblk | grep /opt", warn=True).stdout or opt_disk in conn.sudo("fdisk -l | grep 'BIOS boot'").stdout: + disk_names = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort").stdout.split('\n') for disk in disk_names: - if disk != '' and disk not in datalab.fab.conn.sudo('lsblk | grep -E "(mnt|media)"').stdout and disk not in datalab.fab.conn.sudo("fdisk -l | grep 'BIOS boot'").stdout: - datalab.fab.conn.sudo("umount -l /opt") - datalab.fab.conn.sudo("mount /dev/{}1 /opt".format(disk)) - datalab.fab.conn.sudo('sed -i "/opt/ s|/dev/{}|/dev/{}1|g" /etc/fstab'.format(opt_disk, disk)) + if disk != '' and disk not in conn.sudo('lsblk | grep -E "(mnt|media)"').stdout and disk not in conn.sudo("fdisk -l | grep 'BIOS boot'").stdout: + conn.sudo("umount -l /opt") + conn.sudo("mount /dev/{}1 /opt".format(disk)) + conn.sudo('sed -i "/opt/ s|/dev/{}|/dev/{}1|g" /etc/fstab'.format(opt_disk, disk)) + if creds: + conn.close() def prepare_vm_for_image(creds=False, os_user='', hostname='', keyfile=''): if creds: diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py index 294d7a8..4b538f9 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py +++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py @@ -96,6 +96,8 @@ if __name__ == "__main__": image_conf['instance_name']) datalab.actions_lib.remount_azure_disk(True, image_conf['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) datalab.fab.set_git_proxy(image_conf['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_hostname)) additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"} diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py index be91e28..97cb24f 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py @@ -218,7 +218,9 @@ if __name__ == "__main__": try: subprocess.run("~/scripts/{}.py {}".format('configure_deep_learning_node', params), shell=True, check=True) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, - os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem") + keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) except: traceback.print_exc() raise Exception @@ -283,6 +285,8 @@ if __name__ == "__main__": notebook_config['instance_name']) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_private_hostname)) additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"} diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py index 2e939b9..1388b57 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py @@ -201,7 +201,9 @@ if __name__ == "__main__": try: subprocess.run("~/scripts/{}.py {}".format('configure_jupyter_node', params), shell=True, check=True) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, - os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem") + keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) except: traceback.print_exc() raise Exception @@ -285,6 +287,8 @@ if __name__ == "__main__": notebook_config['instance_name']) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_private_hostname)) additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"} diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py index 8c694e1..ca55247 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py @@ -201,7 +201,9 @@ if __name__ == "__main__": try: subprocess.run("~/scripts/{}.py {}".format('configure_jupyterlab_node', params), shell=True, check=True) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, - os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem") + keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) except: traceback.print_exc() raise Exception @@ -266,6 +268,8 @@ if __name__ == "__main__": notebook_config['instance_name']) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], + instance_hostname, keyfile_name) datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_private_hostname)) additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"} diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py index 4ed3ad7..38d253a 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py @@ -201,7 +201,9 @@ if __name__ == "__main__": try: subprocess.run("~/scripts/{}.py {}".format('configure_rstudio_node', params), shell=True, check=True) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, - os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem") + keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) except: traceback.print_exc() raise Exception @@ -284,6 +286,8 @@ if __name__ == "__main__": notebook_config['instance_name']) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], + instance_hostname, keyfile_name) datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_private_hostname)) additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"} diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py index 137f355..6b46434 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py @@ -199,7 +199,9 @@ if __name__ == "__main__": try: subprocess.run("~/scripts/{}.py {}".format('configure_tensor_node', params), shell=True, check=True) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, - os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem") + keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) except: traceback.print_exc() raise Exception @@ -282,6 +284,8 @@ if __name__ == "__main__": notebook_config['instance_name']) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], + instance_hostname, keyfile_name) datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_private_hostname)) additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"} diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py index b4a29b4..9ece655 100644 --- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py +++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py @@ -212,7 +212,9 @@ if __name__ == "__main__": try: subprocess.run("~/scripts/{}.py {}".format('configure_zeppelin_node', params), shell=True, check=True) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, - os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem") + keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], instance_hostname, + keyfile_name) except: traceback.print_exc() raise Exception @@ -295,6 +297,8 @@ if __name__ == "__main__": notebook_config['instance_name']) datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) + datalab.actions_lib.ensure_right_mount_paths(True, notebook_config['datalab_ssh_user'], + instance_hostname, keyfile_name) datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name, 'http://{}:3128'.format(edge_instance_private_hostname)) additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
