This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2372
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2372 by this push:
new 1d85ccc [DataLab-2372] - [AWS][Azure] Deeplearning notebook updated
for using aws predefined image
1d85ccc is described below
commit 1d85ccc92d581c2c6128c856ad892613d78b6087
Author: bodnarmykola <[email protected]>
AuthorDate: Fri Apr 30 13:44:41 2021 +0300
[DataLab-2372] - [AWS][Azure] Deeplearning notebook updated for using aws
predefined image
---
.../src/deeplearning/scripts/configure_deep_learning_node.py | 4 ++--
infrastructure-provisioning/src/general/conf/datalab.ini | 4 ++--
.../src/general/files/aws/deeplearning_description.json | 4 ++--
.../src/general/files/azure/deeplearning_description.json | 6 +++---
infrastructure-provisioning/src/general/lib/os/fab.py | 6 +++---
.../src/general/scripts/aws/common_prepare_notebook.py | 4 ++--
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git
a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index 39642f0..9678bc8 100644
---
a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++
b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -106,7 +106,7 @@ if __name__ == "__main__":
print("Mount additional volume")
prepare_disk(args.os_user)
- if not os.environ['aws_deeplearning_image_name']:
+ if os.environ['conf_deeplearning_cloud_ami'] == 'false':
# INSTALL LANGUAGES
print("Install Java")
ensure_jre_jdk(args.os_user)
@@ -156,7 +156,7 @@ if __name__ == "__main__":
ensure_additional_python_libs(args.os_user)
print("Install Matplotlib")
ensure_matplot(args.os_user)
- else:
+ elif os.environ['conf_deeplearning_cloud_ami'] == 'true':
# CONFIGURE JUPYTER NOTEBOOK
print("Configure Jupyter")
configure_jupyter(args.os_user, jupyter_conf_file, templates_dir,
args.jupyter_version, args.exploratory_name)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini
b/infrastructure-provisioning/src/general/conf/datalab.ini
index 480c6c9..b92b732 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -91,6 +91,8 @@ repository_user =
repository_pass =
### Repository url
repository_address =
+### Deeplearning native cloud AMI enabled
+deeplearning_cloud_ami = true
#--- [aws] section contains all common parameters related to Amazon ---#
[aws]
@@ -126,8 +128,6 @@ region = us-west-2
debian_image_name =
ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20201026
### Amazon ami name based on RedHat conf_os_family for all DataLab instances
redhat_image_name = RHEL-7.4_HVM-20180103-x86_64-2-Hourly2-GP2
-### Amazon ami name for Deeplearning notebook
-deeplearning_image_name = Deep Learning AMI (Ubuntu 18.04) Version 42.1
### Amazon account ID
# account_id =
### Amazon billing bucket
diff --git
a/infrastructure-provisioning/src/general/files/aws/deeplearning_description.json
b/infrastructure-provisioning/src/general/files/aws/deeplearning_description.json
index 5339445..a0519a8 100644
---
a/infrastructure-provisioning/src/general/files/aws/deeplearning_description.json
+++
b/infrastructure-provisioning/src/general/files/aws/deeplearning_description.json
@@ -8,10 +8,10 @@
"exploratory_environment_versions" :
[
{
- "template_name": "AWS Deep Learning 42.1",
+ "template_name": "Deep Learning AMI (Ubuntu 18.04) Version 42.1",
"description": "MXNet-1.8.0 & 1.7.0, TensorFlow-2.4.1, 2.1.3 & 1.15.5,
PyTorch-1.4.0 & 1.8.0, Neuron, & others. NVIDIA CUDA, cuDNN, NCCL, Intel
MKL-DNN, Docker, NVIDIA-Docker & EFA support. Uses Anaconda virtual
environments, configured to keep the different framework installations separate
and easy to switch between frameworks as Jupyter kernels.",
"environment_type": "exploratory",
- "version": "deeplearning-42.1",
+ "version": "Deep Learning AMI (Ubuntu 18.04) Version 42.1",
"vendor": "AWS"
}
]
diff --git
a/infrastructure-provisioning/src/general/files/azure/deeplearning_description.json
b/infrastructure-provisioning/src/general/files/azure/deeplearning_description.json
index 9094ca0..c0db4b8 100644
---
a/infrastructure-provisioning/src/general/files/azure/deeplearning_description.json
+++
b/infrastructure-provisioning/src/general/files/azure/deeplearning_description.json
@@ -8,10 +8,10 @@
"exploratory_environment_versions" :
[
{
- "template_name": "Deep Learning 2.4",
- "description": "Base image with Deep Learning and Jupyter",
+ "template_name": "Data Science Virtual Machine - Ubuntu 18.04",
+ "description": "Pre-configured with NVIDIA drivers, CUDA Toolkit, and
cuDNN library for GPU workloads with the following highlights: Jupyter,
JupyterLab, and JupyterHub; Deep learning with TensorFlow and PyTorch; Machine
learning with xgboost, Vowpal Wabbit, and LightGBM; Julia; Azure SDKs and
libraries; Azure Machine Learning SDKs and sample notebooks; R support; Spark.
Uses Anaconda virtual environments, configured to keep the different framework
installations separate and easy t [...]
"environment_type": "exploratory",
- "version": "deeplearning-2.4",
+ "version": "microsoft-dsvm:ubuntu-1804:1804-gen2",
"vendor": "Azure"
}
]
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py
b/infrastructure-provisioning/src/general/lib/os/fab.py
index 18d3818..89e1ba0 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -191,7 +191,7 @@ def put_resource_status(resource, status, datalab_path,
os_user, hostname):
def configure_jupyter(os_user, jupyter_conf_file, templates_dir,
jupyter_version, exploratory_name):
if not exists(conn,'/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
- if not os.environ['aws_deeplearning_image_name']:
+ if os.environ['conf_deeplearning_cloud_ami'] == 'false' or
os.environ['application'] != 'deeplearning':
conn.sudo('pip3 install notebook=={}
--no-cache-dir'.format(jupyter_version))
conn.sudo('pip3 install jupyter --no-cache-dir')
conn.sudo('rm -rf {}'.format(jupyter_conf_file))
@@ -204,7 +204,7 @@ def configure_jupyter(os_user, jupyter_conf_file,
templates_dir, jupyter_version
conn.sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >>
{1}'.format(id_generator(), jupyter_conf_file))
conn.sudo('''echo "c.NotebookApp.token = u''" >>
{}'''.format(jupyter_conf_file))
conn.sudo('echo \'c.KernelSpecManager.ensure_native_kernel =
False\' >> {}'.format(jupyter_conf_file))
- if os.environ['aws_deeplearning_image_name']:
+ if os.environ['conf_deeplearning_cloud_ami'] == 'true':
conn.sudo(
'''echo "c.NotebookApp.kernel_spec_manager_class =
'environment_kernels.EnvironmentKernelSpecManager'" >> {}'''.format(
jupyter_conf_file))
@@ -215,7 +215,7 @@ def configure_jupyter(os_user, jupyter_conf_file,
templates_dir, jupyter_version
conn.sudo("chmod 644 /tmp/jupyter-notebook.service")
if os.environ['application'] == 'tensor':
conn.sudo("sed -i '/ExecStart/s|-c \"|-c \"export
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; |g'
/tmp/jupyter-notebook.service")
- elif os.environ['application'] == 'deeplearning' and not
os.environ['aws_plearning_image_name']:
+ elif os.environ['application'] == 'deeplearning' and
os.environ['conf_deeplearning_cloud_ami'] == 'false':
conn.sudo("sed -i '/ExecStart/s|-c \"|-c \"export
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64:/usr/lib64/openmpi/lib:
; export PYTHONPATH=/home/" + os_user +
"/caffe/python:/home/" + os_user +
"/pytorch/build:$PYTHONPATH ; |g' /tmp/jupyter-notebook.service")
conn.sudo("sed -i 's|CONF_PATH|{}|'
/tmp/jupyter-notebook.service".format(jupyter_conf_file))
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index 17fde95..270ae56 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -104,8 +104,8 @@ if __name__ == "__main__":
notebook_config['ami_id'] =
datalab.meta_lib.get_ami_id(os.environ['aws_{}_image_name'.format(
os.environ['conf_os_family'])])
image_id =
datalab.meta_lib.get_ami_id_by_name(notebook_config['notebook_image_name'],
'available')
- if os.environ['aws_deeplearning_image_name'] and
os.environ['application'] == 'deeplearning':
- image_id =
datalab.meta_lib.get_ami_id(os.environ['aws_deeplearning_image_name'])
+ if os.environ['conf_deeplearning_cloud_ami'] == 'true' and
os.environ['application'] == 'deeplearning' and image_id == '':
+ image_id =
datalab.meta_lib.get_ami_id(notebook_config['notebook_image_name'])
if image_id != '':
notebook_config['ami_id'] = image_id
print('Pre-configured image found. Using:
{}'.format(notebook_config['ami_id']))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
