This is an automated email from the ASF dual-hosted git repository.

mykolabodnar pushed a commit to branch DLAB-515
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git


The following commit(s) were added to refs/heads/DLAB-515 by this push:
     new da70c67  [DLAB-515]: Notebooks deploing fixed/Hrrps_proxy added
da70c67 is described below

commit da70c673e51199438c950c1022508d2ce37f5ba7
Author: bodnarmykola <[email protected]>
AuthorDate: Mon May 4 11:30:30 2020 +0300

    [DLAB-515]: Notebooks deploing fixed/Hrrps_proxy added
---
 .../src/general/conf/dlab.ini                      | 14 +++----
 .../src/general/lib/os/debian/notebook_lib.py      | 45 ++++++++++++++--------
 .../general/scripts/aws/common_prepare_notebook.py |  2 +-
 3 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini 
b/infrastructure-provisioning/src/general/conf/dlab.ini
index c0b8f43..805b33e 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -62,7 +62,7 @@ allowed_ip_cidr = '0.0.0.0/0'
 network_type = public
 ### Additional tags in format 'Key1:Value1;Key2:Value2'
 # additional_tags =
-pip_version = 9.0.3
+pip_version = 20.1
 ### Billing tag key
 billing_tag_key = product
 ### Billing tag value
@@ -231,7 +231,7 @@ hadoop_version = 2.7
 ### Version of Jupyter to be installed on notebook
 jupyter_version = 6.0.2
 ### Version of TensorFlow to be installed on notebook
-tensorflow_version = 1.8.0
+tensorflow_version = 2.1.0
 ### Version of Zeppelin to be installed on notebook
 zeppelin_version = 0.8.2
 ### Version of Rstudio to be installed on notebook
@@ -245,7 +245,7 @@ multiple_clusters = false
 ### R China mirror
 r_mirror = http://mirror.lzu.edu.cn/CRAN/
 ### NVidia driver version for Tensor/DeepLearning notebooks
-nvidia_version = 440.64.00
+nvidia_version = 418.126.02
 ### Caffe library version for DeepLearning notebook
 caffe_version = 1.0
 ### Caffe2 library version for DeepLearning notebook
@@ -262,13 +262,13 @@ keras_version = 2.1.6
 ### Theano library version for Tensor/DeepLearning notebook
 theano_version = 1.0.3
 ### Version of CUDA
-cuda_version = 10.2
+cuda_version = 10.1
 ### Name of CUDA file
-cuda_file_name = cuda_10.2.89_440.33.01_linux.run
+cuda_file_name = cuda_10.1.243_418.87.00_linux.run
 ### Version of CUDNN
 cudnn_version = 7.6.5
 ### Name of CUDNN file
-cudnn_file_name = cudnn-10.2-linux-x64-v7.6.5.tgz
+cudnn_file_name = cudnn-10.1-linux-x64-v7.6.5.32.tgz
 ### R enabled on Jupyter/Zeppelin notebook
 r_enabled = true
 ### Temporary fixed python libraries due to dependencies
@@ -277,7 +277,7 @@ ipykernel_version = 4.8.2
 ### Version of ungit if previous needed. Use latest as default.
 ungit_version = 1.4.36
 ### Numpy version
-numpy_version = 1.14.3
+numpy_version = 1.18.3
 ### Apache Ivy version
 ivy_version = 2.4.0
 ### Matplotlib version
diff --git 
a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py 
b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index f6af16d..b4b6576 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -140,9 +140,9 @@ def install_rstudio(os_user, local_spark_path, 
rstudio_pass, rstudio_version):
             sudo('''echo 'Sys.setenv(https_proxy = \"{}\")' >> 
/home/{}/.Rprofile'''.format(https_proxy, os_user))
             sudo('rstudio-server start')
             sudo('echo "{0}:{1}" | chpasswd'.format(os_user, rstudio_pass))
-            sudo("sed -i '/exit 0/d' /etc/rc.local")
-            sudo('''bash -c "echo \'sed -i 's/^#SPARK_HOME/SPARK_HOME/' 
/home/{}/.Renviron\' >> /etc/rc.local"'''.format(os_user))
-            sudo("bash -c 'echo exit 0 >> /etc/rc.local'")
+            #sudo("sed -i '/exit 0/d' /etc/rc.local")
+            #sudo('''bash -c "echo \'sed -i 's/^#SPARK_HOME/SPARK_HOME/' 
/home/{}/.Renviron\' >> /etc/rc.local"'''.format(os_user))
+            #sudo("bash -c 'echo exit 0 >> /etc/rc.local'")
             sudo('touch /home/{}/.ensure_dir/rstudio_ensured'.format(os_user))
         except:
             sys.exit(1)
@@ -212,7 +212,7 @@ def ensure_additional_python_libs(os_user):
         try:
             manage_pkg('-y install', 'remote', 'libjpeg8-dev zlib1g-dev')
             if os.environ['application'] in ('jupyter', 'zeppelin'):
-                sudo('pip2 install NumPy=={} SciPy pandas Sympy Pillow sklearn 
--no-cache-dir'.format(os.environ['notebook_numpy_version']))
+                sudo('pip2 install NumPy==1.14.3 SciPy pandas Sympy Pillow 
sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version']))
                 sudo('pip3 install NumPy=={} SciPy pandas Sympy Pillow sklearn 
--no-cache-dir'.format(os.environ['notebook_numpy_version']))
             if os.environ['application'] in ('tensor', 'deeplearning'):
                 sudo('pip2 install opencv-python h5py --no-cache-dir')
@@ -259,8 +259,9 @@ def ensure_python2_libraries(os_user):
 def ensure_python3_libraries(os_user):
     if not exists('/home/' + os_user + 
'/.ensure_dir/python3_libraries_ensured'):
         try:
-            manage_pkg('-y install', 'remote', 'python3-setuptools')
+            #manage_pkg('-y install', 'remote', 'python3-setuptools')
             manage_pkg('-y install', 'remote', 'python3-pip')
+            sudo('pip3 install setuptools')
             try:
                 sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} 
--no-cache-dir' \
                      .format(os.environ['notebook_tornado_version'], 
os.environ['notebook_ipykernel_version']))
@@ -286,38 +287,52 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
             sudo('update-initramfs -u')
             with settings(warn_only=True):
                 reboot(wait=150)
-            manage_pkg('-y install', 'remote', 'dkms')
+            manage_pkg('-y install', 'remote', 'dkms libglvnd-dev')
             kernel_version = run('uname -r | tr -d "[..0-9-]"')
             if kernel_version == 'azure':
                 manage_pkg('-y install', 'remote', 'linux-modules-`uname -r`')
             else:
-                #legacy support for old kernels
-                sudo('if [[ $(apt-cache search linux-image-`uname -r`) ]]; 
then apt-get -y install linux-image-`uname -r`; else apt-get -y install 
linux-modules-`uname -r`; fi;')
-            sudo('wget 
http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run
 -O /home/{1}/NVIDIA-Linux-x86_64-{0}.run'.format(nvidia_version, os_user))
+                # legacy support for old kernels
+                sudo('if [[ $(apt-cache search linux-image-`uname -r`) ]]; 
then apt-get -y '
+                     'install linux-image-`uname -r`; else apt-get -y install 
linux-modules-`uname -r`; fi;')
+            sudo('wget 
http://us.download.nvidia.com/tesla/{0}/NVIDIA-Linux-x86_64-{0}.run -O '
+                 
'/home/{1}/NVIDIA-Linux-x86_64-{0}.run'.format(nvidia_version, os_user))
             sudo('/bin/bash /home/{0}/NVIDIA-Linux-x86_64-{1}.run -s 
--dkms'.format(os_user, nvidia_version))
             sudo('rm -f /home/{0}/NVIDIA-Linux-x86_64-{1}.run'.format(os_user, 
nvidia_version))
             # install cuda
-            sudo('python3.5 -m pip install --upgrade pip=={0} wheel numpy=={1} 
--no-cache-dir'. format(os.environ['conf_pip_version'], 
os.environ['notebook_numpy_version']))
-            sudo('wget -P /opt 
https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}'.format(cuda_version,
 cuda_file_name))
+            sudo('python3 -m pip install --upgrade pip=={0} wheel numpy=={1} 
--no-cache-dir'.format(
+                os.environ['conf_pip_version'], 
os.environ['notebook_numpy_version']))
+            sudo('wget -P /opt 
http://developer.download.nvidia.com/compute/cuda/{0}/Prod/local_installers/{1}'.format(
+                cuda_version, cuda_file_name))
             sudo('sh /opt/{} --silent --toolkit'.format(cuda_file_name))
             sudo('mv /usr/local/cuda-{} /opt/'.format(cuda_version))
             sudo('ln -s /opt/cuda-{0} 
/usr/local/cuda-{0}'.format(cuda_version))
             sudo('rm -f /opt/{}'.format(cuda_file_name))
             # install cuDNN
-            run('wget 
http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O 
/tmp/{1}'.format(cudnn_version, cudnn_file_name))
+            run('wget 
http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O 
/tmp/{1}'.format(
+                cudnn_version, cudnn_file_name))
             run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
             sudo('mkdir -p /opt/cudnn/include')
             sudo('mkdir -p /opt/cudnn/lib64')
             sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
             sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
             sudo('chmod a+r /opt/cudnn/include/cudnn.h 
/opt/cudnn/lib64/libcudnn*')
-            run('echo "export 
LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> 
~/.bashrc')
+            run(
+                'echo "export 
LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> 
~/.bashrc')
             # install TensorFlow and run TensorBoard
-            sudo('python2.7 -m pip install --upgrade 
https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl
 --no-cache-dir'.format(tensorflow_version))
-            sudo('python3 -m pip install --upgrade 
https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl
 --no-cache-dir'.format(tensorflow_version))
+            # sudo('python2.7 -m pip install --upgrade 
https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl
 --no-cache-dir'.format(tensorflow_version))
+            sudo('python3 -m pip install --upgrade '
+                 
'https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp36-cp36m-manylinux2010_x86_64.whl'
+                 ' --no-cache-dir'.format(tensorflow_version))
             sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R 
/var/log/tensorboard'.format(os_user))
             put('{}tensorboard.service'.format(templates_dir), 
'/tmp/tensorboard.service')
             sudo("sed -i 's|OS_USR|{}|' 
/tmp/tensorboard.service".format(os_user))
+            http_proxy = run('echo $http_proxy')
+            https_proxy = run('echo $https_proxy')
+            sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTP_PROXY={}\"\'  
/tmp/tensorboard.service'.format(
+                http_proxy))
+            sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\'  
/tmp/tensorboard.service'.format(
+                https_proxy))
             sudo("chmod 644 /tmp/tensorboard.service")
             sudo('\cp /tmp/tensorboard.service /etc/systemd/system/')
             sudo("systemctl daemon-reload")
diff --git 
a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
 
b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index 5c481ac..5a1a6f0 100644
--- 
a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++ 
b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -78,7 +78,7 @@ if __name__ == "__main__":
                                                                       
notebook_config['project_name'],
                                                                       
notebook_config['endpoint_name'],
                                                                       
notebook_config['exploratory_name'], args.uuid)
-        notebook_config['primary_disk_size'] = (lambda x: '30' if x == 
'deeplearning' else '12')(
+        notebook_config['primary_disk_size'] = (lambda x: '30' if x == 
'deeplearning' else '16')(
             os.environ['application'])
         notebook_config['role_profile_name'] = '{}-{}-{}-nb-de-profile'.format(
             notebook_config['service_base_name'], 
notebook_config['project_name'], notebook_config['endpoint_name'])


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to