This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2572 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit fca8c124a55079d4802132397772b477e953a40e Author: leonidfrolov <[email protected]> AuthorDate: Tue Mar 22 15:45:42 2022 +0200 [DATALAB-2572]: fixed libs install for gcp deeplearning --- .../src/general/lib/os/debian/notebook_lib.py | 26 +++++++++++----------- .../src/general/lib/os/fab.py | 10 ++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py index cb468f7..66f347c 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py @@ -273,24 +273,24 @@ def ensure_python3_libraries(os_user): manage_pkg('-y install', 'remote', 'libkrb5-dev') manage_pkg('-y install', 'remote', 'libbz2-dev libsqlite3-dev tk-dev libncursesw5-dev libreadline-dev ' 'liblzma-dev uuid-dev lzma-dev libgdbm-dev') #necessary for python build - datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff') + datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff') if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true': - datalab.fab.conn.sudo('pip3 install --upgrade --user pyqt5==5.12') - datalab.fab.conn.sudo('pip3 install --upgrade --user pyqtwebengine==5.12') - datalab.fab.conn.sudo('pip3 install setuptools') + datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqt5==5.12') + datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqtwebengine==5.12') + datalab.fab.conn.sudo('-i pip3 install setuptools') else: - datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version'])) + datalab.fab.conn.sudo('-i pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version'])) try: - datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ + datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], os.environ['notebook_nbconvert_version'])) except: - datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ + datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], os.environ['notebook_nbconvert_version'])) - datalab.fab.conn.sudo('pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version'])) - datalab.fab.conn.sudo('pip3 install boto3 --no-cache-dir') - datalab.fab.conn.sudo('pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir') + datalab.fab.conn.sudo('-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version'])) + datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir') + datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir') datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured') except: sys.exit(1) @@ -561,11 +561,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version): 'libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev ' 'libopencv-dev libopenmpi-dev libsnappy-dev openmpi-bin openmpi-doc python-pydot') datalab.fab.conn.sudo( - 'pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image ' + '-i pip3 install flask graphviz hypothesis jupyter matplotlib=={} pydot python-nvd3 pyyaml requests scikit-image ' 'scipy tornado --no-cache-dir'.format(os.environ['notebook_matplotlib_version'])) if os.environ['application'] == 'deeplearning': manage_pkg('-y install', 'remote', 'cmake') - datalab.fab.conn.sudo('pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html') + datalab.fab.conn.sudo('-i pip3 install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html') else: # datalab.fab.conn.sudo('mkdir /opt/cuda-{}'.format(os.environ['notebook_cuda_version'])) # datalab.fab.conn.sudo('mkdir /opt/cuda-{}/include/'.format(os.environ['notebook_cuda_version'])) @@ -605,7 +605,7 @@ def install_theano(os_user, theano_version): def install_mxnet(os_user, mxnet_version): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)): - datalab.fab.conn.sudo('pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version)) + datalab.fab.conn.sudo('-i pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version)) datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user)) diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index df8b3c4..7617b93 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -969,13 +969,13 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyter_ensured'): try: if os.environ['conf_deeplearning_cloud_ami'] == 'false' or os.environ['application'] != 'deeplearning': - conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version)) - conn.sudo('pip3 install jupyter --no-cache-dir') + conn.sudo('-i pip3 install notebook=={} --no-cache-dir'.format(jupyter_version)) + conn.sudo('-i pip3 install jupyter --no-cache-dir') conn.sudo('rm -rf {}'.format(jupyter_conf_file)) elif os.environ['application'] != 'tensor': - conn.sudo('pip3 install environment_kernels') - if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning - conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user)) + conn.sudo('-i pip3 install environment_kernels') + #if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning + # conn.sudo( pip3 install flask'chown -R {0}:{0} /home/{0}/.local'.format(os_user)) conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file)) conn.run('mkdir -p ~/.jupyter/custom/') conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css') --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
