This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2572 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6b56ac7713f249d89bb81361755bf236bc8965e9 Author: leonidfrolov <[email protected]> AuthorDate: Tue Mar 29 11:26:17 2022 +0300 [DATALAB-2572]: changed pip libs install for deeplearning --- .../src/general/lib/azure/actions_lib.py | 2 ++ .../src/general/lib/azure/meta_lib.py | 2 ++ .../src/general/lib/os/debian/notebook_lib.py | 42 +++++++++++++++------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py index 09ec650..1de7e69 100644 --- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py @@ -62,6 +62,8 @@ class AzureActions: client_secret=json.dumps(self.sp_creds['clientSecret']).replace('"', ''), client_id=json.dumps(self.sp_creds['clientId']).replace('"', ''), resource='https://datalake.azure.net/') + logger = logging.getLogger('azure') + logger.setLevel(logging.ERROR) def create_resource_group(self, resource_group_name, region): try: diff --git a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py index 510e875..3447e7e 100644 --- a/infrastructure-provisioning/src/general/lib/azure/meta_lib.py +++ b/infrastructure-provisioning/src/general/lib/azure/meta_lib.py @@ -53,6 +53,8 @@ class AzureMeta: client_secret=json.dumps(self.sp_creds['clientSecret']).replace('"', ''), client_id=json.dumps(self.sp_creds['clientId']).replace('"', ''), resource='https://datalake.azure.net/') + logger = logging.getLogger('azure') + logger.setLevel(logging.ERROR) def get_resource_group(self, resource_group_name): try: diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py index 66f347c..5c51d62 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py @@ -274,23 +274,39 @@ def ensure_python3_libraries(os_user): manage_pkg('-y install', 'remote', 'libbz2-dev libsqlite3-dev tk-dev libncursesw5-dev libreadline-dev ' 'liblzma-dev uuid-dev lzma-dev libgdbm-dev') #necessary for python build datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff') - if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true': + if os.environ['conf_cloud_provider'] == 'aws' and os.environ['conf_deeplearning_cloud_ami'] == 'true': + datalab.fab.conn.sudo('-i pip3 install -U keyrings.alt backoff') datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqt5==5.12') datalab.fab.conn.sudo('-i pip3 install --upgrade --user pyqtwebengine==5.12') datalab.fab.conn.sudo('-i pip3 install setuptools') + try: + datalab.fab.conn.sudo( + '-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ + .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], + os.environ['notebook_nbconvert_version'])) + except: + datalab.fab.conn.sudo( + '-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ + .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], + os.environ['notebook_nbconvert_version'])) + datalab.fab.conn.sudo( + '-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version'])) + datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir') + datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir') else: - datalab.fab.conn.sudo('-i pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version'])) - try: - datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ - .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], - os.environ['notebook_nbconvert_version'])) - except: - datalab.fab.conn.sudo('-i pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ - .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], - os.environ['notebook_nbconvert_version'])) - datalab.fab.conn.sudo('-i pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version'])) - datalab.fab.conn.sudo('-i pip3 install boto3 --no-cache-dir') - datalab.fab.conn.sudo('-i pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir') + datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff') + datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version'])) + try: + datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.21.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ + .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], + os.environ['notebook_nbconvert_version'])) + except: + datalab.fab.conn.sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} nbconvert=={2} sparkmagic --no-cache-dir' \ + .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'], + os.environ['notebook_nbconvert_version'])) + datalab.fab.conn.sudo('pip3 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version'])) + datalab.fab.conn.sudo('pip3 install boto3 --no-cache-dir') + datalab.fab.conn.sudo('pip3 install fabvenv fabric-virtualenv future patchwork --no-cache-dir') datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured') except: sys.exit(1) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
