This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2691 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c5a62ff056c64c96b328fafa9922efb4b8ad56e3 Author: leonidfrolov <[email protected]> AuthorDate: Wed Mar 16 16:18:29 2022 +0200 [DATALAB-2691]: fixed python libs install for aws emr on zeppelin --- .../src/general/lib/aws/actions_lib.py | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py index d74b6dc..dfddc10 100644 --- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py +++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py @@ -1591,7 +1591,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p subprocess.run('wget https://www.python.org/ftp/python/' + python_version + '/Python-' + python_version + '.tgz -O /tmp/Python-' + python_version + '.tgz', shell=True, check=True) subprocess.run('tar zxvf /tmp/Python-' + python_version + '.tgz -C /tmp/', shell=True, check=True) - subprocess.run('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(python_version), shell=True, check=True) + subprocess.run('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ ' + '--with-ensurepip=install'.format(python_version), shell=True, check=True) subprocess.run('cd /tmp/Python-{0}; sudo make altinstall'.format(python_version), shell=True, check=True) subprocess.run('cd /tmp/; sudo rm -rf Python-' + python_version + '/', shell=True, check=True) if region == 'cn-north-1': @@ -1602,8 +1603,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p subprocess.run('sudo echo "[global]" >> /etc/pip.conf', shell=True, check=True) subprocess.run('sudo echo "timeout = 600" >> /etc/pip.conf', shell=True, check=True) subprocess.run('sudo -i virtualenv /opt/python/python' + python_version, shell=True, check=True) - venv_command = '/bin/bash source /opt/python/python' + python_version + '/bin/activate' - pip_command = '/opt/python/python' + python_version + '/bin/pip' + python_version[:3] + venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version) + pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3]) if region == 'cn-north-1': try: subprocess.run(venv_command + ' && sudo -i ' + pip_command + @@ -1634,24 +1635,23 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p subprocess.run('sudo rm -rf /opt/python/python{}/'.format(python_version), shell=True, check=True) sys.exit(1) else: - print(subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install -U pip==9.0.3', shell=True, check=True)) - subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install pyzmq==17.0.0', shell=True, check=True) - subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install ipython ipykernel --no-cache-dir', shell=True, check=True) - subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install NumPy=={}'.format(numpy_version), shell=True, check=True) - subprocess.run(venv_command + ' && sudo -i ' + pip_command + - ' install boto boto3 SciPy Matplotlib=={} pandas Sympy Pillow sklearn ' - '--no-cache-dir'.format(os.environ['notebook_matplotlib_version']), shell=True, check=True) + for lib in ['-U pip==9.0.3', 'pyzmq==17.0.0', + 'ipython ipykernel boto boto3 pybind11 pythran cython NumPy=={} Matplotlib=={} --no-cache-dir' + .format(numpy_version, os.environ['notebook_matplotlib_version']), + 'SciPy pandas Sympy Pillow', 'sklearn --no-cache-dir']: + subprocess.run('bash -c "{0} && sudo -i {1} install {2}"' + .format(venv_command, pip_command, lib), shell=True, check=True) # Need to refactor when we add GPU cluster if application == 'deeplearning': - subprocess.run(venv_command + ' && sudo -i ' + pip_command + - ' install mxnet-cu80 opencv-python keras Theano --no-cache-dir', shell=True, check=True) + subprocess.run('bash -c "{0} && sudo -i {1} install mxnet-cu80 opencv-python keras Theano ' + '--no-cache-dir"'.format(venv_command, pip_command), shell=True, check=True) python_without_dots = python_version.replace('.', '') - subprocess.run(venv_command + ' && sudo -i ' + pip_command + - ' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl ' - '--no-cache-dir'.format(python_without_dots[:2]), shell=True, check=True) + subprocess.run('bash -c "{0} && sudo -i {1} install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}' + '-cp{0}m-linux_x86_64.whl --no-cache-dir"' + .format(venv_command, pip_command, python_without_dots[:2]), shell=True, check=True) subprocess.run('sudo rm -rf /usr/bin/python{}-dp'.format(python_version[0:3]), shell=True, check=True) - subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version, - python_version[0:3]), shell=True, check=True) + subprocess.run('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp' + .format(python_version, python_version[0:3]), shell=True, check=True) def spark_defaults(args): --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
