This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DATALAB-2398 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 294316085f3257cdc4929f88c1b4187b59326cd6 Author: bodnarmykola <[email protected]> AuthorDate: Fri May 28 12:19:29 2021 +0300 [DATALAB-2398] - Installing libs for python venv fixed --- .../src/general/lib/os/fab.py | 54 ++++++++++++++-------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index 8b54a2e..9447242 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -79,13 +79,14 @@ def install_pip_pkg(requisites, pip_version, lib_group): status = list() error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:" try: - if pip_version == 'pip3' and not exists(conn, '/bin/pip3'): - for v in range(4, 8): - if exists(conn, '/bin/pip3.{}'.format(v)): - conn.sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v)) - conn.sudo('{} install -U pip=={} setuptools=={}'.format(pip_version, os.environ['conf_pip_version'], os.environ['notebook_setuptools_version'])) - conn.sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version'])) - conn.sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version'])) + venv_install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3]) + #if pip_version == 'pip3' and not exists(conn, '/bin/pip3'): + # for v in range(4, 8): + # if exists(conn, '/bin/pip3.{}'.format(v)): + # conn.sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v)) + #conn.sudo('{} install -U pip=={} setuptools=={}'.format(pip_version, os.environ['conf_pip_version'], os.environ['notebook_setuptools_version'])) + #conn.sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version'])) + #conn.sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version'])) for pip_pkg in requisites: name, vers = pip_pkg if pip_pkg[1] == '' or pip_pkg[1] == 'N/A': @@ -94,20 +95,30 @@ def install_pip_pkg(requisites, pip_version, lib_group): else: version = pip_pkg[1] pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1]) - conn.sudo('{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > ' - ' /tmp/{0}install_{3}.log; then echo "" > /tmp/{0}install_{3}.log;fi'.format(pip_version, pip_pkg, error_parser, name)) - err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace('"', "'").replace('\n', ' ') - conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "not_found" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name)) - res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '') - conn.sudo('cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name)) - installed_out = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '') + conn.sudo( + '''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; then echo "" > /tmp/{4}install_{3}.log;fi' '''.format( + venv_install_command, pip_pkg, error_parser, name, pip_version)) + err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace( + '"', "'").replace('\n', ' ') + conn.sudo( + '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format( + venv_install_command, name, pip_version)) + res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace( + '\n', '') + conn.sudo( + '''bash -l -c 'cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi' '''.format( + pip_version, name)) + installed_out = conn.sudo( + '''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '') changed_pip_pkg = False if 'not_found' in res: changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-') changed_pip_pkg = changed_pip_pkg[0] - conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > ' - '/tmp/{0}install_{1}.list;fi'.format(pip_version, changed_pip_pkg)) - res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace('\n', '') + conn.sudo( + '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format( + venv_install_command, changed_pip_pkg, pip_version)) + res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace( + '\n', '') if err and name not in installed_out: status_msg = 'installation_error' if 'ERROR: No matching distribution found for {}'.format(name) in err: @@ -120,10 +131,11 @@ def install_pip_pkg(requisites, pip_version, lib_group): version = [i for i in ver if changed_pip_pkg.lower() in i][0].split('==')[1] else: version = \ - [i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1] + [i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1] status_msg = "installed" versions = [] - if 'Could not find a version that satisfies the requirement' in err and 'ERROR: No matching distribution found for {}=='.format(name) in err: + if 'Could not find a version that satisfies the requirement' in err and 'ERROR: No matching distribution found for {}=='.format( + name) in err: versions = err[err.find("(from versions: ") + 16: err.find(") ")] if versions != '' and versions != 'none': versions = versions.split(', ') @@ -132,7 +144,9 @@ def install_pip_pkg(requisites, pip_version, lib_group): else: versions = [] - conn.sudo('cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi'.format(pip_version, name)) + conn.sudo( + '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi' '''.format( + pip_version, name)) dep = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:] if dep == '': dep = [] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
