This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DLAB-1966 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit c1d23dc763b0ca777fb6fd1add9770e67c497b60 Author: leonidfrolov <[email protected]> AuthorDate: Tue Jul 28 18:14:27 2020 +0300 [DLAB-1966]: fixed R package installation for dataengine-service, yum installation, some other pip, R and apt improvements --- .../src/general/lib/os/debian/notebook_lib.py | 6 ++- .../src/general/lib/os/fab.py | 14 +++++-- .../src/general/lib/os/redhat/notebook_lib.py | 47 +++++++++++++++++++--- .../general/scripts/os/install_additional_libs.py | 3 ++ 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py index d3bcefa..7a9a12b 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py @@ -380,7 +380,7 @@ def install_os_pkg(requisites): sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install --allow-downgrades {0} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > ' '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(os_pkg, error_parser)) err = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)).replace('"', "'") - sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 20 "({1})" /tmp/tee.tmp > ' + sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})" /tmp/tee.tmp > ' '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(os_pkg, new_pkgs_parser)) dep = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)) if dep == '': @@ -390,7 +390,9 @@ def install_os_pkg(requisites): .replace('\n', '').replace(' ', ' ').replace(' {} '.format(os_pkg.split("=")[0]), ' ').strip().split(' ') for n, i in enumerate(dep): - dep[n] = sudo('apt show {} 2>&1 | grep Version:'.format(i)).replace('Version: ', '{} v.'.format(i)) + sudo('apt show {0} 2>&1 | if ! grep Version: > ' + '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(i)) + dep[n] =sudo('cat /tmp/os_install_{}.log'.format(i)).replace('Version: ', '{} v.'.format(i)) dep = [i for i in dep if i] versions = [] sudo('apt list --installed | if ! grep {0}/ > /tmp/os_install_{1}.list; then echo "" > /tmp/os_install_{1}.list;fi'.format(os_pkg.split("=")[0], os_pkg)) diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index a277cf1..6b9d47c 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -54,7 +54,7 @@ def dataengine_dir_prepare(cluster_dir): def install_pip_pkg(requisites, pip_version, lib_group): status = list() - error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires" + error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:" try: if pip_version == 'pip3' and not exists('/bin/pip3'): sudo('ln -s /bin/pip3.5 /bin/pip3') @@ -111,7 +111,9 @@ def install_pip_pkg(requisites, pip_version, lib_group): if i == pip_pkg.split("==")[0]: dep[n] = '' else: - dep[n] = sudo('{} show {} 2>&1 | grep Version:'.format(pip_version, i)).replace('Version: ', '{} v.'.format(i)) + sudo('{0} show {1} 2>&1 | if ! grep Version: /tmp/tee.tmp > ' + '/tmp/{0}_install_{1}.log; then echo "" > /tmp/{0}_install_{1}.log;fi'.format(pip_version, i)) + dep[n] = sudo('cat /tmp/{0}_install_{1}.log'.format(pip_version, i)).replace('Version: ', '{} v.'.format(i)) dep = [i for i in dep if i] status.append({"group": lib_group, "name": pip_pkg.split("==")[0], "version": version, "status": status_msg, "error_message": err, "available_versions": versions, "add_pkgs": dep}) @@ -439,7 +441,10 @@ def install_r_pkg(requisites): else: dep = dep.split(' ') for n, i in enumerate(dep): - dep[n] = '{} v.{}'.format(dep[n], dep_ver[n]) + if i == name: + dep[n] = '' + else: + dep[n] = '{} v.{}'.format(dep[n], dep_ver[n]) dep = [i for i in dep if i] err = sudo('cat /tmp/install_{0}.log'.format(name)).replace('"', "'") sudo('R -e \'installed.packages()[,c(3:4)]\' | if ! grep -w {0} > /tmp/install_{0}.list; then echo "" > /tmp/install_{0}.list;fi'.format(name)) @@ -454,7 +459,8 @@ def install_r_pkg(requisites): sudo('R -e \'install.packages("versions", repos="https://cloud.r-project.org";, dep=TRUE)\'') versions = sudo('R -e \'library(versions); available.versions("' + name + '")\' 2>&1 | grep -A 50 ' '\'date available\' | awk \'{print $2}\'').replace('\r\n', ' ')[5:].split(' ') - status_msg = 'invalid_version' + if versions != '': + status_msg = 'invalid_version' else: versions = [] status.append({"group": "r_pkg", "name": name, "version": version, "status": status_msg, "error_message": err, "available_versions": versions, "add_pkgs": dep}) diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py index ce075c0..cf7d8d1 100644 --- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py @@ -346,19 +346,54 @@ def install_nodejs(os_user): def install_os_pkg(requisites): status = list() error_parser = "Could not|No matching|Error:|failed|Requires:|Errno" + new_pkgs_parser = "Dependency Installed:" try: print("Updating repositories and installing requested tools: {}".format(requisites)) manage_pkg('update-minimal --security -y --skip-broken', 'remote', '') sudo('export LC_ALL=C') for os_pkg in requisites: - manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | if ! grep -w -E "({1})" > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(os_pkg, error_parser)) + name, vers = os_pkg + if vers != '' and vers !='N/A': + version = vers + os_pkg = "{}-{}".format(name, vers) + else: + version = 'N/A' + os_pkg = name + manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(os_pkg, error_parser)) + install_output = sudo('cat /tmp/tee.tmp') err = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)).replace('"', "'") - try: - res = sudo('python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\'][0]"'.format(os_pkg)) + sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})" /tmp/tee.tmp > ' + '/tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(os_pkg, new_pkgs_parser)) + dep = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)) + if dep == '': + dep = [] + else: + dep = dep[len(new_pkgs_parser): dep.find("Complete!") - 1].replace(' ', '').strip().split('\r\n') + for n, i in enumerate(dep): + i = i.split('.')[0] + sudo('yum info {0} 2>&1 | if ! grep Version > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(i)) + dep[n] =sudo('cat /tmp/os_install_{}.log'.format(i)).replace('Version : ', '{} v.'.format(i)) + dep = [i for i in dep if i] + versions = [] + res = sudo( + 'python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\'][0]"'.format( + name)) + if err: + status_msg = 'installation_error' + elif res: version = res.split('\r\n')[1].replace("'", "\"") - status.append({"group": "os_pkg", "name": os_pkg, "version": version, "status": "installed"}) - except: - status.append({"group": "os_pkg", "name": os_pkg, "status": "failed", "error_message": err}) + status_msg = "installed" + if 'No package {} available'.format(os_pkg) in install_output: + versions = sudo ('yum --showduplicates list ' + name + ' | expand | grep -A 10 "Available Packages" | grep -v "Available Packages"| awk \'{print $2}\'').replace('\r\n', '').split(' ') + if versions != '': + status_msg = 'invalid_version' + for n, i in enumerate(versions): + if ':' in i: + versions[n] = i.split(':')[1].split('-')[0] + else: + versions[n] = i.split('-')[0] + status.append({"group": "os_pkg", "name": name, "version": version, "status": status_msg, + "error_message": err, "add_pkgs": dep, "available_versions": versions}) return status except Exception as err: for os_pkg in requisites: diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py index ca81a97..075175c 100644 --- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py +++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py @@ -115,6 +115,9 @@ if __name__ == "__main__": or os.environ['application'] in ('rstudio', 'tensor-rstudio'): try: print('Installing R packages: {}'.format(pkgs['libraries']['r_pkg'])) + if os.environ['conf_resource'] in ('dataengine-service'): + manage_pkg('-y install', 'remote', 'libcurl libcurl-devel') + sudo('R -e "install.packages(\'devtools\', repos = \'https://cloud.r-project.org\')"') status = install_r_pkg(pkgs['libraries']['r_pkg']) general_status = general_status + status except KeyError: --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
