This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch epm-v2.2-rc1-versions
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 27cda4cd838cc3b80efdf0abd01212bebc29d81d
Author: Mykola_Bodnar1 <bodnarmyk...@gmail.com>
AuthorDate: Wed Dec 18 15:50:30 2019 +0200
[DLAB-1346] - EMR version updated
[DLAB-1348] - Spark version updated
[DLAB-1350] - Jupyter version updated
---
infrastructure-provisioning/src/general/conf/dlab.ini | 4 ++--
.../general/files/aws/dataengine-service_description.json | 4 ++--
infrastructure-provisioning/src/general/lib/os/fab.py | 4 +++-
.../general/scripts/aws/dataengine-service_jars_parser.py | 14 ++++++--------
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini
b/infrastructure-provisioning/src/general/conf/dlab.ini
index 7873b43..ede348d 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -216,11 +216,11 @@ docker_version = 17.09.0
### Size of the additional volume for notebook instance
disk_size = 30
### Version of Apache Spark to be installed on notebook
-spark_version = 2.3.2
+spark_version = 2.4.4
### Version of Apache Hadoop to be installed on notebook
hadoop_version = 2.7
### Version of Jupyter to be installed on notebook
-jupyter_version = 5.7.4
+jupyter_version = 6.0.2
### Version of TensorFlow to be installed on notebook
tensorflow_version = 1.8.0
### Version of Zeppelin to be installed on notebook
diff --git
a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
index 7caee2a..b4e5ba2 100644
---
a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
+++
b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
@@ -24,7 +24,7 @@
},
"templates":
[
- {"version":"emr-5.12.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.3"}, {"Name":"Spark", "Version": "2.2.1"}, {"Name":"Hive", "Version":
"2.3.2"}]},
- {"version":"emr-5.19.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.5"}, {"Name":"Spark", "Version": "2.3.2"}, {"Name":"Hive", "Version":
"2.3.3"}]}
+ {"version":"emr-5.19.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.5"}, {"Name":"Spark", "Version": "2.3.2"}, {"Name":"Hive", "Version":
"2.3.3"}]},
+ {"version":"emr-5.28.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.5"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version":
"2.3.6"}]}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py
b/infrastructure-provisioning/src/general/lib/os/fab.py
index aba4c6a..7a07130 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -146,7 +146,7 @@ def put_resource_status(resource, status, dlab_path,
os_user, hostname):
def configure_jupyter(os_user, jupyter_conf_file, templates_dir,
jupyter_version, exploratory_name):
if not exists('/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
- sudo('pip2 install notebook=={}
--no-cache-dir'.format(jupyter_version))
+ sudo('pip2 install notebook==5.7.8 --no-cache-dir')
sudo('pip2 install jupyter --no-cache-dir')
sudo('pip3.5 install notebook=={}
--no-cache-dir'.format(jupyter_version))
sudo('pip3.5 install jupyter --no-cache-dir')
@@ -636,6 +636,8 @@ def configure_data_engine_service_pip(hostname, os_user,
keyfile):
sudo('ln -s /usr/bin/pip-3.4 /usr/bin/pip3')
elif not exists('/usr/bin/pip3') and sudo("python3.5 -V 2>/dev/null | awk
'{print $2}'"):
sudo('ln -s /usr/bin/pip-3.5 /usr/bin/pip3')
+ elif not exists('/usr/bin/pip3') and sudo("python3.6 -V 2>/dev/null | awk
'{print $2}'"):
+ sudo('ln -s /usr/bin/pip-3.6 /usr/bin/pip3')
sudo('echo "export PATH=$PATH:/usr/local/bin" >> /etc/profile')
sudo('source /etc/profile')
run('source /etc/profile')
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
index f011e6a..34755b5 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
@@ -54,14 +54,12 @@ if __name__ == "__main__":
else:
endpoint = "https://s3-{}.amazonaws.com".format(args.region)
os.system('touch /tmp/python_version')
- python_ver = subprocess.check_output("python3.5 -V 2>/dev/null | awk
'{print $2}'", shell=True)
- if python_ver != '':
- with open('/tmp/python_version', 'w') as outfile:
- outfile.write(python_ver)
- else:
- python_ver = subprocess.check_output("python3.4 -V 2>/dev/null | awk
'{print $2}'", shell=True)
- with open('/tmp/python_version', 'w') as outfile:
- outfile.write(python_ver)
+ for v in range(4, 7):
+ python_ver_checker = "python3.{} -V 2>/dev/null".format(v) + " | awk
'{print $2}'"
+ python_ver = subprocess.check_output(python_ver_checker, shell=True)
+ if python_ver != '':
+ with open('/tmp/python_version', 'w') as outfile:
+ outfile.write(python_ver)
os.system('/bin/tar -zhcvf /tmp/jars.tar.gz '
'--no-recursion '
'--absolute-names '
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org
For additional commands, e-mail: commits-h...@dlab.apache.org
