This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1409
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 3f32a0fda357f3871af73a0955ce2715e978b697
Author: Mykola_Bodnar1 <bodnarmyk...@gmail.com>
AuthorDate: Mon Jan 13 15:39:32 2020 +0200
[DLAB-1409] - Scala version is actual on Jupyter UI for DES/Jupyter
---
.../scripts/aws/jupyter_dataengine-service_create_configs.py | 2 +-
.../src/general/scripts/gcp/dataengine-service_jars_parser.py | 6 ++++++
.../scripts/gcp/jupyter_dataengine-service_create_configs.py | 4 ++--
.../scripts/gcp/jupyter_install_dataengine-service_kernels.py | 8 ++++++--
.../src/general/scripts/os/jupyter_dataengine_create_configs.py | 2 +-
5 files changed, 16 insertions(+), 6 deletions(-)
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
index 9af6935..7103301 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
@@ -85,7 +85,7 @@ def r_kernel(args):
def toree_kernel(args):
spark_path = '/opt/' + args.emr_version + '/' + args.cluster_name +
'/spark/'
- scala_version = local('scala -e
"println(scala.util.Properties.versionNumberString)"', capture=True)
+ scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala
version \K.{0,7}"', capture=True)
if args.emr_version == 'emr-4.3.0' or args.emr_version == 'emr-4.6.0' or
args.emr_version == 'emr-4.8.0':
local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/')
kernel_path = kernels_dir + "toree_" + args.cluster_name +
"/kernel.json"
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
index 4fd5a4a..c240f6e 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
@@ -56,6 +56,11 @@ if __name__ == "__main__":
spark_ver = subprocess.check_output("dpkg -l | grep spark-core | tr -s ' '
'-' | cut -f 4 -d '-'", shell=True).decode('UTF-8')
with open('/tmp/spark_version', 'w') as outfile:
outfile.write(spark_ver)
+ os.system('touch /tmp/scala_version')
+ scala_ver = subprocess.check_output("spark-submit --version 2>&1 | grep -o
-P 'Scala version \K.{0,7}'",
+ shell=True).decode('UTF-8')
+ with open('/tmp/scala_version', 'w') as outfile:
+ outfile.write(scala_ver)
os.system('touch /tmp/hadoop_version')
hadoop_ver = subprocess.check_output("dpkg -l | grep hadoop | head -n 1 |
tr -s ' ' '-' | cut -f 3 -d '-'", shell=True).decode('UTF-8')
with open('/tmp/hadoop_version', 'w') as outfile:
@@ -79,6 +84,7 @@ if __name__ == "__main__":
os.system('gsutil -m cp {0} gs://{1}/{2}/{3}/'.format(spark_def_path,
args.bucket, args.user_name, args.cluster_name))
os.system('gsutil -m cp /tmp/python_version
gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name))
os.system('gsutil -m cp /tmp/spark_version
gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name))
+ os.system('gsutil -m cp /tmp/scala_version
gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name))
os.system('gsutil -m cp /tmp/r_version
gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name))
os.system('gsutil -m cp /tmp/hadoop_version
gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name))
os.system('gsutil -m cp /tmp/spark.tar.gz
gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name))
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
index 959f9b5..9bc8e37 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
@@ -50,6 +50,7 @@ parser.add_argument('--pip_mirror', type=str, default='')
parser.add_argument('--application', type=str, default='')
parser.add_argument('--r_version', type=str, default='')
parser.add_argument('--r_enabled', type=str, default='')
+parser.add_argument('--scala_version', type=str, default='')
args = parser.parse_args()
dataproc_dir = '/opt/{}/jars/'.format(args.dataproc_version)
@@ -79,7 +80,6 @@ def r_kernel(args):
def toree_kernel(args):
spark_path = '/opt/{0}/{1}/spark/'.format(args.dataproc_version,
args.cluster_name)
- scala_version = local('scala -e
"println(scala.util.Properties.versionNumberString)"', capture=True)
local('mkdir -p {0}toree_{1}/'.format(kernels_dir, args.cluster_name))
local('tar zxvf /tmp/toree_kernel.tar.gz -C
{0}toree_{1}/'.format(kernels_dir, args.cluster_name))
local('sudo mv {0}toree_{1}/toree-0.2.0-incubating/*
{0}toree_{1}/'.format(kernels_dir, args.cluster_name))
@@ -93,7 +93,7 @@ def toree_kernel(args):
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('OS_USER', args.os_user)
text = text.replace('DATAENGINE-SERVICE_VERSION', args.dataproc_version)
- text = text.replace('SCALA_VERSION', scala_version)
+ text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/kernel_var.json')
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index 710974c..cb17668 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -70,6 +70,10 @@ def configure_notebook(args):
if __name__ == "__main__":
+ GCPActions().get_from_bucket(args.bucket,
'{0}/{1}/scala_version'.format(args.project_name, args.cluster_name),
+ '/tmp/scala_version')
+ with file('/tmp/scala_version') as f:
+ scala_version = str(f.read()).replace(',', '')
env.hosts = "{}".format(args.notebook_ip)
env.user = args.os_user
env.key_filename = "{}".format(args.keyfile)
@@ -84,7 +88,7 @@ if __name__ == "__main__":
r_enabled = os.environ['notebook_r_enabled']
sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile |
grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep
proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo
"https_proxy=$proxy" >> ~/.wgetrc')
- sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export
conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket
{2} --cluster_name {3} --dataproc_version {4} --spark_version {5}
--hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror
{10} --application {11} --r_version {12} --r_enabled {13}'
+ sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export
conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket
{2} --cluster_name {3} --dataproc_version {4} --spark_version {5}
--hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror
{10} --application {11} --r_version {12} --r_enabled {13} --scala_version {14}'
.format(os.environ['gcp_project_id'], os.environ['conf_resource'],
args.bucket, args.cluster_name,
args.dataproc_version, spark_version, hadoop_version,
args.region, args.project_name, args.os_user,
- args.pip_mirror, args.application, r_version, r_enabled))
+ args.pip_mirror, args.application, r_version, r_enabled,
scala_version))
diff --git
a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
index 5b4a51c..60a3246 100644
---
a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
+++
b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
@@ -79,7 +79,7 @@ def r_kernel(args):
def toree_kernel(args):
spark_path = '/opt/' + args.cluster_name + '/spark/'
- scala_version = local('scala -e
"println(scala.util.Properties.versionNumberString)"', capture=True)
+ scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala
version \K.{0,7}"', capture=True)
local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/')
local('tar zxvf /tmp/{}/toree_kernel.tar.gz -C '.format(args.cluster_name)
+ kernels_dir + 'toree_' + args.cluster_name + '/')
local('sudo mv {0}toree_{1}/toree-0.2.0-incubating/*
{0}toree_{1}/'.format(kernels_dir, args.cluster_name))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org
For additional commands, e-mail: commits-h...@dlab.apache.org
