This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-1409 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 3f32a0fda357f3871af73a0955ce2715e978b697 Author: Mykola_Bodnar1 <bodnarmyk...@gmail.com> AuthorDate: Mon Jan 13 15:39:32 2020 +0200 [DLAB-1409] - Scala version is actual on Jupyter UI for DES/Jupyter --- .../scripts/aws/jupyter_dataengine-service_create_configs.py | 2 +- .../src/general/scripts/gcp/dataengine-service_jars_parser.py | 6 ++++++ .../scripts/gcp/jupyter_dataengine-service_create_configs.py | 4 ++-- .../scripts/gcp/jupyter_install_dataengine-service_kernels.py | 8 ++++++-- .../src/general/scripts/os/jupyter_dataengine_create_configs.py | 2 +- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py index 9af6935..7103301 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py +++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py @@ -85,7 +85,7 @@ def r_kernel(args): def toree_kernel(args): spark_path = '/opt/' + args.emr_version + '/' + args.cluster_name + '/spark/' - scala_version = local('scala -e "println(scala.util.Properties.versionNumberString)"', capture=True) + scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture=True) if args.emr_version == 'emr-4.3.0' or args.emr_version == 'emr-4.6.0' or args.emr_version == 'emr-4.8.0': local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/') kernel_path = kernels_dir + "toree_" + args.cluster_name + "/kernel.json" diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py index 4fd5a4a..c240f6e 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py @@ -56,6 +56,11 @@ if __name__ == "__main__": spark_ver = subprocess.check_output("dpkg -l | grep spark-core | tr -s ' ' '-' | cut -f 4 -d '-'", shell=True).decode('UTF-8') with open('/tmp/spark_version', 'w') as outfile: outfile.write(spark_ver) + os.system('touch /tmp/scala_version') + scala_ver = subprocess.check_output("spark-submit --version 2>&1 | grep -o -P 'Scala version \K.{0,7}'", + shell=True).decode('UTF-8') + with open('/tmp/scala_version', 'w') as outfile: + outfile.write(scala_ver) os.system('touch /tmp/hadoop_version') hadoop_ver = subprocess.check_output("dpkg -l | grep hadoop | head -n 1 | tr -s ' ' '-' | cut -f 3 -d '-'", shell=True).decode('UTF-8') with open('/tmp/hadoop_version', 'w') as outfile: @@ -79,6 +84,7 @@ if __name__ == "__main__": os.system('gsutil -m cp {0} gs://{1}/{2}/{3}/'.format(spark_def_path, args.bucket, args.user_name, args.cluster_name)) os.system('gsutil -m cp /tmp/python_version gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name)) os.system('gsutil -m cp /tmp/spark_version gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name)) + os.system('gsutil -m cp /tmp/scala_version gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name)) os.system('gsutil -m cp /tmp/r_version gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name)) os.system('gsutil -m cp /tmp/hadoop_version gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name)) os.system('gsutil -m cp /tmp/spark.tar.gz gs://{0}/{1}/{2}/'.format(args.bucket, args.user_name, args.cluster_name)) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py index 959f9b5..9bc8e37 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py @@ -50,6 +50,7 @@ parser.add_argument('--pip_mirror', type=str, default='') parser.add_argument('--application', type=str, default='') parser.add_argument('--r_version', type=str, default='') parser.add_argument('--r_enabled', type=str, default='') +parser.add_argument('--scala_version', type=str, default='') args = parser.parse_args() dataproc_dir = '/opt/{}/jars/'.format(args.dataproc_version) @@ -79,7 +80,6 @@ def r_kernel(args): def toree_kernel(args): spark_path = '/opt/{0}/{1}/spark/'.format(args.dataproc_version, args.cluster_name) - scala_version = local('scala -e "println(scala.util.Properties.versionNumberString)"', capture=True) local('mkdir -p {0}toree_{1}/'.format(kernels_dir, args.cluster_name)) local('tar zxvf /tmp/toree_kernel.tar.gz -C {0}toree_{1}/'.format(kernels_dir, args.cluster_name)) local('sudo mv {0}toree_{1}/toree-0.2.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name)) @@ -93,7 +93,7 @@ def toree_kernel(args): text = text.replace('SPARK_PATH', spark_path) text = text.replace('OS_USER', args.os_user) text = text.replace('DATAENGINE-SERVICE_VERSION', args.dataproc_version) - text = text.replace('SCALA_VERSION', scala_version) + text = text.replace('SCALA_VERSION', args.scala_version) with open(kernel_path, 'w') as f: f.write(text) local('touch /tmp/kernel_var.json') diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py index 710974c..cb17668 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py @@ -70,6 +70,10 @@ def configure_notebook(args): if __name__ == "__main__": + GCPActions().get_from_bucket(args.bucket, '{0}/{1}/scala_version'.format(args.project_name, args.cluster_name), + '/tmp/scala_version') + with file('/tmp/scala_version') as f: + scala_version = str(f.read()).replace(',', '') env.hosts = "{}".format(args.notebook_ip) env.user = args.os_user env.key_filename = "{}".format(args.keyfile) @@ -84,7 +88,7 @@ if __name__ == "__main__": r_enabled = os.environ['notebook_r_enabled'] sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf') sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc') - sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11} --r_version {12} --r_enabled {13}' + sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11} --r_version {12} --r_enabled {13} --scala_version {14}' .format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name, args.dataproc_version, spark_version, hadoop_version, args.region, args.project_name, args.os_user, - args.pip_mirror, args.application, r_version, r_enabled)) + args.pip_mirror, args.application, r_version, r_enabled, scala_version)) diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py index 5b4a51c..60a3246 100644 --- a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py +++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py @@ -79,7 +79,7 @@ def r_kernel(args): def toree_kernel(args): spark_path = '/opt/' + args.cluster_name + '/spark/' - scala_version = local('scala -e "println(scala.util.Properties.versionNumberString)"', capture=True) + scala_version = local('spark-submit --version 2>&1 | grep -o -P "Scala version \K.{0,7}"', capture=True) local('mkdir -p ' + kernels_dir + 'toree_' + args.cluster_name + '/') local('tar zxvf /tmp/{}/toree_kernel.tar.gz -C '.format(args.cluster_name) + kernels_dir + 'toree_' + args.cluster_name + '/') local('sudo mv {0}toree_{1}/toree-0.2.0-incubating/* {0}toree_{1}/'.format(kernels_dir, args.cluster_name)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org