This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2587 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f8e71e25eb282ecb469bcee2a6fdfea73788f959 Author: leonidfrolov <[email protected]> AuthorDate: Thu Nov 18 18:10:25 2021 +0200 [DATALAB-2587]: fixed remote pyspark kernels for zeppelin --- .../os/zeppelin_dataengine_create_configs.py | 42 +++++++++++----------- .../templates/os/dataengine_interpreter_spark.json | 15 +++++--- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py index 7f424eb..4f31d74 100644 --- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py +++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py @@ -123,27 +123,27 @@ def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user subprocess.run('sudo systemctl start livy-server-' + str(livy_port), shell=True, check=True) else: template_file = "/tmp/{}/dataengine_interpreter.json".format(args.cluster_name) - p_versions = ["2", "3.8"] - for p_version in p_versions: - fr = open(template_file, 'r+') - text = fr.read() - text = text.replace('CLUSTERNAME', cluster_name) - text = text.replace('PYTHONVERSION', p_version) - text = text.replace('SPARK_HOME', cluster_dir + 'spark/') - text = text.replace('PYTHONVER_SHORT', p_version[:1]) - text = text.replace('MASTER', str(spark_master)) - tmp_file = "/tmp/dataengine_spark_py" + p_version + "_interpreter.json" - fw = open(tmp_file, 'w') - fw.write(text) - fw.close() - for _ in range(5): - try: - subprocess.run("curl --noproxy localhost -H 'Content-Type: application/json' -X POST -d " + - "@/tmp/dataengine_spark_py" + p_version + - "_interpreter.json http://localhost:8080/api/interpreter/setting";, shell=True, check=True) - break - except: - subprocess.run('sleep 5', shell=True, check=True) + p_version = os.environ['notebook_python_venv_version'] + fr = open(template_file, 'r+') + text = fr.read() + text = text.replace('CLUSTERNAME', cluster_name) + text = text.replace('PYTHONVERSION', p_version[:3]) + text = text.replace('PYTHONVER_FULL', p_version) + text = text.replace('SPARK_HOME', cluster_dir + 'spark/') + text = text.replace('PYTHONVER_SHORT', p_version[:1]) + text = text.replace('MASTER', str(spark_master)) + tmp_file = "/tmp/dataengine_spark_py" + p_version + "_interpreter.json" + fw = open(tmp_file, 'w') + fw.write(text) + fw.close() + for _ in range(5): + try: + subprocess.run("curl --noproxy localhost -H 'Content-Type: application/json' -X POST -d " + + "@/tmp/dataengine_spark_py" + p_version + + "_interpreter.json http://localhost:8080/api/interpreter/setting";, shell=True, check=True) + break + except: + subprocess.run('sleep 5', shell=True, check=True) subprocess.run('touch /home/' + os_user + '/.ensure_dir/dataengine_' + cluster_name + '_interpreter_ensured', shell=True, check=True) except Exception as err: print('Error: {0}'.format(err)) diff --git a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json index f3bcebb..ded07c1 100644 --- a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json +++ b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json @@ -1,5 +1,5 @@ { - "name": "CLUSTERNAME_pyPYTHONVER_SHORT", + "name": "CLUSTERNAME", "group": "spark", "properties": { "zeppelin.spark.useHiveContext": { @@ -58,10 +58,17 @@ "description": "", "type": "checkbox" }, - "zeppelin.pyspark.python": { + "PYSPARK_PYTHON": { "envName": "ZEPPELIN_PYSPARK_PYTHON", - "propertyName": "zeppelin.pyspark.python", - "value": "pythonPYTHONVERSION", + "propertyName": "PYSPARK_PYTHON", + "value": "/opt/python/pythonPYTHONVER_FULL/bin/pythonPYTHONVERSION", + "description": "", + "type": "string" + }, + "PYSPARK_DRIVER_PYTHON": { + "envName": "ZEPPELIN_PYSPARK_DRIVER_PYTHON", + "propertyName": "PYSPARK_DRIVER_PYTHON", + "value": "/opt/python/pythonPYTHONVER_FULL/bin/pythonPYTHONVERSION", "description": "", "type": "string" }, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
