This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/develop by this push:
new 1a4315c Jupyter virtual environment fixes
new 9fcef89 Merge pull request #1292 from owlleg6/develop
1a4315c is described below
commit 1a4315cc3ce0174c040a50e651cabfd24ef1777a
Author: oleh_mykolaishyn <[email protected]>
AuthorDate: Wed Oct 6 13:54:24 2021 +0300
Jupyter virtual environment fixes
---
infrastructure-provisioning/src/general/lib/os/fab.py | 5 ++---
.../src/general/templates/os/py3spark_local_template.json | 2 +-
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py
b/infrastructure-provisioning/src/general/lib/os/fab.py
index 435efd8..3fa3cea 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -442,7 +442,7 @@ def ensure_pyspark_local_kernel(os_user,
pyspark_local_path_dir, templates_dir,
conn.sudo('mkdir -p ' + pyspark_local_path_dir)
conn.sudo('touch ' + pyspark_local_path_dir + 'kernel.json')
conn.put(templates_dir + 'pyspark_local_template.json',
'/tmp/pyspark_local_template.json')
- conn.sudo('''bash -l -c "PYJ=`find /opt/spark/ -name '*py4j*.zip'
| tr '\\n' ':' | sed 's|:$||g'`; sed -i 's|PY4J|'$PYJ'|g'
/tmp/pyspark_local_template.json" ''')
+ conn.sudo('''bash -l -c "sed -i \"s:PY4J:$(find /opt/spark/ -name
'*py4j*.zip'):g\" /tmp/pyspark_local_template.json" ''')
conn.sudo('sed -i "s|SP_VER|' + spark_version + '|g"
/tmp/pyspark_local_template.json')
conn.sudo('sed -i
\'/PYTHONPATH\"\:/s|\(.*\)"|\\1/home/{0}/caffe/python:/home/{0}/pytorch/build:"|\'
/tmp/pyspark_local_template.json'.format(os_user))
conn.sudo('\cp /tmp/pyspark_local_template.json ' +
pyspark_local_path_dir + 'kernel.json')
@@ -457,8 +457,7 @@ def ensure_py3spark_local_kernel(os_user,
py3spark_local_path_dir, templates_dir
conn.sudo('mkdir -p ' + py3spark_local_path_dir)
conn.sudo('touch ' + py3spark_local_path_dir + 'kernel.json')
conn.put(templates_dir + 'py3spark_local_template.json',
'/tmp/py3spark_local_template.json')
- conn.sudo(
- '''bash -l -c "PYJ=`find /opt/spark/ -name '*py4j*.zip' | tr
'\\n' ':' | sed 's|:$||g'`; sed -i 's|PY4J|'$PYJ'|g'
/tmp/py3spark_local_template.json" ''')
+ conn.sudo('''bash -l -c "sed -i \"s:PY4J:$(find /opt/spark/ -name
'*py4j*.zip'):g\" /tmp/py3spark_local_template.json" ''')
conn.sudo('sed -i "s|PYTHON_VENV_PATH|' + python_venv_path + '|g"
/tmp/py3spark_local_template.json')
conn.sudo('sed -i "s|PYTHON_VENV_VERSION|' + python_venv_version +
'|g" /tmp/py3spark_local_template.json')
conn.sudo('sed -i "s|PYTHON_VENV_SHORT_VERSION|' +
python_venv_version[:3] + '|g" /tmp/py3spark_local_template.json')
diff --git
a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
index 428524d..1771efe 100644
---
a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
+++
b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
@@ -11,7 +11,7 @@
"env": {
"PYSPARK_PYTHON": "pythonPYTHON_VENV_SHORT_VERSION",
"SPARK_HOME": "/opt/spark/",
- "PYTHONPATH": "PY4J:/opt/spark/python/:",
+ "PYTHONPATH": "PY4J:/opt/spark/python/:PYTHON_VENV_PATH:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
"PYSPARK_SUBMIT_ARGS": "--name LocalPySpark pyspark-shell"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
