This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2691
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2691 by this push:
new 6bd677f [DATALAB-2691]: fixed matplotlib version for zeppelin emr
configs
6bd677f is described below
commit 6bd677f2ef3c6ac674d1bc520eaa3948ce4ca31c
Author: leonidfrolov <[email protected]>
AuthorDate: Wed Mar 16 18:30:54 2022 +0200
[DATALAB-2691]: fixed matplotlib version for zeppelin emr configs
---
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 9 +++++----
.../scripts/aws/zeppelin_dataengine-service_create_configs.py | 4 +++-
.../scripts/aws/zeppelin_install_dataengine-service_kernels.py | 7 +++++--
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index dfddc10..a6ec29e 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1582,7 +1582,8 @@ def create_aws_config_files(generate_full_config=False):
sys.exit(1)
-def installing_python(region, bucket, user_name, cluster_name, application='',
pip_mirror='', numpy_version='1.14.3'):
+def installing_python(region, bucket, user_name, cluster_name, application='',
numpy_version='1.14.3',
+ matplotlib_version='3.3.4', pip_mirror=''):
get_cluster_python_version(region, bucket, user_name, cluster_name)
with open('/tmp/python_version') as f:
python_version = f.read()
@@ -1617,7 +1618,7 @@ def installing_python(region, bucket, user_name,
cluster_name, application='', p
subprocess.run(venv_command + ' && sudo -i ' + pip_command + '
install NumPy=={0}'.format(numpy_version), shell=True, check=True)
subprocess.run(venv_command + ' && sudo -i ' + pip_command +
' install -i https://{0}/simple --trusted-host {0}
--timeout 60000 boto boto3 SciPy '
- 'Matplotlib=={1} pandas Sympy Pillow sklearn
--no-cache-dir'.format(pip_mirror, os.environ['notebook_matplotlib_version']),
shell=True, check=True)
+ 'Matplotlib=={1} pandas Sympy Pillow sklearn
--no-cache-dir'.format(pip_mirror, matplotlib_version), shell=True, check=True)
# Need to refactor when we add GPU cluster
if application == 'deeplearning':
subprocess.run(venv_command + ' && sudo -i ' + pip_command
+
@@ -1636,8 +1637,8 @@ def installing_python(region, bucket, user_name,
cluster_name, application='', p
sys.exit(1)
else:
for lib in ['-U pip==9.0.3', 'pyzmq==17.0.0',
- 'ipython ipykernel boto boto3 pybind11 pythran cython
NumPy=={} Matplotlib=={} --no-cache-dir'
- .format(numpy_version,
os.environ['notebook_matplotlib_version']),
+ 'ipython ipykernel boto boto3 pybind11 pythran cython
NumPy=={0} Matplotlib=={1} '
+ '--no-cache-dir'.format(numpy_version,
matplotlib_version),
'SciPy pandas Sympy Pillow', 'sklearn --no-cache-dir']:
subprocess.run('bash -c "{0} && sudo -i {1} install {2}"'
.format(venv_command, pip_command, lib),
shell=True, check=True)
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
index c86fc54..4ad2b36 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
@@ -37,6 +37,7 @@ parser.add_argument('--emr_version', type=str, default='')
parser.add_argument('--spark_version', type=str, default='')
parser.add_argument('--scala_version', type=str, default='')
parser.add_argument('--hadoop_version', type=str, default='')
+parser.add_argument('--matplotlib_version', type=str, default='')
parser.add_argument('--region', type=str, default='')
parser.add_argument('--excluded_lines', type=str, default='')
parser.add_argument('--project_name', type=str, default='')
@@ -92,7 +93,8 @@ if __name__ == "__main__":
configuring_notebook(args.emr_version)
if args.multiple_clusters == 'true':
install_remote_livy(args)
- installing_python(args.region, args.bucket, args.project_name,
args.cluster_name, args.application, args.numpy_version)
+ installing_python(args.region, args.bucket, args.project_name,
args.cluster_name, args.application,
+ args.numpy_version, args.matplotlib_version)
configure_zeppelin_emr_interpreter(args.emr_version,
args.cluster_name, args.region, spark_dir, args.os_user,
yarn_dir, args.bucket,
args.project_name, endpoint_url, args.multiple_clusters)
update_zeppelin_interpreters(args.multiple_clusters, args.r_enabled)
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
index 311cb04..df9c51a 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
@@ -75,6 +75,7 @@ if __name__ == "__main__":
livy_version = os.environ['notebook_livy_version']
r_enabled = os.environ['notebook_r_enabled']
numpy_version = os.environ['notebook_numpy_version']
+ matplotlib_version = os.environ['notebook_matplotlib_version']
command = "/usr/bin/python3
/usr/local/bin/zeppelin_dataengine-service_create_configs.py " \
"--bucket {0} " \
"--cluster_name {1} " \
@@ -91,8 +92,9 @@ if __name__ == "__main__":
"--livy_version {12} " \
"--multiple_clusters {13} " \
"--numpy_version {14} " \
- "--application {15} " \
- "--r_enabled {16}" \
+ "--matplotlib_version {15} " \
+ "--application {16} " \
+ "--r_enabled {17}" \
.format(args.bucket,
args.cluster_name,
args.emr_version,
@@ -108,6 +110,7 @@ if __name__ == "__main__":
livy_version,
os.environ['notebook_multiple_clusters'],
numpy_version,
+ matplotlib_version,
args.application,
r_enabled)
conn.sudo(command)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
