This is an automated email from the ASF dual-hosted git repository. mhladun pushed a commit to branch gcp-conda in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 248392e4a00bf5cb0dfbbc235b5bdb727323dc93 Author: Marian_Hladun <[email protected]> AuthorDate: Wed Mar 16 17:06:02 2022 +0200 Refactor jup conda --- .../templates/locations/jupyter-conda.conf} | 0 infrastructure-provisioning/src/general/conf/datalab.ini | 4 ++-- .../{jupyter-gpu-conda_Dockerfile => jupyter-conda_Dockerfile} | 4 ++-- ...u-conda_description.json => jupyter-conda_description.json} | 2 +- infrastructure-provisioning/src/general/lib/os/fab.py | 10 ++++++---- .../src/general/scripts/gcp/common_prepare_notebook.py | 2 +- ...pyter-gpu-conda_configure.py => jupyter-conda_configure.py} | 2 +- .../src/general/scripts/gcp/ssn_configure.py | 2 +- .../src/{jupyter-gpu-conda => jupyter-conda}/fabfile.py | 2 +- .../scripts/configure_jupyter-conda_node.py} | 0 .../templates/locations/jupyter-conda.conf} | 0 infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json | 2 +- infrastructure-provisioning/src/ssn/scripts/docker_build.py | 2 +- .../datalab/backendapi/core/commands/CommandExecutorMock.java | 2 +- .../com/epam/datalab/backendapi/domain/NotebookTemplate.java | 2 +- .../datalab/backendapi/service/impl/LibraryServiceImpl.java | 2 +- .../java/com/epam/datalab/backendapi/util/BillingUtils.java | 2 +- .../self-service/src/main/resources/mongo/gcp/mongo_roles.json | 2 +- .../create-environment/create-environment.component.ts | 6 +++--- .../app/resources/resources-grid/resources-grid.component.html | 2 +- 20 files changed, 26 insertions(+), 24 deletions(-) diff --git a/infrastructure-provisioning/src/project/templates/locations/jupyter-gpu-conda.conf b/infrastructure-provisioning/src/edge/templates/locations/jupyter-conda.conf similarity index 100% rename from infrastructure-provisioning/src/project/templates/locations/jupyter-gpu-conda.conf rename to infrastructure-provisioning/src/edge/templates/locations/jupyter-conda.conf diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini index 2fcd74f..23eb030 100644 --- a/infrastructure-provisioning/src/general/conf/datalab.ini +++ b/infrastructure-provisioning/src/general/conf/datalab.ini @@ -253,7 +253,7 @@ ssn_instance_size = n1-standard-2 edge_instance_size = n1-standard-1 ### GPU type for Tensor/DeepLaerning notebooks gpu_accelerator_type = nvidia-tesla-k80 -### GPU type for jupyter-gpu and jupyter-gpu-conda notebooks +### GPU type for jupyter-gpu and jupyter-conda notebooks jupyter_gpu_type = nvidia-tesla-a100 #--- [ssn] section contains all parameters that are using for self-service node provisioning ---# @@ -393,7 +393,7 @@ slave_instance_spot_pct_price = 70 ### Count of slave nodes for Data Engine # instance_count = ### Type of notebooks for creating Data Engine from notebook images -image_notebooks = jupyter,jupyter-gpu,jupyter-gpu-conda,jupyterlab,rstudio,zeppelin,tensor,tensor-rstudio,deeplearning +image_notebooks = jupyter,jupyter-gpu,jupyter-conda,jupyterlab,rstudio,zeppelin,tensor,tensor-rstudio,deeplearning ### Persent of RAM allocated for an operating system os_memory = 75 ### Explicit allocation RAM for an operating system diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter-conda_Dockerfile similarity index 96% rename from infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_Dockerfile rename to infrastructure-provisioning/src/general/files/gcp/jupyter-conda_Dockerfile index 170cda9..610996c 100644 --- a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_Dockerfile +++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-conda_Dockerfile @@ -24,9 +24,9 @@ FROM docker.datalab-base:latest ARG OS -COPY jupyter-gpu-conda/ /root/ +COPY jupyter-conda/ /root/ COPY general/scripts/os/* /root/scripts/ -COPY general/scripts/gcp/jupyter-gpu-conda_* /root/scripts/ +COPY general/scripts/gcp/jupyter-conda_* /root/scripts/ COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/ COPY general/templates/os/${OS}/ungit.service /root/templates/ diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_description.json b/infrastructure-provisioning/src/general/files/gcp/jupyter-conda_description.json similarity index 90% rename from infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_description.json rename to infrastructure-provisioning/src/general/files/gcp/jupyter-conda_description.json index 5528513..7e09b50 100644 --- a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu-conda_description.json +++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-conda_description.json @@ -11,7 +11,7 @@ "template_name": "Jupyter notebook 6.1.6 with GPU and Anaconda", "description": "Base image with jupyter node creation routines", "environment_type": "exploratory", - "version": "jupyter_notebook_gpu_conda", + "version": "jupyter_notebook_conda", "vendor": "GCP" } ] diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index ad4b6e7..dc1da09 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -385,11 +385,13 @@ def ensure_anaconda(): conn.sudo('wget https://repo.anaconda.com/archive/Anaconda3-2021.11-Linux-x86_64.sh -O /tmp/anaconda.sh') conn.sudo('bash /tmp/anaconda.sh -b -p /opt/anaconda3') conn.sudo('chown -R datalab-user /opt/anaconda3') - conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda create -y -p /opt/anaconda3/envs/jupyter-gpu-conda git pip ipykernel -c anaconda') - conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda activate jupyter-gpu-conda && /opt/anaconda3/envs/jupyter-gpu-conda/bin/pip install numpy scipy pandas scikit-learn transformers==4.4.2 gensim==4.0.1 tokenizers==0.10.1 python-levenshtein==0.12.2') - conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda activate jupyter-gpu-conda && /opt/anaconda3/envs/jupyter-gpu-conda/bin/pip install -U torch==1.10.0+cu111 torchvision==0.11.3+cu111 torchaudio==0.10.2+cu111 -f https://download.pytorch.org/whl/cu111/torch_stable.html --no-cache-dir') + #conn.sudo(''' bash -l -c "echo 'export PATH=/opt/anaconda3/bin/:\$PATH' >> /home/datalab-user/.bashrc" ''') + #conn.run('source /home/datalab-user/.bashrc') + conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda create -y -p /opt/anaconda3/envs/jupyter-conda git pip ipykernel -c anaconda') + conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda activate jupyter-conda && /opt/anaconda3/envs/jupyter-conda/bin/pip install numpy scipy pandas scikit-learn transformers==4.4.2 gensim==4.0.1 tokenizers==0.10.1 python-levenshtein==0.12.2') + conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda activate jupyter-conda && /opt/anaconda3/envs/jupyter-conda/bin/pip install -U torch==1.10.0+cu111 torchvision==0.11.3+cu111 torchaudio==0.10.2+cu111 -f https://download.pytorch.org/whl/cu111/torch_stable.html --no-cache-dir') conn.sudo('chown -R datalab-user /home/datalab-user/.local/share/jupyter/kernels') - conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda activate jupyter-gpu-conda && python -m ipykernel install --user --name=jupyter-gpu-conda') + conn.run('source /opt/anaconda3/etc/profile.d/conda.sh && conda activate jupyter-conda && python -m ipykernel install --user --name=jupyter-conda') conn.sudo('chown -R root /home/datalab-user/.local/share/jupyter/kernels') conn.sudo('systemctl restart jupyter-notebook') except Exception as err: diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py index c6ee2d3..7aeda0e 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py @@ -167,7 +167,7 @@ if __name__ == "__main__": else: notebook_config['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type'] - if os.environ['application'] in ('jupyter-gpu', 'jupyter-gpu-conda'): + if os.environ['application'] in ('jupyter-gpu', 'jupyter-conda'): notebook_config['gpu_accelerator_type'] = os.environ['gcp_jupyter_gpu_type'] notebook_config['gpu_accelerator_count'] = '1' diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu-conda_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-conda_configure.py similarity index 99% rename from infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu-conda_configure.py rename to infrastructure-provisioning/src/general/scripts/gcp/jupyter-conda_configure.py index fec7583..571107c 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu-conda_configure.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-conda_configure.py @@ -165,7 +165,7 @@ if __name__ == "__main__": os.environ['notebook_scala_version'], notebook_config['exploratory_name'], edge_instance_private_ip) try: - subprocess.run("~/scripts/{}.py {}".format('configure_jupyter-gpu-conda_node', params), shell=True, check=True) + subprocess.run("~/scripts/{}.py {}".format('configure_jupyter-conda_node', params), shell=True, check=True) except: traceback.print_exc() raise Exception diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py index a7fbe2a..93c79eb 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py @@ -203,7 +203,7 @@ if __name__ == "__main__": {"name": "edge", "tag": "latest"}, {"name": "jupyter", "tag": "latest"}, {"name": "jupyter-gpu", "tag": "latest"}, - {"name": "jupyter-gpu-conda", "tag": "latest"}, + {"name": "jupyter-conda", "tag": "latest"}, {"name": "jupyterlab", "tag": "latest"}, {"name": "rstudio", "tag": "latest"}, {"name": "zeppelin", "tag": "latest"}, diff --git a/infrastructure-provisioning/src/jupyter-gpu-conda/fabfile.py b/infrastructure-provisioning/src/jupyter-conda/fabfile.py similarity index 98% rename from infrastructure-provisioning/src/jupyter-gpu-conda/fabfile.py rename to infrastructure-provisioning/src/jupyter-conda/fabfile.py index cab62b9..30a4e69 100644 --- a/infrastructure-provisioning/src/jupyter-gpu-conda/fabfile.py +++ b/infrastructure-provisioning/src/jupyter-conda/fabfile.py @@ -54,7 +54,7 @@ def run(ctx): try: params = "--uuid {}".format(notebook_config['uuid']) - subprocess.run("~/scripts/{}.py {}".format('jupyter-gpu-conda_configure', params), shell=True, check=True) + subprocess.run("~/scripts/{}.py {}".format('jupyter-conda_configure', params), shell=True, check=True) except Exception as err: traceback.print_exc() append_result("Failed configuring Notebook node.", str(err)) diff --git a/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py b/infrastructure-provisioning/src/jupyter-conda/scripts/configure_jupyter-conda_node.py similarity index 100% rename from infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py rename to infrastructure-provisioning/src/jupyter-conda/scripts/configure_jupyter-conda_node.py diff --git a/infrastructure-provisioning/src/edge/templates/locations/jupyter-gpu-conda.conf b/infrastructure-provisioning/src/project/templates/locations/jupyter-conda.conf similarity index 100% rename from infrastructure-provisioning/src/edge/templates/locations/jupyter-gpu-conda.conf rename to infrastructure-provisioning/src/project/templates/locations/jupyter-conda.conf diff --git a/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json b/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json index 8ed47e3..214a526 100644 --- a/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json +++ b/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json @@ -113,7 +113,7 @@ "_id": "nbCreateJupyterGpuConda", "description": "Create Notebook Jupyter Gpu with Anaconda", "exploratories": [ - "docker.datalab-jupyter-gpu-conda" + "docker.datalab-jupyter-conda" ], "groups": [ "$anyuser" diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py index 17ad490..4a1670e 100644 --- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py +++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py @@ -36,7 +36,7 @@ if sys.argv[1] == 'all': 'project', 'jupyter', 'jupyter-gpu', - 'jupyter-gpu-conda', + 'jupyter-conda', 'jupyterlab', 'rstudio', 'zeppelin', diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java index 69c8719..5395b30 100644 --- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java +++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java @@ -79,7 +79,7 @@ public class CommandExecutorMock implements ICommandExecutor { "docker.datalab-deeplearning:latest", "docker.datalab-jupyter:latest", "docker.datalab-jupyter-gpu:latest", - "docker.datalab-jupyter-gpu-conda:latest", + "docker.datalab-jupyter-conda:latest", "docker.datalab-jupyterlab:latest", "docker.datalab-superset:latest", "docker.datalab-rstudio:latest", diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java index 0616626..317eb41 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java @@ -27,7 +27,7 @@ import lombok.Getter; public enum NotebookTemplate { JUPYTER("Jupyter notebook 6.1.6"), JUPYTER_GPU("Jupyter notebook 6.1.6 with GPU"), - JUPYTER_GPU_CONDA("Jupyter notebook 6.1.6 with GPU and Anaconda"), + JUPYTER_CONDA("Jupyter notebook 6.1.6 with GPU and Anaconda"), JUPYTER_LAB("JupyterLab 0.35.6"), ZEPPELIN("Apache Zeppelin 0.9.1"), DEEP_LEARNING("Deep Learning 2.4"), diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java index 9d035c9..c9c9024 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java @@ -160,7 +160,7 @@ public class LibraryServiceImpl implements LibraryService { final String templateName = userInstanceDTO.getTemplateName(); List<LibraryGroups> groups = new ArrayList<>(Arrays.asList(GROUP_PIP3, GROUP_OTHERS, GROUP_OS_PKG)); - if (isTemplateGroup(templateName, Stream.of(JUPYTER, ZEPPELIN, JUPYTER_GPU, JUPYTER_GPU_CONDA))) { + if (isTemplateGroup(templateName, Stream.of(JUPYTER, ZEPPELIN, JUPYTER_GPU, JUPYTER_CONDA))) { groups.addAll(Arrays.asList(GROUP_R_PKG, GROUP_JAVA)); } if (isTemplateGroup(templateName, Stream.of(DEEP_LEARNING, TENSOR, TENSOR_GCP, diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java index 3d4fc63..5136bdc 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java @@ -39,7 +39,7 @@ import static com.epam.datalab.dto.billing.BillingResourceType.*; @Slf4j public class BillingUtils { - private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "jupyter-gpu", "jupyter-gpu-conda", "deeplearning"}; + private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "jupyter-gpu", "jupyter-conda", "deeplearning"}; private static final String[] BILLING_FILTERED_REPORT_HEADERS = {"DataLab ID", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"}; private static final String[] COMPLETE_REPORT_REPORT_HEADERS = {"DataLab ID", "User", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"}; diff --git a/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json b/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json index 3fa018f..50e6578 100644 --- a/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json +++ b/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json @@ -137,7 +137,7 @@ "type": "NOTEBOOK", "cloud": "GCP", "exploratories": [ - "docker.datalab-jupyter-gpu-conda" + "docker.datalab-jupyter-conda" ], "groups": [ "$anyuser" diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/create-environment/create-environment.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/create-environment/create-environment.component.ts index 93bdffb..0654434 100644 --- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/create-environment/create-environment.component.ts +++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/create-environment/create-environment.component.ts @@ -177,13 +177,13 @@ export class ExploratoryEnvironmentCreateComponent implements OnInit { if (this.selectedCloud === 'gcp' && (template?.image === 'docker.datalab-jupyter' || /** template?.image === 'docker.datalab-jupyter-gpu' || */ -/** template?.image === 'docker.datalab-jupyter-gpu-conda' || */ +/** template?.image === 'docker.datalab-jupyter-conda' || */ template?.image === 'docker.datalab-deeplearning' || template?.image === 'docker.datalab-tensor')) { this.gpuTypes = template?.computationGPU ? HelpUtils.sortGpuTypes(template.computationGPU) : []; - if(template?.image === 'docker.datalab-tensor' /**|| template?.image === 'docker.datalab-jupyter-gpu-conda'|| template?.image === 'docker.datalab-jupyter-gpu' */|| template?.image === 'docker.datalab-deeplearning') { + if(template?.image === 'docker.datalab-tensor' /**|| template?.image === 'docker.datalab-jupyter-conda'|| template?.image === 'docker.datalab-jupyter-gpu' */|| template?.image === 'docker.datalab-deeplearning') { this.addGpuFields(); } } @@ -194,7 +194,7 @@ export class ExploratoryEnvironmentCreateComponent implements OnInit { if (template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('tensorflow') === -1 && template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('deeplearning') === -1 /** && template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('jupyter-gpu') === -1 - && template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('jupyter-gpu-conda') === -1 */ + && template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('jupyter-conda') === -1 */ && template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('deep learning') === -1 && template.exploratory_environment_versions[0].template_name.toLowerCase().indexOf('data science') === -1 ) { diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html index 676dc50..73fc403 100644 --- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html +++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html @@ -265,7 +265,7 @@ *ngIf="element.status === 'running' && element.image !== 'docker.datalab-superset' && element.image !== 'docker.datalab-jupyterlab' - && element.image !== 'docker.datalab-jupyter-gpu-conda' + && element.image !== 'docker.datalab-jupyter-conda' && element.image !== 'docker.datalab-jupyter-gpu'" matTooltip="Only one compute can be associated with analytical tool at a time" matTooltipPosition="above" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
