This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch DLAB-1571 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 8feb625f1b65477ddd6c474c0b6183f37504194d Merge: cb63850 3f5451e Author: Oleh Fuks <[email protected]> AuthorDate: Wed Mar 25 11:27:51 2020 +0200 Merge branch 'develop' into DLAB-1571 # Conflicts: # services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java # services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java # services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java # services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.html # services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.scss # services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts .../scripts/deploy_repository/deploy_repository.py | 2 +- .../scripts/configure_deep_learning_node.py | 4 +- infrastructure-provisioning/src/edge/fabfile.py | 76 -- .../src/general/lib/aws/actions_lib.py | 60 +- .../src/general/lib/azure/actions_lib.py | 45 +- .../src/general/lib/gcp/actions_lib.py | 14 +- .../src/general/lib/gcp/meta_lib.py | 19 +- .../src/general/lib/os/debian/edge_lib.py | 27 +- .../src/general/lib/os/fab.py | 15 +- .../scripts/aws/common_create_notebook_image.py | 48 +- .../general/scripts/aws/common_create_policy.py | 14 +- .../general/scripts/aws/common_create_subnet.py | 2 +- .../scripts/aws/common_download_git_certfile.py | 2 +- ...common_notebook_configure_dataengine-service.py | 110 ++- .../aws/common_notebook_configure_dataengine.py | 73 +- .../general/scripts/aws/common_prepare_notebook.py | 173 ++-- .../general/scripts/aws/common_start_notebook.py | 39 +- .../general/scripts/aws/common_stop_notebook.py | 47 +- .../scripts/aws/common_terminate_notebook.py | 38 +- .../scripts/aws/common_terminate_notebook_image.py | 17 +- .../scripts/aws/dataengine-service_configure.py | 198 ++-- .../scripts/aws/dataengine-service_prepare.py | 257 +++--- .../scripts/aws/dataengine-service_terminate.py | 37 +- .../general/scripts/aws/dataengine_configure.py | 254 +++--- .../src/general/scripts/aws/dataengine_prepare.py | 156 ++-- .../src/general/scripts/aws/dataengine_start.py | 44 +- .../src/general/scripts/aws/dataengine_stop.py | 28 +- .../general/scripts/aws/dataengine_terminate.py | 39 +- .../general/scripts/aws/deeplearning_configure.py | 318 +++---- .../src/general/scripts/aws/edge_configure.py | 279 +++--- .../src/general/scripts/aws/edge_start.py | 31 +- .../src/general/scripts/aws/edge_status.py | 23 +- .../src/general/scripts/aws/edge_stop.py | 26 +- .../src/general/scripts/aws/jupyter_configure.py | 316 +++---- .../general/scripts/aws/jupyterlab_configure.py | 306 ++++--- .../src/general/scripts/aws/project_prepare.py | 432 ++++----- .../src/general/scripts/aws/project_terminate.py | 111 ++- .../src/general/scripts/aws/rstudio_configure.py | 314 +++---- .../src/general/scripts/aws/ssn_configure.py | 428 ++++----- .../src/general/scripts/aws/ssn_prepare.py | 713 ++++++++------- .../src/general/scripts/aws/ssn_terminate.py | 33 +- .../scripts/aws/ssn_terminate_aws_resources.py | 100 ++- .../scripts/aws/tensor-rstudio_configure.py | 338 +++---- .../src/general/scripts/aws/tensor_configure.py | 324 +++---- .../src/general/scripts/aws/zeppelin_configure.py | 326 +++---- .../scripts/azure/common_create_instance.py | 2 +- .../scripts/azure/common_create_notebook_image.py | 78 +- .../scripts/azure/common_download_git_certfile.py | 6 +- .../azure/common_notebook_configure_dataengine.py | 84 +- .../scripts/azure/common_prepare_notebook.py | 103 ++- .../general/scripts/azure/common_start_notebook.py | 53 +- .../general/scripts/azure/common_stop_notebook.py | 43 +- .../scripts/azure/common_terminate_notebook.py | 41 +- .../azure/common_terminate_notebook_image.py | 18 +- .../general/scripts/azure/dataengine_configure.py | 224 ++--- .../general/scripts/azure/dataengine_prepare.py | 103 +-- .../src/general/scripts/azure/dataengine_start.py | 52 +- .../src/general/scripts/azure/dataengine_stop.py | 44 +- .../general/scripts/azure/dataengine_terminate.py | 48 +- .../scripts/azure/deeplearning_configure.py | 173 ++-- .../src/general/scripts/azure/edge_configure.py | 357 +++----- .../src/general/scripts/azure/edge_prepare.py | 6 +- .../src/general/scripts/azure/edge_start.py | 36 +- .../src/general/scripts/azure/edge_status.py | 23 +- .../src/general/scripts/azure/edge_stop.py | 26 +- .../src/general/scripts/azure/edge_terminate.py | 86 +- .../src/general/scripts/azure/jupyter_configure.py | 204 ++--- .../general/scripts/azure/jupyterlab_configure.py | 185 ++-- .../src/general/scripts/azure/project_prepare.py | 364 ++++---- .../src/general/scripts/azure/project_terminate.py | 93 +- .../src/general/scripts/azure/rstudio_configure.py | 183 ++-- .../src/general/scripts/azure/ssn_configure.py | 227 ++--- .../src/general/scripts/azure/ssn_prepare.py | 170 ++-- .../src/general/scripts/azure/ssn_terminate.py | 93 +- .../src/general/scripts/azure/tensor_configure.py | 181 ++-- .../general/scripts/azure/zeppelin_configure.py | 187 ++-- ...common_notebook_configure_dataengine-service.py | 93 +- .../gcp/common_notebook_configure_dataengine.py | 84 +- .../general/scripts/gcp/common_prepare_notebook.py | 274 +++--- .../general/scripts/gcp/common_start_notebook.py | 30 +- .../general/scripts/gcp/common_stop_notebook.py | 52 +- .../scripts/gcp/common_terminate_notebook.py | 42 +- .../scripts/gcp/dataengine-service_configure.py | 142 +-- .../scripts/gcp/dataengine-service_create.py | 2 +- .../scripts/gcp/dataengine-service_prepare.py | 172 ++-- .../scripts/gcp/dataengine-service_terminate.py | 43 +- .../general/scripts/gcp/dataengine_configure.py | 199 ++--- .../src/general/scripts/gcp/dataengine_prepare.py | 297 +++--- .../src/general/scripts/gcp/dataengine_start.py | 57 +- .../src/general/scripts/gcp/dataengine_stop.py | 47 +- .../general/scripts/gcp/dataengine_terminate.py | 53 +- .../general/scripts/gcp/deeplearning_configure.py | 288 +++--- .../src/general/scripts/gcp/edge_configure.py | 331 ++++--- .../src/general/scripts/gcp/edge_start.py | 35 +- .../src/general/scripts/gcp/edge_stop.py | 28 +- .../src/general/scripts/gcp/edge_terminate.py | 92 +- .../src/general/scripts/gcp/jupyter_configure.py | 262 +++--- .../general/scripts/gcp/jupyterlab_configure.py | 299 +++---- .../src/general/scripts/gcp/project_prepare.py | 442 ++++----- .../src/general/scripts/gcp/project_terminate.py | 121 ++- .../src/general/scripts/gcp/rstudio_configure.py | 275 +++--- .../src/general/scripts/gcp/ssn_configure.py | 212 ++--- .../src/general/scripts/gcp/ssn_prepare.py | 229 ++--- .../src/general/scripts/gcp/ssn_terminate.py | 28 +- .../src/general/scripts/gcp/superset_configure.py | 303 ++++--- .../scripts/gcp/tensor-rstudio_configure.py | 287 +++--- .../src/general/scripts/gcp/tensor_configure.py | 277 +++--- .../src/general/scripts/gcp/zeppelin_configure.py | 261 +++--- .../general/scripts/os/common_clean_instance.py | 24 +- .../general/scripts/os/dataengine_install_libs.py | 2 +- .../src/general/scripts/os/dataengine_list_libs.py | 2 +- .../scripts/os/dataengine_reconfigure_spark.py | 2 +- .../src/general/scripts/os/notebook_git_creds.py | 4 +- .../scripts/os/notebook_inactivity_check.py | 4 +- .../general/scripts/os/notebook_install_libs.py | 4 +- .../src/general/scripts/os/notebook_list_libs.py | 4 +- .../scripts/os/notebook_reconfigure_spark.py | 2 +- .../src/general/templates/os/renew_certificates.sh | 2 +- .../src/jupyter/scripts/configure_jupyter_node.py | 4 +- .../scripts/configure_jupyterlab_node.py | 4 +- infrastructure-provisioning/src/project/fabfile.py | 7 - .../src/project/scripts/configure_keycloak.py | 5 +- .../src/rstudio/scripts/configure_rstudio_node.py | 4 +- .../src/ssn/files/azure/mongo_roles.json | 2 +- .../superset/scripts/configure_superset_node.py | 4 +- .../scripts/configure_tensor-rstudio_node.py | 4 +- .../src/tensor/scripts/configure_tensor_node.py | 4 +- .../zeppelin/scripts/configure_zeppelin_node.py | 4 +- .../aws/computational_resources/modules/ami/ami.tf | 2 +- .../computational_resources/modules/common/iam.tf | 2 +- .../modules/common/network.tf | 4 +- .../modules/data_engine/instance.tf | 4 +- .../modules/emr/instance.tf | 2 +- .../modules/notebook/instance.tf | 2 +- .../terraform/aws/endpoint/main/bucket.tf | 37 + .../terraform/aws/endpoint/main/iam.tf | 2 +- .../terraform/aws/endpoint/main/instance.tf | 2 +- .../terraform/aws/endpoint/main/network.tf | 18 +- .../terraform/aws/project/main/iam.tf | 4 +- .../terraform/aws/project/main/instance.tf | 2 +- .../terraform/aws/project/main/network.tf | 8 +- .../aws/ssn-k8s/main/auto_scaling_groups.tf | 8 +- .../terraform/aws/ssn-k8s/main/lb.tf | 6 +- .../terraform/aws/ssn-k8s/main/role_policy.tf | 2 +- .../terraform/aws/ssn-k8s/main/s3.tf | 2 +- .../terraform/aws/ssn-k8s/main/security_groups.tf | 2 +- .../terraform/aws/ssn-k8s/main/vpc.tf | 18 +- .../modules/data_engine/instance.tf | 4 +- .../modules/notebook/instance.tf | 4 +- .../terraform/azure/endpoint/main/bucket.tf | 53 ++ .../terraform/azure/endpoint/main/instance.tf | 4 +- .../terraform/azure/endpoint/main/main.tf | 7 +- .../terraform/azure/endpoint/main/network.tf | 10 +- .../terraform/azure/endpoint/main/outputs.tf | 8 + .../terraform/azure/project/main/instance.tf | 2 +- .../terraform/bin/deploy/endpoint_fab.py | 3 +- infrastructure-provisioning/terraform/bin/dlab.py | 4 +- .../terraform/gcp/endpoint/main/bucket.tf | 36 + .../terraform/gcp/endpoint/main/network.tf | 4 +- .../terraform/gcp/ssn-gke/main/modules/gke/gke.tf | 2 +- .../terraform/gcp/ssn-gke/main/modules/gke/vpc.tf | 4 +- .../dlab/automation/cloud/aws/AmazonHelper.java | 2 +- .../java/com/epam/dlab/dto/StatusEnvBaseDTO.java | 14 + .../handlers/ComputationalCallbackHandler.java | 5 +- .../ComputationalConfigureCallbackHandler.java | 1 + .../handlers/ExploratoryCallbackHandler.java | 8 +- .../handlers/LibInstallCallbackHandler.java | 5 +- .../resources/base/ExploratoryService.java | 2 +- .../aws/dataengine-service_configure.json | 2 +- .../resources/mock_response/aws/edge_create.json | 2 +- .../mock_response/aws/notebook_start.json | 2 +- .../resources/mock_response/aws/notebook_stop.json | 2 +- .../mock_response/aws/notebook_terminate.json | 2 +- .../azure/dataengine-service_configure.json | 2 +- .../resources/mock_response/azure/edge_create.json | 2 +- .../mock_response/azure/notebook_start.json | 2 +- .../mock_response/azure/notebook_stop.json | 2 +- .../mock_response/azure/notebook_terminate.json | 2 +- .../gcp/dataengine-service_configure.json | 2 +- .../core/commands/CommandExecutorMockTest.java | 2 +- .../epam/dlab/backendapi/dao/ComputationalDAO.java | 182 ++-- .../java/com/epam/dlab/backendapi/dao/EnvDAO.java | 74 +- .../epam/dlab/backendapi/dao/ExploratoryDAO.java | 151 +--- .../dlab/backendapi/dao/ExploratoryLibDAO.java | 60 +- .../com/epam/dlab/backendapi/dao/IndexCreator.java | 7 +- .../epam/dlab/backendapi/dao/SchedulerJobDAO.java | 38 +- .../epam/dlab/backendapi/dao/UserRoleDaoImpl.java | 2 + .../dlab/backendapi/domain/CreateProjectDTO.java | 2 +- .../epam/dlab/backendapi/domain/EndpointDTO.java | 6 +- .../epam/dlab/backendapi/domain/ProjectDTO.java | 2 +- .../backendapi/resources/EnvironmentResource.java | 34 +- .../backendapi/resources/ExploratoryResource.java | 29 +- .../resources/ImageExploratoryResource.java | 18 +- .../resources/LibExploratoryResource.java | 31 +- .../dlab/backendapi/resources/ProjectResource.java | 23 +- .../backendapi/resources/SchedulerJobResource.java | 31 +- .../resources/aws/ComputationalResourceAws.java | 36 +- .../azure/ComputationalResourceAzure.java | 58 +- .../callback/CheckInactivityCallback.java | 5 +- .../resources/callback/ComputationalCallback.java | 15 +- .../callback/EnvironmentStatusCallback.java | 4 +- .../resources/callback/ExploratoryCallback.java | 26 +- ...FormDTO.java => ComputationalTemplatesDTO.java} | 20 +- .../dto/ExploratoryImageCreateFormDTO.java | 16 +- .../resources/dto/LibInstallFormDTO.java | 3 + .../resources/dto/SearchLibsFormDTO.java | 4 + .../dlab/backendapi/resources/dto/UserRoleDto.java | 10 +- .../resources/gcp/ComputationalResourceGcp.java | 44 +- .../backendapi/service/ComputationalService.java | 21 +- .../backendapi/service/EnvironmentService.java | 10 +- .../backendapi/service/ExploratoryService.java | 25 +- .../service/ImageExploratoryService.java | 10 +- .../dlab/backendapi/service/InactivityService.java | 8 +- .../dlab/backendapi/service/LibraryService.java | 10 +- .../dlab/backendapi/service/ProjectService.java | 2 +- .../backendapi/service/SchedulerJobService.java | 106 +-- .../service/impl/ComputationalServiceImpl.java | 158 ++-- .../service/impl/EnvironmentServiceImpl.java | 24 +- .../service/impl/ExploratoryServiceImpl.java | 120 +-- .../service/impl/ImageExploratoryServiceImpl.java | 10 +- .../service/impl/InactivityServiceImpl.java | 4 +- .../service/impl/LibraryServiceImpl.java | 50 +- .../service/impl/ProjectServiceImpl.java | 30 +- .../service/impl/ReuploadKeyServiceImpl.java | 66 +- .../service/impl/SchedulerJobServiceImpl.java | 81 +- .../epam/dlab/backendapi/util/RequestBuilder.java | 35 +- .../src/main/resources/mongo/aws/mongo_roles.json | 28 + .../main/resources/mongo/azure/mongo_roles.json | 25 +- .../src/main/resources/mongo/gcp/mongo_roles.json | 26 + .../manage-environment-dilog.component.html | 2 +- .../management-grid/management-grid.component.html | 7 +- .../management-grid/management-grid.component.scss | 26 +- .../management/management.component.html | 2 +- .../management/management.component.ts | 1 + .../administration/project/project-data.service.ts | 11 +- .../project-list/project-list.component.html | 4 +- .../project-list/project-list.component.scss | 8 +- .../app/administration/roles/roles.component.html | 84 +- .../app/administration/roles/roles.component.scss | 29 +- .../app/administration/roles/roles.component.ts | 118 ++- .../services/applicationServiceFacade.service.ts | 9 + .../services/dataengineConfiguration.service.ts | 16 +- .../core/services/librariesInstallation.service.ts | 12 +- .../services/managementEnvironments.service.ts | 4 +- .../src/app/core/services/scheduler.service.ts | 8 +- .../src/app/core/services/userResource.service.ts | 12 +- .../reporting-grid/reporting-grid.component.scss | 18 +- .../src/app/reporting/reporting.component.ts | 2 +- .../cluster-details/cluster-details.component.ts | 4 +- ...utational-resource-create-dialog.component.html | 9 +- ...mputational-resource-create-dialog.component.ts | 30 +- .../computational-resources-list.component.scss | 25 +- .../computational-resources-list.component.ts | 2 +- .../ami-create-dialog.component.ts | 3 +- .../detail-dialog/detail-dialog.component.ts | 4 +- .../install-libraries.component.scss | 13 + .../install-libraries.component.ts | 4 +- .../install-libraries/install-libraries.model.ts | 9 +- .../resources-grid/resources-grid.component.html | 4 +- .../resources-grid/resources-grid.component.scss | 12 + .../resources-grid/resources-grid.component.ts | 8 +- .../app/resources/scheduler/scheduler.component.ts | 23 +- .../src/app/resources/scheduler/scheduler.model.ts | 12 +- .../webapp/src/app/shared/form-controls/index.ts | 5 +- .../multi-level-select-dropdown.component.html | 70 ++ .../multi-level-select-dropdown.component.scss | 319 +++++++ .../multi-level-select-dropdown.component.ts | 102 +++ .../confirmation-dialog.component.html | 10 +- .../confirmation-dialog.component.scss | 3 + .../confirmation-dialog.model.ts | 6 +- .../notification-dialog.component.ts | 14 +- .../webapp/src/assets/styles/_dialogs.scss | 2 +- .../resources/webapp/src/assets/styles/_theme.scss | 8 +- .../webapp/src/dictionary/azure.dictionary.ts | 8 +- .../resources/EnvironmentResourceTest.java | 89 +- .../resources/ExploratoryResourceTest.java | 57 +- .../resources/ImageExploratoryResourceTest.java | 25 +- .../resources/LibExploratoryResourceTest.java | 602 +++++++------ .../backendapi/resources/ProjectResourceTest.java | 32 +- .../resources/SchedulerJobResourceTest.java | 383 ++++---- .../service/impl/ComputationalServiceImplTest.java | 993 +++++++++++---------- .../service/impl/EnvironmentServiceImplTest.java | 74 +- .../service/impl/ExploratoryServiceImplTest.java | 586 ++++++------ .../impl/ImageExploratoryServiceImplTest.java | 61 +- .../service/impl/LibraryServiceImplTest.java | 464 +++++----- .../service/impl/ReuploadKeyServiceImplTest.java | 49 +- .../service/impl/SchedulerJobServiceImplTest.java | 250 +++--- 287 files changed, 11576 insertions(+), 10482 deletions(-) diff --cc services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java index 80a6b3c,3a18dab..cbe1c7f --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java @@@ -155,28 -136,10 +136,25 @@@ public class ExploratoryDAO extends Bas return getUserInstances(and(eq(PROJECT, project)), false); } + public List<UserInstanceDTO> fetchExploratoryFieldsForProjectWithComp(String project) { + return getUserInstances(and(eq(PROJECT, project)), true); + } + + public List<UserInstanceDTO> findExploratories(String project, String endpoint, String user) { + return getUserInstances(and(eq(PROJECT, project), eq(ENDPOINT, endpoint), eq(USER, user)), true); + } + - /** - * Finds and returns the info of all user's notebooks whose status is present among predefined ones. - * - * @param user user name. - * @param computationalFieldsRequired true/false. - * @param statuses array of statuses. - */ + public List<UserInstanceDTO> fetchUserExploratoriesWhereStatusIn(String user, boolean computationalFieldsRequired, + UserInstanceStatus... statuses) { + final List<String> statusList = statusList(statuses); + return getUserInstances( + and( + eq(USER, user), + in(STATUS, statusList) + ), + computationalFieldsRequired); + } + /** * Finds and returns the info of all user's notebooks whose status or status of affiliated computational resource * is present among predefined ones. diff --cc services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java index f744bc5,2b93a8e..2fe7d8e --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java @@@ -32,31 -32,21 +32,24 @@@ import java.util.Optional public interface ExploratoryService { - String start(UserInfo userInfo, String exploratoryName, String project); + String start(UserInfo userInfo, String exploratoryName, String project); - String stop(UserInfo userInfo, String exploratoryName); + String stop(UserInfo userInfo, String project, String exploratoryName); - String terminate(UserInfo userInfo, String exploratoryName); + String terminate(UserInfo userInfo, String project, String exploratoryName); - String create(UserInfo userInfo, Exploratory exploratory, String project); + String create(UserInfo userInfo, Exploratory exploratory, String project); - void updateExploratoryStatuses(String user, UserInstanceStatus status); + void updateProjectExploratoryStatuses(String project, String endpoint, UserInstanceStatus status); - void updateProjectExploratoryStatuses(String project, String endpoint, UserInstanceStatus status); - - void updateExploratoriesReuploadKeyFlag(String user, boolean reuploadKeyRequired, - UserInstanceStatus... exploratoryStatuses); - - List<UserInstanceDTO> getInstancesWithStatuses(String user, UserInstanceStatus exploratoryStatus, - UserInstanceStatus computationalStatus); + void updateClusterConfig(UserInfo userInfo, String project, String exploratoryName, List<ClusterConfig> config); + Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName); + List<UserInstanceDTO> findAll(); + + void updateClusterConfig(UserInfo userInfo, String exploratoryName, List<ClusterConfig> config); - Optional<UserInstanceDTO> getUserInstance(String user, String exploratoryName); - - List<ClusterConfig> getClusterConfig(UserInfo user, String exploratoryName); + List<ClusterConfig> getClusterConfig(UserInfo user, String project, String exploratoryName); - ExploratoryCreatePopUp getUserInstances(UserInfo user); + ExploratoryCreatePopUp getUserInstances(UserInfo user); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
