This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2363
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2363 by this push:
new 08a3fcb [DATALAB-2363] - Convey to front-end image list and
description
08a3fcb is described below
commit 08a3fcb1ae797a4b6376849c5cc2b2f46adb1543
Author: KinashYurii <[email protected]>
AuthorDate: Fri May 21 16:11:07 2021 +0300
[DATALAB-2363] - Convey to front-end image list and description
---
.../computational/FullComputationalTemplate.java | 9 +-
.../datalab/dto/imagemetadata/ApplicationDto.java | 60 +-----------
.../imagemetadata/ComputationalMetadataDTO.java | 3 +-
.../ExploratoryEnvironmentImages.java | 13 +++
.../dto/imagemetadata/ExploratoryMetadataDTO.java | 6 +-
.../dto/imagemetadata/ImageMetadataDTO.java | 20 +---
.../datalab/dto/imagemetadata/TemplateDTO.java | 57 +----------
.../resources/dto/ExploratoryCreateFormDTO.java | 9 --
.../impl/InfrastructureTemplateServiceImpl.java | 107 ++++++++++-----------
9 files changed, 84 insertions(+), 200 deletions(-)
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/base/computational/FullComputationalTemplate.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/base/computational/FullComputationalTemplate.java
index 09cc0b8..bbcf652 100644
---
a/services/datalab-model/src/main/java/com/epam/datalab/dto/base/computational/FullComputationalTemplate.java
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/base/computational/FullComputationalTemplate.java
@@ -21,13 +21,10 @@ package com.epam.datalab.dto.base.computational;
import com.epam.datalab.dto.imagemetadata.ComputationalMetadataDTO;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
+import lombok.AllArgsConstructor;
+@AllArgsConstructor
public class FullComputationalTemplate {
@JsonUnwrapped
- private ComputationalMetadataDTO computationalMetadataDTO;
-
-
- public FullComputationalTemplate(ComputationalMetadataDTO metadataDTO) {
- this.computationalMetadataDTO = metadataDTO;
- }
+ private final ComputationalMetadataDTO computationalMetadataDTO;
}
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ApplicationDto.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ApplicationDto.java
index ad096fe..4023cab 100644
---
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ApplicationDto.java
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ApplicationDto.java
@@ -20,66 +20,14 @@
package com.epam.datalab.dto.imagemetadata;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
-
-import java.util.Objects;
+import lombok.*;
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
public class ApplicationDto {
@JsonProperty("Version")
private String version;
@JsonProperty("Name")
private String name;
-
- public ApplicationDto() {
- }
-
-
- public ApplicationDto(String version, String name) {
- this.version = version;
- this.name = name;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ApplicationDto that = (ApplicationDto) o;
-
- if (version != null ? !version.equals(that.version) : that.version !=
null) {
- return false;
- }
- return name != null ? name.equals(that.name) : that.name == null;
-
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(version, name);
- }
-
- @Override
- public String toString() {
- return ReflectionToStringBuilder.toString(this);
- }
}
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ComputationalMetadataDTO.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ComputationalMetadataDTO.java
index a70a74a..eb5d540 100644
---
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ComputationalMetadataDTO.java
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ComputationalMetadataDTO.java
@@ -47,8 +47,7 @@ public class ComputationalMetadataDTO extends
ImageMetadataDTO {
private String requestId;
@JsonProperty(value = "computation_resources_shapes")
private Map<String, List<ComputationalResourceShapeDto>>
computationResourceShapes;
- //GPU
-// @JsonProperty("gpu_types")
+
private List<String> computationGPU;
public ComputationalMetadataDTO(String imageName) {
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryEnvironmentImages.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryEnvironmentImages.java
new file mode 100644
index 0000000..08ab5b5
--- /dev/null
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryEnvironmentImages.java
@@ -0,0 +1,13 @@
+package com.epam.datalab.dto.imagemetadata;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+public class ExploratoryEnvironmentImages {
+
+ @JsonProperty("Image family")
+ private String imageFamily;
+ @JsonProperty("Description")
+ private String description;
+}
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryMetadataDTO.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryMetadataDTO.java
index 07dbff5..6a477e5 100644
---
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryMetadataDTO.java
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ExploratoryMetadataDTO.java
@@ -34,17 +34,15 @@ import java.util.List;
@EqualsAndHashCode(callSuper = false)
@JsonIgnoreProperties
public class ExploratoryMetadataDTO extends ImageMetadataDTO {
- @JsonProperty
protected String image;
@JsonProperty("exploratory_environment_versions")
private List<ExploratoryEnvironmentVersion> exploratoryEnvironmentVersions;
@JsonProperty("exploratory_environment_shapes")
private HashMap<String, List<ComputationalResourceShapeDto>>
exploratoryEnvironmentShapes;
+ @JsonProperty("exploratory_environment_images")
+ private List<ExploratoryEnvironmentImages> exploratoryEnvironmentImages;
@JsonProperty("request_id")
private String requestId;
-
- //GPU
-// @JsonProperty("gpu_types")
private List<String> computationGPU;
public ExploratoryMetadataDTO(String imageName) {
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ImageMetadataDTO.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ImageMetadataDTO.java
index 33d508a..88a6a5e 100644
---
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ImageMetadataDTO.java
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/ImageMetadataDTO.java
@@ -20,6 +20,7 @@
package com.epam.datalab.dto.imagemetadata;
import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.*;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
/**
@@ -29,23 +30,10 @@ import
org.apache.commons.lang3.builder.ReflectionToStringBuilder;
* hierarchy as it will requite type information to be serialized within json
* which is not we really want.
*/
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
public abstract class ImageMetadataDTO {
@JsonProperty("image_type")
private ImageType imageType;
-
- public ImageType getImageType() {
- return imageType;
- }
-
- public void setImageType(ImageType imageType) {
- this.imageType = imageType;
- }
-
- public abstract void setImage(String image);
-
- @Override
- public String toString() {
- return ReflectionToStringBuilder.toString(this);
- }
-
}
diff --git
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/TemplateDTO.java
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/TemplateDTO.java
index 746ab8a..58a8987 100644
---
a/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/TemplateDTO.java
+++
b/services/datalab-model/src/main/java/com/epam/datalab/dto/imagemetadata/TemplateDTO.java
@@ -20,65 +20,16 @@
package com.epam.datalab.dto.imagemetadata;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import lombok.*;
import java.util.List;
-import java.util.Objects;
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
public class TemplateDTO {
@JsonProperty
private String version;
@JsonProperty
private List<ApplicationDto> applications;
-
- public TemplateDTO() {
- }
-
- public TemplateDTO(String version) {
- this.version = version;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public List<ApplicationDto> getApplications() {
- return applications;
- }
-
- public void setApplications(List<ApplicationDto> applications) {
- this.applications = applications;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- TemplateDTO that = (TemplateDTO) o;
-
- if (version != null ? !version.equals(that.version) : that.version !=
null) {
- return false;
- }
- return applications != null ? applications.equals(that.applications) :
that.applications == null;
-
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(version, applications);
- }
-
- @Override
- public String toString() {
- return ReflectionToStringBuilder.toString(this);
- }
}
diff --git
a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ExploratoryCreateFormDTO.java
b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ExploratoryCreateFormDTO.java
index 8440ac5..0d80ff5 100644
---
a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ExploratoryCreateFormDTO.java
+++
b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ExploratoryCreateFormDTO.java
@@ -35,39 +35,30 @@ public class ExploratoryCreateFormDTO {
@NotBlank
@JsonProperty
private String image;
-
@NotBlank
@JsonProperty("template_name")
private String templateName;
-
@NotBlank
@JsonProperty
private String name;
-
@NotBlank
@JsonProperty
private String project;
@JsonProperty("custom_tag")
private String exploratoryTag;
-
@NotBlank
@JsonProperty
private String endpoint;
-
@NotBlank
@JsonProperty
private String shape;
-
@NotBlank
@JsonProperty
private String version;
-
@JsonProperty("notebook_image_name")
private String imageName;
-
@JsonProperty("cluster_config")
private List<ClusterConfig> clusterConfig;
-
@JsonProperty("gpu_enabled")
private Boolean enabledGPU;
@JsonProperty("gpuType")
diff --git
a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
index e0ee035..03c56e5 100644
---
a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
+++
b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureTemplateServiceImpl.java
@@ -82,31 +82,11 @@ public class InfrastructureTemplateServiceImpl implements
InfrastructureTemplate
@Override
public List<ExploratoryMetadataDTO> getExploratoryTemplates(UserInfo user,
String project, String endpoint) {
-
log.debug("Loading list of exploratory templates for user {} for
project {}", user.getName(), project);
try {
EndpointDTO endpointDTO = endpointService.get(endpoint);
- log.info("TEST LOG!!! endpoint: {}", endpointDTO.toString());
-
- ExploratoryMetadataDTO[] array =
- provisioningService.get(endpointDTO.getUrl() +
DOCKER_EXPLORATORY,
- user.getAccessToken(),
- ExploratoryMetadataDTO[].class);
- log.info("loaded: {}", (Object) array);
-
- final Set<String> roles =
userGroupDao.getUserGroups(user.getName());
- log.info("TEST LOG!!! roles: {}", roles);
- List<ExploratoryMetadataDTO> s= Arrays.stream(array)
- .peek(e -> e.setImage(getSimpleImageName(e.getImage())))
- .filter(e -> exploratoryGpuIssuesAzureFilter(e,
endpointDTO.getCloudProvider()) &&
- UserRoles.checkAccess(user, RoleType.EXPLORATORY,
e.getImage(), roles))
- .peek(e -> filterShapes(user,
e.getExploratoryEnvironmentShapes(), RoleType.EXPLORATORY_SHAPES, roles))
- .peek(e -> addGpu(e, project))
- .collect(Collectors.toList());
- log.info("loaded: {}", s);
-
- return s;
-
+ Set<String> roles = userGroupDao.getUserGroups(user.getName());
+ return changeExploratoryMetadata(user, project, endpointDTO,
roles);
} catch (DatalabException e) {
log.error("Could not load list of exploratory templates for user:
{}", user.getName(), e);
throw e;
@@ -143,6 +123,50 @@ public class InfrastructureTemplateServiceImpl implements
InfrastructureTemplate
}
}
+ protected FullComputationalTemplate
getCloudFullComputationalTemplate(ComputationalMetadataDTO metadataDTO,
+
CloudProvider cloudProvider) {
+
+ switch (cloudProvider) {
+ case AWS:
+ return new AwsFullComputationalTemplate(metadataDTO,
+ AwsEmrConfiguration.builder()
+
.minEmrInstanceCount(configuration.getMinEmrInstanceCount())
+
.maxEmrInstanceCount(configuration.getMaxEmrInstanceCount())
+
.maxEmrSpotInstanceBidPct(configuration.getMaxEmrSpotInstanceBidPct())
+
.minEmrSpotInstanceBidPct(configuration.getMinEmrSpotInstanceBidPct())
+ .build());
+ case GCP:
+ return new GcpFullComputationalTemplate(metadataDTO,
+ GcpDataprocConfiguration.builder()
+
.minInstanceCount(configuration.getMinInstanceCount())
+
.maxInstanceCount(configuration.getMaxInstanceCount())
+
.minDataprocPreemptibleInstanceCount(configuration.getMinDataprocPreemptibleCount())
+ .build());
+ case AZURE:
+ log.error("Dataengine service is not supported currently for
{}", AZURE);
+ throw new UnsupportedOperationException("Dataengine service is
not supported currently for " + AZURE);
+ default:
+ throw new UnsupportedOperationException("Dataengine service is
not supported currently for " + cloudProvider);
+ }
+ }
+
+ private List<ExploratoryMetadataDTO> changeExploratoryMetadata(UserInfo
user, String project, EndpointDTO endpointDTO, Set<String> roles) {
+ return getExploratoryMetadata(user, endpointDTO).stream()
+ .peek(e -> e.setImage(getSimpleImageName(e.getImage())))
+ .filter(e -> exploratoryGpuIssuesAzureFilter(e,
endpointDTO.getCloudProvider()) &&
+ UserRoles.checkAccess(user, RoleType.EXPLORATORY,
e.getImage(), roles))
+ .peek(e -> filterShapes(user,
e.getExploratoryEnvironmentShapes(), RoleType.EXPLORATORY_SHAPES, roles))
+ .peek(e -> addGpu(e, project))
+ .collect(Collectors.toList());
+ }
+
+ private List<ExploratoryMetadataDTO> getExploratoryMetadata(UserInfo user,
EndpointDTO endpointDTO) {
+ return Arrays.asList(provisioningService.get(endpointDTO.getUrl() +
DOCKER_EXPLORATORY,
+ user.getAccessToken(),
+ ExploratoryMetadataDTO[].class));
+ }
+
+
/**
* Removes shapes for which user does not have an access
*
@@ -203,33 +227,16 @@ public class InfrastructureTemplateServiceImpl implements
InfrastructureTemplate
}
}
- protected FullComputationalTemplate
getCloudFullComputationalTemplate(ComputationalMetadataDTO metadataDTO,
-
CloudProvider cloudProvider) {
-
- switch (cloudProvider) {
- case AWS:
- return new AwsFullComputationalTemplate(metadataDTO,
- AwsEmrConfiguration.builder()
-
.minEmrInstanceCount(configuration.getMinEmrInstanceCount())
-
.maxEmrInstanceCount(configuration.getMaxEmrInstanceCount())
-
.maxEmrSpotInstanceBidPct(configuration.getMaxEmrSpotInstanceBidPct())
-
.minEmrSpotInstanceBidPct(configuration.getMinEmrSpotInstanceBidPct())
- .build());
- case GCP:
- return new GcpFullComputationalTemplate(metadataDTO,
- GcpDataprocConfiguration.builder()
-
.minInstanceCount(configuration.getMinInstanceCount())
-
.maxInstanceCount(configuration.getMaxInstanceCount())
-
.minDataprocPreemptibleInstanceCount(configuration.getMinDataprocPreemptibleCount())
- .build());
- case AZURE:
- log.error("Dataengine service is not supported currently for
{}", AZURE);
- throw new UnsupportedOperationException("Dataengine service is
not supported currently for " + AZURE);
- default:
- throw new UnsupportedOperationException("Dataengine service is
not supported currently for " + cloudProvider);
+ private void addGpu(ExploratoryMetadataDTO e, String projectName) {
+ log.info("Trying to read GPU from DB for {}", projectName);
+ try {
+ gpuDAO.getGPUByProjectName(projectName).ifPresent(edgeGPU ->
e.setComputationGPU(edgeGPU.getGpus()));
+ } catch (Exception ex) {
+ log.info(ex.getMessage());
}
}
+
private static class AwsFullComputationalTemplate extends
FullComputationalTemplate {
@JsonProperty("limits")
private AwsEmrConfiguration awsEmrConfiguration;
@@ -263,12 +270,4 @@ public class InfrastructureTemplateServiceImpl implements
InfrastructureTemplate
}
}
- private void addGpu(ExploratoryMetadataDTO e, String projectName) {
- log.info("Trying to read GPU from DB for {}", projectName);
- try {
- gpuDAO.getGPUByProjectName(projectName).ifPresent(edgeGPU ->
e.setComputationGPU(edgeGPU.getGpus()));
- } catch (Exception ex) {
- log.info(ex.getMessage());
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
