This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch DLAB-1571 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit b6877821f08bc25d45f80e9cdc4a4cba7da4186c Author: Oleh Fuks <[email protected]> AuthorDate: Mon Mar 2 17:21:29 2020 +0200 Added shape and status filter --- .../epam/dlab/billing/gcp/conf/SecurityConfig.java | 3 ++- .../billing-gcp/src/main/resources/application.yml | 3 --- .../computational/UserComputationalResource.java | 6 ++++-- .../dlab/backendapi/domain/BillingReportDTO.java | 3 +++ .../service/impl/BillingServiceImplNew.java | 15 +++++++++------ .../com/epam/dlab/backendapi/util/BillingUtils.java | 21 ++++++++++++++++++--- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/SecurityConfig.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/SecurityConfig.java index 7e2d5d7..ad960b0 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/SecurityConfig.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/SecurityConfig.java @@ -60,6 +60,7 @@ class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter { http .anonymous().disable() .authorizeRequests() - .antMatchers("/*").authenticated(); + .anyRequest() + .authenticated(); } } \ No newline at end of file diff --git a/services/billing-gcp/src/main/resources/application.yml b/services/billing-gcp/src/main/resources/application.yml index cfd2b36..81ac3d6 100644 --- a/services/billing-gcp/src/main/resources/application.yml +++ b/services/billing-gcp/src/main/resources/application.yml @@ -26,9 +26,6 @@ server.ssl.key-alias: billing logging: file: /var/opt/dlab/log/ssn/billing.log level: - org: - springframework: - security: DEBUG com: epam: trace diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java index 678025f..56da2f6 100644 --- a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java +++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java @@ -61,15 +61,17 @@ public class UserComputationalResource { private String masterNodeShape; @JsonProperty("dataengine_instance_shape") private String dataengineShape; + @JsonProperty("dataengine_instance_count") + private int dataengineInstanceCount; @JsonProperty("instance_id") private String instanceId; protected List<ClusterConfig> config; - private Map<String,String> tags; + private Map<String, String> tags; public UserComputationalResource(String computationalName, String computationalId, String imageName, String templateName, String status, Date uptime, SchedulerJobDTO schedulerData, boolean reuploadKeyRequired, List<ResourceURL> resourceUrl, - LocalDateTime lastActivity, Map<String,String> tags) { + LocalDateTime lastActivity, Map<String, String> tags) { this.computationalName = computationalName; this.computationalId = computationalId; this.imageName = imageName; diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java index 25f1831..8816f33 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java @@ -19,6 +19,7 @@ package com.epam.dlab.backendapi.domain; +import com.epam.dlab.dto.UserInstanceStatus; import com.epam.dlab.dto.billing.BillingResourceType; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; @@ -44,4 +45,6 @@ public class BillingReportDTO { private String currency; @JsonProperty("resource_type") private BillingResourceType resourceType; + private UserInstanceStatus status; + private String shape; } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java index 19413bf..2b44f44 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java @@ -111,7 +111,8 @@ public class BillingServiceImplNew implements BillingServiceNew { private Stream<BillingReportDTO> projectEdges(String serviceBaseName, String projectName, List<ProjectEndpointDTO> endpoints) { return endpoints .stream() - .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName())); + .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName(), + endpoint.getStatus().toString())); } private BillingReportDTO getOrDefault(Map<String, BillingReportDTO> billableResources, String tag) { @@ -174,15 +175,15 @@ public class BillingServiceImplNew implements BillingServiceNew { private Predicate<BillingReportDTO> getBillingReportFilter(BillingFilter filter) { return br -> (filter.getUsers().isEmpty() || filter.getUsers().contains(br.getUser())) && (filter.getProjects().isEmpty() || filter.getProjects().contains(br.getProject())) && - (filter.getResourceTypes().isEmpty() || filter.getResourceTypes().contains(br.getResourceType().name())); + (filter.getResourceTypes().isEmpty() || filter.getResourceTypes().contains(br.getResourceType().name())) && + (filter.getStatuses().isEmpty() || filter.getStatuses().contains(br.getStatus())) && + (filter.getShapes().isEmpty() || filter.getShapes().contains(br.getShape())); } private Predicate<BillingData> getBillingDataFilter(BillingFilter filter) { - LocalDate endDate = LocalDate.parse(filter.getDateEnd()); - LocalDate startDate = LocalDate.parse(filter.getDateStart()); return br -> (filter.getDlabId().isEmpty() || filter.getDlabId().equalsIgnoreCase(br.getTag())) && - (filter.getDateStart().isEmpty() || startDate.isEqual(br.getUsageDateFrom()) || startDate.isBefore(br.getUsageDateFrom())) && - (filter.getDateEnd().isEmpty() || endDate.isEqual(br.getUsageDateTo()) || endDate.isAfter(br.getUsageDateTo())) && + (filter.getDateStart().isEmpty() || LocalDate.parse(filter.getDateStart()).isEqual(br.getUsageDateFrom()) || LocalDate.parse(filter.getDateStart()).isBefore(br.getUsageDateFrom())) && + (filter.getDateEnd().isEmpty() || LocalDate.parse(filter.getDateEnd()).isEqual(br.getUsageDateTo()) || LocalDate.parse(filter.getDateEnd()).isAfter(br.getUsageDateTo())) && (filter.getProducts().isEmpty() || filter.getProducts().contains(br.getProduct())); } @@ -199,6 +200,8 @@ public class BillingServiceImplNew implements BillingServiceNew { .dlabId(billingData.getTag()) .resourceType(billingReportDTO.getResourceType()) .resourceName(billingReportDTO.getResourceName()) + .status(billingReportDTO.getStatus()) + .shape(billingReportDTO.getShape()) .build(); } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java index beff13c..02d4682 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java @@ -21,6 +21,8 @@ package com.epam.dlab.backendapi.util; import com.epam.dlab.backendapi.domain.BillingReportDTO; import com.epam.dlab.dto.UserInstanceDTO; +import com.epam.dlab.dto.UserInstanceStatus; +import com.epam.dlab.dto.base.DataEngineType; import com.epam.dlab.dto.billing.BillingResourceType; import com.epam.dlab.dto.computational.UserComputationalResource; @@ -37,12 +39,16 @@ public class BillingUtils { private static final String VOLUME_SECONDARY = "Volume secondary"; private static final String SHARED_RESOURCE = "Shared resource"; - public static Stream<BillingReportDTO> edgeBillingDataStream(String project, String sbn, String endpoint) { + private static final String DATAENGINE_NAME_FORMAT = "%d x %s"; + private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %s%sSlave: %d x %s"; + + public static Stream<BillingReportDTO> edgeBillingDataStream(String project, String sbn, String endpoint, String status) { final String userEdgeId = String.format(EDGE_FORMAT, sbn, project.toLowerCase(), endpoint); final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint); final String edgeBucketId = String.format(EDGE_BUCKET_FORMAT, sbn, project.toLowerCase()); return Stream.of( - BillingReportDTO.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingResourceType.EDGE).build(), + BillingReportDTO.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingResourceType.EDGE) + .status(UserInstanceStatus.of(status)).build(), BillingReportDTO.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingResourceType.VOLUME).build(), BillingReportDTO.builder().resourceName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingResourceType.EDGE_BUCKET).build() ); @@ -70,7 +76,8 @@ public class BillingUtils { final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId); final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId); final Stream<BillingReportDTO> exploratoryStream = Stream.of( - withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY).build(), + withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY) + .status(UserInstanceStatus.of(userInstance.getStatus())).shape(userInstance.getShape()).build(), withExploratoryName(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(), withExploratoryName(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build()); return Stream.concat(computationalStream, exploratoryStream); @@ -83,10 +90,18 @@ public class BillingUtils { .resourceName(cr.getComputationalName()) .resourceType(BillingResourceType.COMPUTATIONAL) .project(userInstance.getProject()) + .status(UserInstanceStatus.of(cr.getStatus())) + .shape(getComputationalShape(cr)) .build(); } private static BillingReportDTO.BillingReportDTOBuilder withExploratoryName(UserInstanceDTO userInstance) { return BillingReportDTO.builder().user(userInstance.getUser()).project(userInstance.getProject()); } + + private static String getComputationalShape(UserComputationalResource resource) { + return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ? + String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) : + String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), System.lineSeparator(), null, null); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
