This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1571 by this push:
new 525c476 Added detailed billing feature
525c476 is described below
commit 525c476e28413fda0b0142f7423b6c2e897cf365
Author: Oleh Fuks <[email protected]>
AuthorDate: Wed Mar 4 16:27:24 2020 +0200
Added detailed billing feature
---
.../com/epam/dlab/rest/client/RESTService.java | 6 ++-
.../epam/dlab/backendapi/dao/ExploratoryDAO.java | 4 ++
.../resources/InfrastructureInfoResource.java | 9 +++-
.../resources/dto/ProjectInfrastructureInfo.java | 3 ++
.../dlab/backendapi/service/BillingServiceNew.java | 4 ++
.../service/InfrastructureInfoService.java | 2 +-
.../service/impl/BillingServiceImplNew.java | 38 ++++++++++++++---
.../impl/InfrastructureInfoServiceImpl.java | 48 +++++++++++++++-------
.../epam/dlab/backendapi/util/BillingUtils.java | 21 ++++++----
.../resources/InfrastructureInfoResourceTest.java | 15 +++++--
10 files changed, 116 insertions(+), 34 deletions(-)
diff --git
a/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
b/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
index d14ce42..5b4dbba 100644
---
a/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
+++
b/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java
@@ -74,7 +74,11 @@ public class RESTService {
}
public <T> T get(String path, String accessToken, GenericType<T>
genericType) {
- Invocation.Builder builder = getBuilder(path, accessToken,
Collections.emptyMap());
+ return get(path, accessToken, genericType,
Collections.emptyMap());
+ }
+
+ public <T> T get(String path, String accessToken, GenericType<T>
genericType, Map<String, Object> queryParams) {
+ Invocation.Builder builder = getBuilder(path, accessToken,
queryParams);
log.debug("REST get secured {} {}", path, accessToken);
return builder.get(genericType);
}
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
index 160ccce..80a6b3c 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
@@ -155,6 +155,10 @@ public class ExploratoryDAO extends BaseDAO {
return getUserInstances(and(eq(PROJECT, project)), false);
}
+ public List<UserInstanceDTO> findExploratories(String project, String
endpoint, String user) {
+ return getUserInstances(and(eq(PROJECT, project), eq(ENDPOINT,
endpoint), eq(USER, user)), true);
+ }
+
/**
* Finds and returns the info of all user's notebooks whose status is
present among predefined ones.
*
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
index db8197f..ee1ed78 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
@@ -28,7 +28,12 @@ import com.google.inject.Inject;
import io.dropwizard.auth.Auth;
import lombok.extern.slf4j.Slf4j;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
@@ -78,7 +83,7 @@ public class InfrastructureInfoResource {
@GET
@Path("/info")
public List<ProjectInfrastructureInfo> getUserResources(@Auth UserInfo
userInfo) {
- return
infrastructureInfoService.getUserResources(userInfo.getName());
+ return infrastructureInfoService.getUserResources(userInfo);
}
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
index 44837a2..e1f09e9 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
@@ -20,6 +20,7 @@
package com.epam.dlab.backendapi.resources.dto;
import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.dto.billing.BillingData;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.ToString;
@@ -40,5 +41,7 @@ public class ProjectInfrastructureInfo {
@JsonProperty
private Iterable<Document> exploratory;
@JsonProperty
+ private List<BillingData> exploratoryBilling;
+ @JsonProperty
private List<EndpointDTO> endpoints;
}
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
index 8ea0b24..0ec17a9 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java
@@ -23,6 +23,8 @@ import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.domain.BillingReport;
import com.epam.dlab.backendapi.domain.BillingReportLine;
import com.epam.dlab.backendapi.resources.dto.BillingFilter;
+import com.epam.dlab.dto.UserInstanceDTO;
+import com.epam.dlab.dto.billing.BillingData;
import java.util.List;
@@ -32,4 +34,6 @@ public interface BillingServiceNew {
String downloadReport(UserInfo userInfo, BillingFilter filter);
List<BillingReportLine> getBillingReportLines(UserInfo userInfo,
BillingFilter filter);
+
+ List<BillingData> getExploratoryRemoteBillingData(UserInfo user, String
endpoint, List<UserInstanceDTO> userInstanceDTOS);
}
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
index aa23d1d..a385925 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
@@ -27,7 +27,7 @@ import com.epam.dlab.dto.InfrastructureMetaInfoDTO;
import java.util.List;
public interface InfrastructureInfoService {
- List<ProjectInfrastructureInfo> getUserResources(String user);
+ List<ProjectInfrastructureInfo> getUserResources(UserInfo user);
HealthStatusPageDTO getHeathStatus(UserInfo user, boolean fullReport,
boolean isAdmin);
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java
index 41b50b9..ec595e8 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java
@@ -35,6 +35,7 @@ import com.epam.dlab.backendapi.service.ExploratoryService;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.util.BillingUtils;
import com.epam.dlab.constants.ServiceConsts;
+import com.epam.dlab.dto.UserInstanceDTO;
import com.epam.dlab.dto.billing.BillingData;
import com.epam.dlab.exceptions.DlabException;
import com.epam.dlab.rest.client.RESTService;
@@ -51,6 +52,7 @@ import java.net.URISyntaxException;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -66,6 +68,9 @@ import java.util.stream.Stream;
@Slf4j
public class BillingServiceImplNew implements BillingServiceNew {
+ private static final String BILLING_PATH = "/api/billing";
+ private static final String BILLING_REPORT_PATH = "/api/billing/report";
+
private final ProjectService projectService;
private final EndpointService endpointService;
private final ExploratoryService exploratoryService;
@@ -112,7 +117,7 @@ public class BillingServiceImplNew implements
BillingServiceNew {
return builder.toString();
} catch (Exception e) {
log.error("Cannot write billing data ", e);
- throw new DlabException("Cannot write billing file due to " +
e.getMessage());
+ throw new DlabException("Cannot write billing file ", e);
}
}
@@ -159,11 +164,34 @@ public class BillingServiceImplNew implements
BillingServiceNew {
return billableResources.getOrDefault(tag,
BillingReportLine.builder().dlabId(tag).build());
}
+ public List<BillingData> getExploratoryRemoteBillingData(UserInfo user,
String endpoint, List<UserInstanceDTO> userInstanceDTOS) {
+ List<String> dlabIds = null;
+ try {
+ dlabIds = userInstanceDTOS
+ .stream()
+ .map(instance ->
Stream.concat(BillingUtils.getExploratoryIds(instance.getExploratoryId()).stream(),
instance.getResources()
+ .stream()
+ .map(cr ->
BillingUtils.getComputationalIds(cr.getComputationalId()))
+ .flatMap(Collection::stream)
+ ))
+ .flatMap(a -> a)
+ .collect(Collectors.toList());
+
+ EndpointDTO endpointDTO = endpointService.get(endpoint);
+ return provisioningService.get(getBillingUrl(endpointDTO.getUrl(),
BILLING_PATH), user.getAccessToken(),
+ new GenericType<List<BillingData>>() {
+ }, Collections.singletonMap("dlabIds", String.join(",",
dlabIds)));
+ } catch (Exception e) {
+ log.error("Cannot retrieve billing information for {} {}",
dlabIds, e.getMessage(), e);
+ throw new DlabException("Cannot retrieve billing information ", e);
+ }
+ }
+
private List<BillingData> getRemoteBillingData(UserInfo userInfo) {
List<EndpointDTO> endpoints = endpointService.getEndpoints();
ExecutorService executor =
Executors.newFixedThreadPool(endpoints.size());
List<Callable<List<BillingData>>> callableTasks = new ArrayList<>();
- endpoints.forEach(e -> callableTasks.add(getTask(userInfo,
getBillingUrl(e.getUrl()))));
+ endpoints.forEach(e -> callableTasks.add(getTask(userInfo,
getBillingUrl(e.getUrl(), BILLING_REPORT_PATH))));
List<BillingData> billingData;
try {
@@ -176,7 +204,7 @@ public class BillingServiceImplNew implements
BillingServiceNew {
} catch (Exception e) {
executor.shutdown();
log.error("Cannot retrieve billing information {}",
e.getMessage(), e);
- throw new DlabException("Cannot retrieve billing information");
+ throw new DlabException("Cannot retrieve billing information", e);
}
executor.shutdown();
return billingData;
@@ -191,7 +219,7 @@ public class BillingServiceImplNew implements
BillingServiceNew {
}
}
- private String getBillingUrl(String endpointUrl) {
+ private String getBillingUrl(String endpointUrl, String path) {
URI uri;
try {
uri = new URI(endpointUrl);
@@ -203,7 +231,7 @@ public class BillingServiceImplNew implements
BillingServiceNew {
.setScheme(uri.getScheme())
.setHost(uri.getHost())
.setPort(8081)
- .setPath("/api/billing/report")
+ .setPath(path)
.toString();
}
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index ecfe6cf..eaf7071 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -28,6 +28,7 @@ import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.epam.dlab.backendapi.domain.ProjectEndpointDTO;
import com.epam.dlab.backendapi.resources.dto.HealthStatusPageDTO;
import com.epam.dlab.backendapi.resources.dto.ProjectInfrastructureInfo;
+import com.epam.dlab.backendapi.service.BillingServiceNew;
import com.epam.dlab.backendapi.service.EndpointService;
import com.epam.dlab.backendapi.service.InfrastructureInfoService;
import com.epam.dlab.backendapi.service.ProjectService;
@@ -35,6 +36,7 @@ import com.epam.dlab.dto.InfrastructureMetaInfoDTO;
import com.epam.dlab.dto.aws.edge.EdgeInfoAws;
import com.epam.dlab.dto.azure.edge.EdgeInfoAzure;
import com.epam.dlab.dto.base.edge.EdgeInfo;
+import com.epam.dlab.dto.billing.BillingData;
import com.epam.dlab.dto.gcp.edge.EdgeInfoGcp;
import com.epam.dlab.exceptions.DlabException;
import com.google.inject.Inject;
@@ -42,6 +44,7 @@ import com.jcabi.manifests.Manifests;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -54,25 +57,32 @@ public class InfrastructureInfoServiceImpl implements
InfrastructureInfoService
private static final String RELEASE_NOTES_FORMAT =
"https://github.com/apache/incubator-dlab/blob/%s" +
"/RELEASE_NOTES.md";
- @Inject
- private ExploratoryDAO expDAO;
- @Inject
- private EnvDAO envDAO;
- @Inject
- private SelfServiceApplicationConfiguration configuration;
- @Inject
- private BillingDAO billingDAO;
- @Inject
- private ProjectService projectService;
- @Inject
- private EndpointService endpointService;
+ private final ExploratoryDAO expDAO;
+ private final EnvDAO envDAO;
+ private final SelfServiceApplicationConfiguration configuration;
+ private final BillingDAO billingDAO;
+ private final ProjectService projectService;
+ private final EndpointService endpointService;
+ private final BillingServiceNew billingServiceNew;
+ @Inject
+ public InfrastructureInfoServiceImpl(ExploratoryDAO expDAO, EnvDAO
envDAO, SelfServiceApplicationConfiguration configuration,
+
BillingDAO billingDAO, ProjectService projectService, EndpointService
endpointService,
+
BillingServiceNew billingServiceNew) {
+ this.expDAO = expDAO;
+ this.envDAO = envDAO;
+ this.configuration = configuration;
+ this.billingDAO = billingDAO;
+ this.projectService = projectService;
+ this.endpointService = endpointService;
+ this.billingServiceNew = billingServiceNew;
+ }
@Override
- public List<ProjectInfrastructureInfo> getUserResources(String user) {
+ public List<ProjectInfrastructureInfo> getUserResources(UserInfo user) {
log.debug("Loading list of provisioned resources for user {}",
user);
try {
- Iterable<Document> documents =
expDAO.findExploratory(user);
+ Iterable<Document> documents =
expDAO.findExploratory(user.getName());
List<EndpointDTO> allEndpoints =
endpointService.getEndpoints();
return StreamSupport.stream(documents.spliterator(),
false)
@@ -85,12 +95,20 @@ public class InfrastructureInfoServiceImpl implements
InfrastructureInfoService
.filter(endpoint -> endpoints.stream()
.anyMatch(endpoint1 -> endpoint1.getName().equals(endpoint.getName())))
.collect(Collectors.toList());
+
+ List<BillingData> collect =
e.getValue()
+ .stream()
+ .map(exp ->
billingServiceNew.getExploratoryRemoteBillingData(user, (String)
exp.get("endpoint"),
+
expDAO.findExploratories(e.getKey(), (String) exp.get("endpoint"),
user.getName())))
+
.flatMap(Collection::stream)
+
.collect(Collectors.toList());
+
final Map<String, Map<String,
String>> projectEdges =
endpoints.stream()
.collect(Collectors.toMap(ProjectEndpointDTO::getName,
endpointDTO -> getSharedInfo(endpointDTO.getEdgeInfo())));
return new
ProjectInfrastructureInfo(e.getKey(),
-
billingDAO.getBillingProjectQuoteUsed(e.getKey()), projectEdges, e.getValue(),
endpointResult);
+
billingDAO.getBillingProjectQuoteUsed(e.getKey()), projectEdges, e.getValue(),
collect, endpointResult);
})
.collect(Collectors.toList());
} catch (Exception e) {
diff --git
a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index 0f96e4c..b61eb26 100644
---
a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++
b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -82,35 +82,42 @@ public class BillingUtils {
.stream()
.filter(cr -> cr.getComputationalId() != null)
.flatMap(cr ->
Stream.of(computationalBillableResource(userInstance, cr),
-
withExploratoryName(userInstance).resourceName(cr.getComputationalName() + ":"
+ VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT,
cr.getComputationalId()))
+
withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" +
VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT,
cr.getComputationalId()))
.resourceType(BillingResourceType.VOLUME).build()));
final String exploratoryId = userInstance.getExploratoryId();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT,
exploratoryId);
final String secondaryVolumeId =
String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
-
withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY)
+
withUserProject(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY)
.status(UserInstanceStatus.of(userInstance.getStatus())).shape(userInstance.getShape()).build(),
-
withExploratoryName(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(),
-
withExploratoryName(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build());
+
withUserProject(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(),
+
withUserProject(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build());
return Stream.concat(computationalStream, exploratoryStream);
}
private static BillingReportLine
computationalBillableResource(UserInstanceDTO userInstance,
UserComputationalResource cr) {
- return withExploratoryName(userInstance)
+ return withUserProject(userInstance)
.dlabId(cr.getComputationalId())
.resourceName(cr.getComputationalName())
.resourceType(BillingResourceType.COMPUTATIONAL)
- .project(userInstance.getProject())
.status(UserInstanceStatus.of(cr.getStatus()))
.shape(getComputationalShape(cr))
.build();
}
- private static BillingReportLine.BillingReportLineBuilder
withExploratoryName(UserInstanceDTO userInstance) {
+ private static BillingReportLine.BillingReportLineBuilder
withUserProject(UserInstanceDTO userInstance) {
return
BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject());
}
+ public static List<String> getComputationalIds(String computationalId) {
+ return Arrays.asList(computationalId,
String.format(VOLUME_PRIMARY_FORMAT, computationalId));
+ }
+
+ public static List<String> getExploratoryIds(String exploratoryId) {
+ return Arrays.asList(exploratoryId,
String.format(VOLUME_PRIMARY_FORMAT, exploratoryId),
String.format(VOLUME_SECONDARY_FORMAT, exploratoryId));
+ }
+
private static String getComputationalShape(UserComputationalResource
resource) {
return DataEngineType.fromDockerImageName(resource.getImageName()) ==
DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT,
resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
diff --git
a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
index b92335e..a8a872a 100644
---
a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
+++
b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
@@ -37,7 +37,16 @@ import javax.ws.rs.core.Response;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.refEq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
public class InfrastructureInfoResourceTest extends TestBase {
@@ -159,7 +168,7 @@ public class InfrastructureInfoResourceTest extends
TestBase {
@Test
public void getUserResourcesWithException() {
doThrow(new DlabException("Could not load list of provisioned
resources for user"))
-
.when(infrastructureInfoService).getUserResources(anyString());
+
.when(infrastructureInfoService).getUserResources(any(UserInfo.class));
final Response response = resources.getJerseyTest()
.target("/infrastructure/info")
.request()
@@ -169,7 +178,7 @@ public class InfrastructureInfoResourceTest extends
TestBase {
assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR,
response.getStatus());
assertEquals(MediaType.APPLICATION_JSON,
response.getHeaderString(HttpHeaders.CONTENT_TYPE));
-
verify(infrastructureInfoService).getUserResources(USER.toLowerCase());
+ verify(infrastructureInfoService).getUserResources(any());
verifyNoMoreInteractions(infrastructureInfoService);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]