This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch DLAB-1571 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 72a2f1c2d1fc951d65696272651ccf17f68315c9 Author: Oleh Fuks <[email protected]> AuthorDate: Tue Feb 25 17:31:21 2020 +0200 [DLAB-1571] Convey billing for remote gcp endpoint --- services/billing-gcp/pom.xml | 5 + .../gcp/conf/BillingApplicationConfiguration.java | 2 +- .../BillingController.java} | 23 ++- .../com/epam/dlab/billing/gcp/dao/BillingDAO.java | 3 + .../billing/gcp/dao/impl/BigQueryBillingDAO.java | 58 ++++++- .../epam/dlab/billing/gcp/model/BillingData.java | 21 +-- .../dlab/billing/gcp/model/GcpBillingData.java | 7 + .../billing/gcp/repository/BillingRepository.java | 3 +- .../billing/gcp/scheduler/BillingScheduler.java | 2 +- .../dlab/billing/gcp/service/BillingService.java | 2 + .../billing/gcp/service/BillingServiceImpl.java | 45 +++-- .../epam/dlab/billing/gcp/util/BillingUtils.java | 33 ++-- .../billing-gcp/src/main/resources/application.yml | 12 +- .../com/epam/dlab/dto/billing/BillingData.java} | 22 ++- .../dlab/dto/billing/BillingResourceType.java} | 14 +- .../com/epam/dlab/rest/client/RESTService.java | 14 +- .../epam/dlab/backendapi/dao/BaseBillingDAO.java | 16 +- .../dlab/backendapi/dao/azure/AzureBillingDAO.java | 4 +- .../dlab/backendapi/domain/BillingReportDTO.java} | 47 ++---- .../backendapi/modules/CloudProviderModule.java | 3 + .../dlab/backendapi/resources/BillingResource.java | 16 +- .../backendapi/resources/dto/BillingFilter.java | 37 ++-- .../dlab/backendapi/service/BillingService.java | 2 +- .../backendapi/service/BillingServiceNew.java} | 10 +- .../backendapi/service/ExploratoryService.java | 2 + .../service/impl/BillingServiceImplNew.java | 187 +++++++++++++++++++++ .../service/impl/ExploratoryServiceImpl.java | 5 + .../epam/dlab/backendapi/util/BillingUtils.java | 95 +++++++++++ .../InfrastructureTemplateServiceBaseTest.java | 10 +- 29 files changed, 549 insertions(+), 151 deletions(-) diff --git a/services/billing-gcp/pom.xml b/services/billing-gcp/pom.xml index 114b25d..6ae8114 100644 --- a/services/billing-gcp/pom.xml +++ b/services/billing-gcp/pom.xml @@ -73,6 +73,11 @@ <version>${org.mockito.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.epam.dlab</groupId> + <artifactId>dlab-model</artifactId> + <version>${project.parent.version}</version> + </dependency> </dependencies> <build> diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java index f565c6f..5cb1ec5 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/conf/BillingApplicationConfiguration.java @@ -28,7 +28,7 @@ import org.springframework.context.annotation.Configuration; public class BillingApplicationConfiguration { - @Bean +// @Bean public BigQuery bigQueryService() { return BigQueryOptions.getDefaultInstance().getService(); } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java similarity index 53% copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java copy to services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java index 7e6b0b7..dbcfa10 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java @@ -17,13 +17,28 @@ * under the License. */ -package com.epam.dlab.billing.gcp.dao; +package com.epam.dlab.billing.gcp.controller; -import com.epam.dlab.billing.gcp.model.GcpBillingData; +import com.epam.dlab.billing.gcp.dao.BillingDAO; +import com.epam.dlab.dto.billing.BillingData; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; import java.util.List; -public interface BillingDAO { +@RestController +public class BillingController { - List<GcpBillingData> getBillingData() throws InterruptedException; + private final BillingDAO billingDAO; + + public BillingController(BillingDAO billingDAO) { + this.billingDAO = billingDAO; + } + + @GetMapping("/report") + public ResponseEntity<List<BillingData>> getBilling() { + return new ResponseEntity<>(billingDAO.getBillingReport(), HttpStatus.OK); + } } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java index 7e6b0b7..55c2c2d 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java @@ -20,10 +20,13 @@ package com.epam.dlab.billing.gcp.dao; import com.epam.dlab.billing.gcp.model.GcpBillingData; +import com.epam.dlab.dto.billing.BillingData; import java.util.List; public interface BillingDAO { List<GcpBillingData> getBillingData() throws InterruptedException; + + List<BillingData> getBillingReport(); } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java index b0ece02..990399a 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java @@ -24,27 +24,43 @@ import com.epam.dlab.billing.gcp.dao.BillingDAO; import com.epam.dlab.billing.gcp.model.BillingHistory; import com.epam.dlab.billing.gcp.model.GcpBillingData; import com.epam.dlab.billing.gcp.repository.BillingHistoryRepository; -import com.google.cloud.bigquery.*; +import com.epam.dlab.dto.billing.BillingData; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableInfo; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.GroupOperation; import org.springframework.stereotype.Component; +import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static org.springframework.data.mongodb.core.aggregation.Aggregation.group; +import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation; + @Component @Slf4j public class BigQueryBillingDAO implements BillingDAO { + private static final String DATE_FORMAT = "yyyy-MM-dd"; private static final String SBN_PARAM = "sbn"; private static final String DATASET_PARAM = "dataset"; private final BillingHistoryRepository billingHistoryRepo; + private final MongoTemplate mongoTemplate; private final String sbn; private static final String GET_BILLING_DATA_QUERY = "SELECT b.sku.description usageType," + @@ -55,15 +71,17 @@ public class BigQueryBillingDAO implements BillingDAO { "CROSS JOIN UNNEST(b.labels) as label\n" + "where label.key = 'name' and cost != 0 and label.value like @sbn\n" + "group by usageType, usage_date_from, usage_date_to, product, value, currency"; - private final BigQuery service; + private BigQuery service = null; private final String dataset; @Autowired - public BigQueryBillingDAO(DlabConfiguration conf, BigQuery service, BillingHistoryRepository billingHistoryRepo) { + public BigQueryBillingDAO(DlabConfiguration conf, BillingHistoryRepository billingHistoryRepo, + MongoTemplate mongoTemplate) { dataset = conf.getBigQueryDataset(); sbn = conf.getSbn(); - this.service = service; +// this.service = null; this.billingHistoryRepo = billingHistoryRepo; + this.mongoTemplate = mongoTemplate; } @Override @@ -82,6 +100,19 @@ public class BigQueryBillingDAO implements BillingDAO { .collect(Collectors.toList()); } + @Override + public List<BillingData> getBillingReport() { + GroupOperation groupOperation = group("product", "currency", "usageType", "dlabId") + .min("from").as("from") + .max("to").as("to") + .sum("cost").as("cost"); + Aggregation aggregation = newAggregation(groupOperation); + + return mongoTemplate.aggregate(aggregation, "billing", GcpBillingData.class).getMappedResults().stream() + .map(this::toBillingReportDTO) + .collect(Collectors.toList()); + } + private Stream<? extends GcpBillingData> bigQueryResultSetStream(Table table) { try { final String tableName = table.getTableId().getTable(); @@ -102,19 +133,32 @@ public class BigQueryBillingDAO implements BillingDAO { } private GcpBillingData toBillingData(FieldValueList fields) { - return GcpBillingData.builder() .usageDateFrom(toLocalDate(fields, "usage_date_from")) .usageDateTo(toLocalDate(fields, "usage_date_to")) - .cost(fields.get("cost").getNumericValue()) + .cost(fields.get("cost").getNumericValue().setScale(3, BigDecimal.ROUND_HALF_UP)) .product(fields.get("product").getStringValue()) .usageType(fields.get("usageType").getStringValue()) .currency(fields.get("currency").getStringValue()) - .tag(fields.get("value").getStringValue()).build(); + .tag(fields.get("value").getStringValue()) + .usageDate(toLocalDate(fields, "usage_date_from").format((DateTimeFormatter.ofPattern(DATE_FORMAT)))) + .build(); } private LocalDate toLocalDate(FieldValueList fieldValues, String timestampFieldName) { return LocalDate.from(Instant.ofEpochMilli(fieldValues.get(timestampFieldName).getTimestampValue() / 1000) .atZone(ZoneId.systemDefault())); } + + private BillingData toBillingReportDTO(GcpBillingData billingData) { + return BillingData.builder() + .usageDateFrom(billingData.getUsageDateFrom()) + .usageDateTo(billingData.getUsageDateTo()) + .product(billingData.getProduct()) + .usageType(billingData.getUsageType()) + .cost(billingData.getCost().setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue()) + .currency(billingData.getCurrency()) + .tag(billingData.getTag()) + .build(); + } } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java index 32a98ed..773d381 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java @@ -19,23 +19,24 @@ package com.epam.dlab.billing.gcp.model; +import com.epam.dlab.dto.billing.BillingResourceType; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Builder; import lombok.Data; import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; import org.springframework.data.mongodb.core.mapping.Field; import java.time.LocalDate; @Data @Builder -@Document(collection = "billing") +//@Document(collection = "billing") +@JsonIgnoreProperties(ignoreUnknown = true) public class BillingData { @Id private String id; private String user; @Field("resource_name") - private String displayName; private String resourceName; @Field("from") private LocalDate usageDateFrom; @@ -54,17 +55,5 @@ public class BillingData { @Field("dlab_id") private String dlabId; @Field("dlab_resource_type") - private ResourceType resourceType; - - - public enum ResourceType { - EDGE, - SSN, - SHARED_BUCKET, - SSN_BUCKET, - EDGE_BUCKET, - VOLUME, - EXPLORATORY, - COMPUTATIONAL - } + private BillingResourceType resourceType; } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java index a11dcce..d688198 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java @@ -21,18 +21,25 @@ package com.epam.dlab.billing.gcp.model; import lombok.Builder; import lombok.Data; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.Field; import java.math.BigDecimal; import java.time.LocalDate; @Data @Builder +@Document(collection = "billing") public class GcpBillingData { + @Field("from") private final LocalDate usageDateFrom; + @Field("to") private final LocalDate usageDateTo; private final String product; private final String usageType; private final BigDecimal cost; private final String currency; + @Field("dlabId") private final String tag; + private final String usageDate; } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java index 9dbfe98..77c362a 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/repository/BillingRepository.java @@ -20,8 +20,9 @@ package com.epam.dlab.billing.gcp.repository; import com.epam.dlab.billing.gcp.model.BillingData; +import com.epam.dlab.billing.gcp.model.GcpBillingData; import org.springframework.data.mongodb.repository.MongoRepository; -public interface BillingRepository extends MongoRepository<BillingData, String> { +public interface BillingRepository extends MongoRepository<GcpBillingData, String> { void deleteByUsageDateRegex(String usageDateRegex); } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java index 9724d43..ceaf8ce 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java @@ -37,6 +37,6 @@ public class BillingScheduler { @Scheduled(cron = "${dlab.cron}") public void getBillingReport() { - billingService.updateBillingData(); +// billingService.updateBillingData(); } } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java index 71015aa..dd69f7a 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java @@ -22,4 +22,6 @@ package com.epam.dlab.billing.gcp.service; public interface BillingService { void updateBillingData(); + + void updateBillingData2(); } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java index 2a20206..3ec4eac 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingServiceImpl.java @@ -97,22 +97,39 @@ public class BillingServiceImpl implements BillingService { .filter(bd -> bd.getDlabId() != null) .collect(Collectors.toMap(BillingData::getDlabId, b -> b)); log.info("Billable resources are: {}", billableResources); - final Map<String, List<BillingData>> billingDataMap = billingDAO.getBillingData() +// final Map<String, List<BillingData>> billingDataMap = billingDAO.getBillingData() +// .stream() +// .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag()))) +// .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0, +// USAGE_DATE_FORMAT.length()))); + +// billingDataMap.forEach((usageDate, billingDataList) -> { +// log.info("Updating billing information for month {}", usageDate); +// billingRepository.deleteByUsageDateRegex("^" + usageDate); +// billingRepository.insert(billingDataList); +// updateExploratoryCost(billingDataList); +// }); + + log.info("Finished updating billing data"); + + + } catch (Exception e) { + log.error("Can not update billing due to: {}", e.getMessage(), e); + } + } + + @Override + public void updateBillingData2() { + try { + Map<String, List<GcpBillingData>> collect = billingDAO.getBillingData() .stream() - .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag()))) - .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0, - USAGE_DATE_FORMAT.length()))); + .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0, USAGE_DATE_FORMAT.length()))); - billingDataMap.forEach((usageDate, billingDataList) -> { + collect.forEach((usageDate, billingDataList) -> { log.info("Updating billing information for month {}", usageDate); billingRepository.deleteByUsageDateRegex("^" + usageDate); billingRepository.insert(billingDataList); - updateExploratoryCost(billingDataList); }); - - log.info("Finished updating billing data"); - - } catch (Exception e) { log.error("Can not update billing due to: {}", e.getMessage(), e); } @@ -164,23 +181,17 @@ public class BillingServiceImpl implements BillingService { } - private BillingData toBillingData(GcpBillingData bd, BillingData billableResource) { + private BillingData toBillingData(GcpBillingData bd) { return BillingData.builder() - .displayName(billableResource.getDisplayName()) .cost(bd.getCost().setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()) .currency(bd.getCurrency()) .product(bd.getProduct()) - .project(billableResource.getProject()) .usageDateTo(bd.getUsageDateTo()) .usageDateFrom(bd.getUsageDateFrom()) .usageDate(bd.getUsageDateFrom().format((DateTimeFormatter.ofPattern(DATE_FORMAT)))) .usageType(bd.getUsageType()) - .user(billableResource.getUser()) - .exploratoryName(billableResource.getExploratoryName()) - .computationalName(billableResource.getComputationalName()) .dlabId(bd.getTag()) - .resourceType(billableResource.getResourceType()) .build(); } } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java index 2f26f10..6a6341e 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java +++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/util/BillingUtils.java @@ -21,6 +21,7 @@ package com.epam.dlab.billing.gcp.util; import com.epam.dlab.billing.gcp.documents.UserInstance; import com.epam.dlab.billing.gcp.model.BillingData; +import com.epam.dlab.dto.billing.BillingResourceType; import java.util.stream.Stream; @@ -40,9 +41,9 @@ public class BillingUtils { final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint); final String edgeBucketId = String.format(EDGE_BUCKET_FORMAT, sbn, project.toLowerCase()); return Stream.of( - BillingData.builder().displayName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingData.ResourceType.EDGE).build(), - BillingData.builder().displayName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingData.ResourceType.VOLUME).build(), - BillingData.builder().displayName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingData.ResourceType.EDGE_BUCKET).build() + BillingData.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingResourceType.EDGE).build(), + BillingData.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingResourceType.VOLUME).build(), + BillingData.builder().resourceName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingResourceType.EDGE_BUCKET).build() ); } @@ -50,12 +51,12 @@ public class BillingUtils { final String ssnId = sbn + "-ssn"; final String bucketName = sbn.replaceAll("_", "-"); return Stream.of( - BillingData.builder().user(SHARED_RESOURCE).displayName("SSN").dlabId(ssnId).resourceType(BillingData.ResourceType.SSN).build(), - BillingData.builder().user(SHARED_RESOURCE).displayName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingData.ResourceType.VOLUME).build(), - BillingData.builder().user(SHARED_RESOURCE).displayName("SSN bucket").dlabId(bucketName + "-ssn" + - "-bucket").resourceType(BillingData.ResourceType.SSN_BUCKET).build(), - BillingData.builder().user(SHARED_RESOURCE).displayName("Collaboration bucket").dlabId(bucketName + - "-shared-bucket").resourceType(BillingData.ResourceType.SHARED_BUCKET).build() + BillingData.builder().user(SHARED_RESOURCE).resourceName("SSN").dlabId(ssnId).resourceType(BillingResourceType.SSN).build(), + BillingData.builder().user(SHARED_RESOURCE).resourceName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingResourceType.VOLUME).build(), + BillingData.builder().user(SHARED_RESOURCE).resourceName("SSN bucket").dlabId(bucketName + "-ssn" + + "-bucket").resourceType(BillingResourceType.SSN_BUCKET).build(), + BillingData.builder().user(SHARED_RESOURCE).resourceName("Collaboration bucket").dlabId(bucketName + + "-shared-bucket").resourceType(BillingResourceType.SHARED_BUCKET).build() ); } @@ -64,15 +65,15 @@ public class BillingUtils { .stream() .filter(cr -> cr.getComputationalId() != null) .flatMap(cr -> Stream.of(computationalBillableResource(userInstance, cr), - withExploratoryName(userInstance).displayName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId())) - .resourceType(BillingData.ResourceType.VOLUME).computationalName(cr.getComputationalName()).build())); + withExploratoryName(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId())) + .resourceType(BillingResourceType.VOLUME).computationalName(cr.getComputationalName()).build())); final String exploratoryId = userInstance.getExploratoryId(); final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId); final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId); final Stream<BillingData> exploratoryStream = Stream.of( - withExploratoryName(userInstance).displayName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingData.ResourceType.EXPLORATORY).build(), - withExploratoryName(userInstance).displayName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingData.ResourceType.VOLUME).build(), - withExploratoryName(userInstance).displayName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingData.ResourceType.VOLUME).build()); + withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY).build(), + withExploratoryName(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(), + withExploratoryName(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build()); return Stream.concat(computationalStream, exploratoryStream); } @@ -80,8 +81,8 @@ public class BillingUtils { UserInstance.ComputationalResource cr) { return withExploratoryName(userInstance) .dlabId(cr.getComputationalId()) - .displayName(cr.getComputationalName()) - .resourceType(BillingData.ResourceType.COMPUTATIONAL) + .resourceName(cr.getComputationalName()) + .resourceType(BillingResourceType.COMPUTATIONAL) .computationalName(cr.getComputationalName()) .project(userInstance.getProject()) .build(); diff --git a/services/billing-gcp/src/main/resources/application.yml b/services/billing-gcp/src/main/resources/application.yml index f1f3ce6..aae4af9 100644 --- a/services/billing-gcp/src/main/resources/application.yml +++ b/services/billing-gcp/src/main/resources/application.yml @@ -3,7 +3,7 @@ spring: mongodb: username: admin password: admin - database: <MONGO_PASSWORD> + database: dlabdb port: 27017 host: localhost dlab: @@ -11,6 +11,16 @@ dlab: bigQueryDataset: <DATASET_NAME> cron: 0 * * * * * +server: + port: 8081 + servlet: + contextPath: /api/billing + +server.ssl.key-store-type: JKS +server.ssl.key-store: /Users/ofuks/keys/dlabcert/billing.jks +server.ssl.key-store-password: KEYSTORE_PASSWORD +server.ssl.key-alias: billing + logging: file: /var/opt/dlab/log/ssn/billing.log level: diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java similarity index 67% copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java copy to services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java index a11dcce..27db613 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/GcpBillingData.java +++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java @@ -17,22 +17,26 @@ * under the License. */ -package com.epam.dlab.billing.gcp.model; +package com.epam.dlab.dto.billing; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; import lombok.Data; -import java.math.BigDecimal; import java.time.LocalDate; @Data @Builder -public class GcpBillingData { - private final LocalDate usageDateFrom; - private final LocalDate usageDateTo; - private final String product; - private final String usageType; - private final BigDecimal cost; - private final String currency; +@JsonIgnoreProperties(ignoreUnknown = true) +public class BillingData { private final String tag; + @JsonProperty("from") + private LocalDate usageDateFrom; + @JsonProperty("to") + private LocalDate usageDateTo; + private String product; + private String usageType; + private Double cost; + private String currency; } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingResourceType.java similarity index 81% copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java copy to services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingResourceType.java index 71015aa..692ad84 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java +++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingResourceType.java @@ -17,9 +17,15 @@ * under the License. */ -package com.epam.dlab.billing.gcp.service; +package com.epam.dlab.dto.billing; -public interface BillingService { - - void updateBillingData(); +public enum BillingResourceType { + EDGE, + SSN, + SHARED_BUCKET, + SSN_BUCKET, + EDGE_BUCKET, + VOLUME, + EXPLORATORY, + COMPUTATIONAL } diff --git a/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java b/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java index ab1d29e..d14ce42 100644 --- a/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java +++ b/services/dlab-webapp-common/src/main/java/com/epam/dlab/rest/client/RESTService.java @@ -19,17 +19,15 @@ package com.epam.dlab.rest.client; -import com.epam.dlab.exceptions.DlabException; import lombok.extern.slf4j.Slf4j; -import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; import javax.ws.rs.client.Entity; import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.GenericType; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; -import java.net.ConnectException; import java.net.URI; import java.util.Collections; import java.util.Map; @@ -71,6 +69,16 @@ public class RESTService { return builder.get(clazz); } + public <T> T get(String path, GenericType<T> genericType) { + return get(path, null, genericType); + } + + public <T> T get(String path, String accessToken, GenericType<T> genericType) { + Invocation.Builder builder = getBuilder(path, accessToken, Collections.emptyMap()); + log.debug("REST get secured {} {}", path, accessToken); + return builder.get(genericType); + } + public <T> T post(String path, Object parameter, Class<T> clazz) { return post(path, null, parameter, clazz); } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java index 9edfaa8..71223db 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java @@ -122,8 +122,8 @@ public abstract class BaseBillingDAO extends BaseDAO implements BillingDAO { } pipeline.add(groupCriteria()); pipeline.add(sortCriteria()); - final Map<String, BaseShape> shapes = getShapes(filter.getShape()); - return prepareReport(filter.getStatuses(), !filter.getShape().isEmpty(), + final Map<String, BaseShape> shapes = getShapes(filter.getShapes()); + return prepareReport(filter.getStatuses(), !filter.getShapes().isEmpty(), getCollection(BILLING).aggregate(pipeline), shapes, isFullReport); } @@ -335,13 +335,13 @@ public abstract class BaseBillingDAO extends BaseDAO implements BillingDAO { List<Bson> searchCriteria = new ArrayList<>(); - if (filter.getUser() != null && !filter.getUser().isEmpty()) { - searchCriteria.add(Filters.in(MongoKeyWords.DLAB_USER, filter.getUser())); + if (filter.getUsers() != null && !filter.getUsers().isEmpty()) { + searchCriteria.add(Filters.in(MongoKeyWords.DLAB_USER, filter.getUsers())); } - if (filter.getResourceType() != null && !filter.getResourceType().isEmpty()) { + if (filter.getResourceTypes() != null && !filter.getResourceTypes().isEmpty()) { searchCriteria.add(Filters.in("dlab_resource_type", - DlabResourceType.getResourceTypeIds(filter.getResourceType()))); + DlabResourceType.getResourceTypeIds(filter.getResourceTypes()))); } if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) { @@ -474,9 +474,9 @@ public abstract class BaseBillingDAO extends BaseDAO implements BillingDAO { protected void setUserFilter(UserInfo userInfo, BillingFilter filter, boolean isFullReport) { if (isFullReport) { - usersToLowerCase(filter.getUser()); + usersToLowerCase(filter.getUsers()); } else { - filter.setUser(Lists.newArrayList(userInfo.getName().toLowerCase())); + filter.setUsers(Lists.newArrayList(userInfo.getName().toLowerCase())); } } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java index 04c5e6d..8eeb52c 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java @@ -42,8 +42,8 @@ public class AzureBillingDAO extends BaseBillingDAO { @Override protected List<Bson> cloudMatchCriteria(BillingFilter filter) { - if (!filter.getService().isEmpty()) { - return Collections.singletonList(Filters.in(MongoKeyWords.METER_CATEGORY, filter.getService())); + if (!filter.getProducts().isEmpty()) { + return Collections.singletonList(Filters.in(MongoKeyWords.METER_CATEGORY, filter.getProducts())); } else { return Collections.emptyList(); } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java similarity index 58% copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java copy to services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java index 32a98ed..25f1831 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/model/BillingData.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportDTO.java @@ -17,54 +17,31 @@ * under the License. */ -package com.epam.dlab.billing.gcp.model; +package com.epam.dlab.backendapi.domain; +import com.epam.dlab.dto.billing.BillingResourceType; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; import lombok.Data; -import org.springframework.data.annotation.Id; -import org.springframework.data.mongodb.core.mapping.Document; -import org.springframework.data.mongodb.core.mapping.Field; import java.time.LocalDate; @Data @Builder -@Document(collection = "billing") -public class BillingData { - @Id - private String id; - private String user; - @Field("resource_name") - private String displayName; +public class BillingReportDTO { + private String dlabId; + @JsonProperty("resource_name") private String resourceName; - @Field("from") + private String project; + private String user; + @JsonProperty("from") private LocalDate usageDateFrom; - @Field("to") + @JsonProperty("to") private LocalDate usageDateTo; - @Field("usage_date") - private String usageDate; private String product; private String usageType; private Double cost; - @Field("currency_code") private String currency; - private String project; - private String exploratoryName; - private String computationalName; - @Field("dlab_id") - private String dlabId; - @Field("dlab_resource_type") - private ResourceType resourceType; - - - public enum ResourceType { - EDGE, - SSN, - SHARED_BUCKET, - SSN_BUCKET, - EDGE_BUCKET, - VOLUME, - EXPLORATORY, - COMPUTATIONAL - } + @JsonProperty("resource_type") + private BillingResourceType resourceType; } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java index 7ea2739..0ee0d10 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/CloudProviderModule.java @@ -33,11 +33,13 @@ import com.epam.dlab.backendapi.resources.azure.ComputationalResourceAzure; import com.epam.dlab.backendapi.resources.gcp.ComputationalResourceGcp; import com.epam.dlab.backendapi.resources.gcp.GcpOauthResource; import com.epam.dlab.backendapi.service.BillingService; +import com.epam.dlab.backendapi.service.BillingServiceNew; import com.epam.dlab.backendapi.service.InfrastructureInfoService; import com.epam.dlab.backendapi.service.InfrastructureTemplateService; import com.epam.dlab.backendapi.service.aws.AwsBillingService; import com.epam.dlab.backendapi.service.azure.AzureBillingService; import com.epam.dlab.backendapi.service.gcp.GcpBillingService; +import com.epam.dlab.backendapi.service.impl.BillingServiceImplNew; import com.epam.dlab.backendapi.service.impl.InfrastructureInfoServiceImpl; import com.epam.dlab.backendapi.service.impl.InfrastructureTemplateServiceImpl; import com.epam.dlab.cloud.CloudModule; @@ -69,6 +71,7 @@ public class CloudProviderModule extends CloudModule { @Override protected void configure() { bindBilling(); + bind(BillingServiceNew.class).to(BillingServiceImplNew.class); bind(InfrastructureInfoService.class).to(InfrastructureInfoServiceImpl.class); bind(InfrastructureTemplateService.class).to(InfrastructureTemplateServiceImpl.class); bind(SchedulerConfiguration.class).toInstance( diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java index 0d27fad..26cc9a9 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BillingResource.java @@ -22,6 +22,7 @@ package com.epam.dlab.backendapi.resources; import com.epam.dlab.auth.UserInfo; import com.epam.dlab.backendapi.resources.dto.BillingFilter; import com.epam.dlab.backendapi.service.BillingService; +import com.epam.dlab.backendapi.service.BillingServiceNew; import com.google.inject.Inject; import io.dropwizard.auth.Auth; import org.bson.Document; @@ -29,6 +30,7 @@ import org.bson.Document; import javax.validation.Valid; import javax.validation.constraints.NotNull; import javax.ws.rs.Consumes; +import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -41,10 +43,12 @@ import javax.ws.rs.core.Response; public class BillingResource { private final BillingService billingService; + private final BillingServiceNew billingServiceNew; @Inject - public BillingResource(BillingService billingService) { + public BillingResource(BillingService billingService, BillingServiceNew billingServiceNew) { this.billingService = billingService; + this.billingServiceNew = billingServiceNew; } @POST @@ -54,6 +58,16 @@ public class BillingResource { return billingService.getBillingReport(userInfo, formDTO); } + @GET + @Path("/report2") + @Produces(MediaType.APPLICATION_JSON) + public Response getBillingReport2( +// @Auth UserInfo userInfo, +// @Valid @NotNull BillingFilter formDTO + ) { + return Response.ok(billingServiceNew.getBillingReport(null, null)).build(); + } + @POST @Path("/report/download") @Produces(MediaType.APPLICATION_OCTET_STREAM) diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java index f820169..52363a8 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java @@ -23,29 +23,36 @@ import com.epam.dlab.dto.UserInstanceStatus; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; import java.util.Collections; import java.util.List; @Data +@NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) public class BillingFilter { - @JsonProperty - protected List<String> user; + @NonNull + private List<String> users; + @NonNull @JsonProperty("dlab_id") - protected String dlabId; - @JsonProperty("resource_type") - protected List<String> resourceType; + private String dlabId; + @NonNull @JsonProperty("date_start") - protected String dateStart; + private String dateStart; + @NonNull @JsonProperty("date_end") - protected String dateEnd; - @JsonProperty("status") - protected List<UserInstanceStatus> statuses = Collections.emptyList(); - @JsonProperty("project") - protected List<String> projects; - @JsonProperty - private List<String> service; - @JsonProperty - private List<String> shape; + private String dateEnd; + @NonNull + @JsonProperty("resource_type") + private List<String> resourceTypes; + @NonNull + private List<UserInstanceStatus> statuses = Collections.emptyList(); + @NonNull + private List<String> projects; + @NonNull + private List<String> products; + @NonNull + private List<String> shapes; } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java index 5f79280..c16bd10 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java @@ -59,7 +59,7 @@ public abstract class BillingService { } public Document getBillingReport(UserInfo userInfo, BillingFilter filter) { - filter.getUser().replaceAll(s -> s.equalsIgnoreCase(BaseBillingDAO.SHARED_RESOURCE_NAME) ? null : s); + filter.getUsers().replaceAll(s -> s.equalsIgnoreCase(BaseBillingDAO.SHARED_RESOURCE_NAME) ? null : s); return getReport(userInfo, filter); } diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java similarity index 72% copy from services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java copy to services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java index 7e6b0b7..b486a23 100644 --- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingServiceNew.java @@ -17,13 +17,15 @@ * under the License. */ -package com.epam.dlab.billing.gcp.dao; +package com.epam.dlab.backendapi.service; -import com.epam.dlab.billing.gcp.model.GcpBillingData; +import com.epam.dlab.auth.UserInfo; +import com.epam.dlab.backendapi.domain.BillingReportDTO; +import com.epam.dlab.backendapi.resources.dto.BillingFilter; import java.util.List; -public interface BillingDAO { +public interface BillingServiceNew { - List<GcpBillingData> getBillingData() throws InterruptedException; + List<BillingReportDTO> getBillingReport(UserInfo userInfo, BillingFilter filter); } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java index 4348819..f744bc5 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java @@ -50,6 +50,8 @@ public interface ExploratoryService { List<UserInstanceDTO> getInstancesWithStatuses(String user, UserInstanceStatus exploratoryStatus, UserInstanceStatus computationalStatus); + List<UserInstanceDTO> findAll(); + void updateClusterConfig(UserInfo userInfo, String exploratoryName, List<ClusterConfig> config); Optional<UserInstanceDTO> getUserInstance(String user, String exploratoryName); diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java new file mode 100644 index 0000000..733f169 --- /dev/null +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImplNew.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.backendapi.service.impl; + +import com.epam.dlab.auth.UserInfo; +import com.epam.dlab.backendapi.conf.SelfServiceApplicationConfiguration; +import com.epam.dlab.backendapi.domain.BillingReportDTO; +import com.epam.dlab.backendapi.domain.EndpointDTO; +import com.epam.dlab.backendapi.domain.ProjectDTO; +import com.epam.dlab.backendapi.domain.ProjectEndpointDTO; +import com.epam.dlab.backendapi.resources.dto.BillingFilter; +import com.epam.dlab.backendapi.service.BillingServiceNew; +import com.epam.dlab.backendapi.service.EndpointService; +import com.epam.dlab.backendapi.service.ExploratoryService; +import com.epam.dlab.backendapi.service.ProjectService; +import com.epam.dlab.backendapi.util.BillingUtils; +import com.epam.dlab.constants.ServiceConsts; +import com.epam.dlab.dto.billing.BillingData; +import com.epam.dlab.exceptions.DlabException; +import com.epam.dlab.rest.client.RESTService; +import com.google.inject.Inject; +import com.google.inject.name.Named; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.utils.URIBuilder; + +import javax.ws.rs.core.GenericType; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class BillingServiceImplNew implements BillingServiceNew { + private final ProjectService projectService; + private final EndpointService endpointService; + private final ExploratoryService exploratoryService; + private final SelfServiceApplicationConfiguration configuration; + private final RESTService provisioningService; + + @Inject + public BillingServiceImplNew(ProjectService projectService, EndpointService endpointService, + ExploratoryService exploratoryService, SelfServiceApplicationConfiguration configuration, + @Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService) { + this.projectService = projectService; + this.endpointService = endpointService; + this.exploratoryService = exploratoryService; + this.configuration = configuration; + this.provisioningService = provisioningService; + } + + @Override + public List<BillingReportDTO> getBillingReport(UserInfo userInfo, BillingFilter filter) { +// filter.getUser().replaceAll(s -> s.equalsIgnoreCase(BaseBillingDAO.SHARED_RESOURCE_NAME) ? null : s); // tell front end not to pass SHARED_RESOURCE_NAME and remove this line + final String serviceBaseName = configuration.getServiceBaseName(); + final Stream<BillingReportDTO> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(serviceBaseName); + final Stream<BillingReportDTO> billableUserInstances = exploratoryService.findAll() + .stream() + .filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId())) + .flatMap(BillingUtils::exploratoryBillingDataStream); + + final Stream<BillingReportDTO> billableEdges = projectService.getProjects() + .stream() + .collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints)) + .entrySet() + .stream() + .flatMap(e -> projectEdges(serviceBaseName, e.getKey(), e.getValue())); + + final Map<String, BillingReportDTO> billableResources = Stream.of(billableUserInstances, billableEdges, ssnBillingDataStream) + .flatMap(s -> s) + .filter(bd -> Objects.nonNull(bd.getDlabId())) + .collect(Collectors.toMap(BillingReportDTO::getDlabId, b -> b)); + log.debug("Billable resources are: {}", billableResources); + + List<BillingReportDTO> billingReport = getRemoteBillingData() + .stream() + .map(bd -> toBillingData(bd, getOrDefault(billableResources, bd.getTag()))) + .collect(Collectors.toList()); + log.debug("Billing report: {}", billingReport); + + return billingReport; + } + + private Stream<BillingReportDTO> projectEdges(String serviceBaseName, String projectName, List<ProjectEndpointDTO> endpoints) { + return endpoints + .stream() + .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName())); + } + + private BillingReportDTO getOrDefault(Map<String, BillingReportDTO> billableResources, String tag) { + return billableResources.getOrDefault(tag, BillingReportDTO.builder().dlabId(tag).build()); + } + + private List<BillingData> getRemoteBillingData() { + List<EndpointDTO> endpoints = endpointService.getEndpoints(); + ExecutorService executor = Executors.newFixedThreadPool(endpoints.size()); + List<Callable<List<BillingData>>> callableTasks = new ArrayList<>(); + endpoints.forEach(e -> + callableTasks.add(getTask(getBillingUrl(e.getUrl())))); + + List<BillingData> billingData; + try { + log.debug("Trying to retrieve billing info for {}", endpoints); + billingData = executor.invokeAll(callableTasks) + .stream() + .map(this::getBillingReportDTOS) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } catch (Exception e) { + executor.shutdown(); + log.error("Cannot retrieve billing information {}", e.getMessage(), e); + throw new DlabException("Cannot retrieve billing information"); + } + executor.shutdown(); + return billingData; + } + + private List<BillingData> getBillingReportDTOS(Future<List<BillingData>> s) { + try { + return s.get(); + } catch (InterruptedException | ExecutionException e) { + log.error("Cannot retrieve billing information {}", e.getMessage(), e); + throw new DlabException("Cannot retrieve billing information"); + } + } + + private String getBillingUrl(String endpointUrl) { + URI uri; + try { + uri = new URI(endpointUrl); + } catch (URISyntaxException e) { + log.error("Wrong URI syntax {}", e.getMessage(), e); + throw new DlabException("Wrong URI syntax"); + } + return new URIBuilder() + .setScheme(uri.getScheme()) + .setHost(uri.getHost()) + .setPort(8081) + .setPath("/api/billing/report") + .toString(); + } + + private Callable<List<BillingData>> getTask(String url) { + return () -> provisioningService.get(url, new GenericType<List<BillingData>>() { + }); + } + + private BillingReportDTO toBillingData(BillingData billingData, BillingReportDTO billingReportDTO) { + return BillingReportDTO.builder() + .currency(billingData.getCurrency()) + .product(billingData.getProduct()) + .project(billingReportDTO.getProject()) + .usageDateTo(billingData.getUsageDateTo()) + .usageDateFrom(billingData.getUsageDateFrom()) + .usageType(billingData.getUsageType()) + .user(billingReportDTO.getUser()) + .dlabId(billingData.getTag()) + .resourceType(billingReportDTO.getResourceType()) + .build(); + } +} diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java index 17b8967..59f9adc 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java @@ -192,6 +192,11 @@ public class ExploratoryServiceImpl implements ExploratoryService { } @Override + public List<UserInstanceDTO> findAll() { + return exploratoryDAO.getInstances(); + } + + @Override public void updateClusterConfig(UserInfo userInfo, String exploratoryName, List<ClusterConfig> config) { final String userName = userInfo.getName(); final String token = userInfo.getAccessToken(); diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java new file mode 100644 index 0000000..6fbb94c --- /dev/null +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.backendapi.util; + +import com.epam.dlab.backendapi.domain.BillingReportDTO; +import com.epam.dlab.dto.UserInstanceDTO; +import com.epam.dlab.dto.billing.BillingResourceType; +import com.epam.dlab.dto.computational.UserComputationalResource; + +import java.util.stream.Stream; + +public class BillingUtils { + + private static final String EDGE_FORMAT = "%s-%s-%s-edge"; + private static final String EDGE_VOLUME_FORMAT = "%s-%s-%s-edge-volume-primary"; + private static final String EDGE_BUCKET_FORMAT = "%s-%s-bucket"; + private static final String VOLUME_PRIMARY_FORMAT = "%s-volume-primary"; + private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary"; + private static final String VOLUME_PRIMARY = "Volume primary"; + private static final String VOLUME_SECONDARY = "Volume secondary"; + private static final String SHARED_RESOURCE = "Shared resource"; + + public static Stream<BillingReportDTO> edgeBillingDataStream(String project, String sbn, String endpoint) { + final String userEdgeId = String.format(EDGE_FORMAT, sbn, project.toLowerCase(), endpoint); + final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint); + final String edgeBucketId = String.format(EDGE_BUCKET_FORMAT, sbn, project.toLowerCase()); + return Stream.of( + BillingReportDTO.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(BillingResourceType.EDGE).build(), + BillingReportDTO.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(BillingResourceType.VOLUME).build(), + BillingReportDTO.builder().resourceName("EDGE bucket").user(SHARED_RESOURCE).project(project).dlabId(edgeBucketId).resourceType(BillingResourceType.EDGE_BUCKET).build() + ); + } + + public static Stream<BillingReportDTO> ssnBillingDataStream(String sbn) { + final String ssnId = sbn + "-ssn"; + final String bucketName = sbn.replaceAll("_", "-"); + return Stream.of( + BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("SSN").dlabId(ssnId).resourceType(BillingResourceType.SSN).build(), + BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("SSN Volume").dlabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(BillingResourceType.VOLUME).build(), + BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("SSN bucket").dlabId(bucketName + "-ssn" + + "-bucket").resourceType(BillingResourceType.SSN_BUCKET).build(), + BillingReportDTO.builder().user(SHARED_RESOURCE).resourceName("Collaboration bucket").dlabId(bucketName + + "-shared-bucket").resourceType(BillingResourceType.SHARED_BUCKET).build() + ); + } + + public static Stream<BillingReportDTO> exploratoryBillingDataStream(UserInstanceDTO userInstance) { + final Stream<BillingReportDTO> computationalStream = userInstance.getResources() + .stream() + .filter(cr -> cr.getComputationalId() != null) + .flatMap(cr -> Stream.of(computationalBillableResource(userInstance, cr), + withExploratoryName(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_FORMAT, cr.getComputationalId())) + .resourceType(BillingResourceType.VOLUME).build())); + final String exploratoryId = userInstance.getExploratoryId(); + final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId); + final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId); + final Stream<BillingReportDTO> exploratoryStream = Stream.of( + withExploratoryName(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(BillingResourceType.EXPLORATORY).build(), + withExploratoryName(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(BillingResourceType.VOLUME).build(), + withExploratoryName(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(BillingResourceType.VOLUME).build()); + return Stream.concat(computationalStream, exploratoryStream); + } + + private static BillingReportDTO computationalBillableResource(UserInstanceDTO userInstance, + UserComputationalResource cr) { + return withExploratoryName(userInstance) + .dlabId(cr.getComputationalId()) + .resourceName(cr.getComputationalName()) + .resourceType(BillingResourceType.COMPUTATIONAL) + .project(userInstance.getProject()) + .build(); + } + + private static BillingReportDTO.BillingReportDTOBuilder withExploratoryName(UserInstanceDTO userInstance) { + return BillingReportDTO.builder().user(userInstance.getUser()) + .project(userInstance.getProject()); + } +} diff --git a/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java b/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java index e53b78c..56c838a 100644 --- a/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java +++ b/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/InfrastructureTemplateServiceBaseTest.java @@ -89,7 +89,7 @@ public class InfrastructureTemplateServiceBaseTest { emDto2.setExploratoryEnvironmentShapes(shapes2); List<ExploratoryMetadataDTO> expectedEmdDtoList = Arrays.asList(emDto1, emDto2); when(userGroupDao.getUserGroups(anyString())).thenReturn(Collections.emptySet()); - when(provisioningService.get(anyString(), anyString(), any())).thenReturn(expectedEmdDtoList.toArray()); + when(provisioningService.get(anyString(), anyString(), any(Class.class))).thenReturn(expectedEmdDtoList.toArray()); when(settingsDAO.getConfOsFamily()).thenReturn("someConfOsFamily"); UserInfo userInfo = new UserInfo("test", "token"); @@ -108,7 +108,7 @@ public class InfrastructureTemplateServiceBaseTest { public void getExploratoryTemplatesWithException() { when(endpointService.get(anyString())).thenReturn(endpointDTO()); doThrow(new DlabException("Could not load list of exploratory templates for user")) - .when(provisioningService).get(anyString(), anyString(), any()); + .when(provisioningService).get(anyString(), anyString(), any(Class.class)); UserInfo userInfo = new UserInfo("test", "token"); try { @@ -131,7 +131,7 @@ public class InfrastructureTemplateServiceBaseTest { ); when(projectDAO.get(anyString())).thenReturn(Optional.of(new ProjectDTO("project", Collections.emptySet(), null, null, null, null, true))); - when(provisioningService.get(anyString(), anyString(), any())).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{})); + when(provisioningService.get(anyString(), anyString(), any(Class.class))).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{})); List<FullComputationalTemplate> expectedFullCmdDtoList = expectedCmdDtoList.stream() .map(e -> infrastructureTemplateServiceBaseChild.getCloudFullComputationalTemplate(e)) @@ -154,7 +154,7 @@ public class InfrastructureTemplateServiceBaseTest { public void getComputationalTemplatesWhenMethodThrowsException() { when(endpointService.get(anyString())).thenReturn(endpointDTO()); doThrow(new DlabException("Could not load list of computational templates for user")) - .when(provisioningService).get(anyString(), anyString(), any()); + .when(provisioningService).get(anyString(), anyString(), any(Class.class)); UserInfo userInfo = new UserInfo("test", "token"); try { @@ -173,7 +173,7 @@ public class InfrastructureTemplateServiceBaseTest { final ComputationalMetadataDTO computationalMetadataDTO = new ComputationalMetadataDTO("dataengine-service"); computationalMetadataDTO.setComputationResourceShapes(Collections.emptyMap()); List<ComputationalMetadataDTO> expectedCmdDtoList = Collections.singletonList(computationalMetadataDTO); - when(provisioningService.get(anyString(), anyString(), any())).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{})); + when(provisioningService.get(anyString(), anyString(), any(Class.class))).thenReturn(expectedCmdDtoList.toArray(new ComputationalMetadataDTO[]{})); when(projectDAO.get(anyString())).thenReturn(Optional.of(new ProjectDTO("project", Collections.emptySet(), null, null, null, null, true))); when(configuration.getMinEmrInstanceCount()).thenReturn(1); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
