This is an automated email from the ASF dual-hosted git repository. bhliva pushed a commit to branch billing_gcp in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 5a9566fba0752dd754661df633dcea15cce3fc3a Author: bhliva <[email protected]> AuthorDate: Tue Mar 12 16:03:13 2019 +0200 DLAB-23 added gcp billing + refactored current billing implementation --- .../epam/dlab/backendapi/dao/BaseBillingDAO.java | 655 +++++++++++++-------- .../dlab/backendapi/dao/aws/AwsBillingDAO.java | 191 ++---- .../dlab/backendapi/dao/azure/AzureBillingDAO.java | 205 ++----- .../dlab/backendapi/dao/gcp/GcpBillingDao.java | 146 ++--- .../backendapi/resources/dto/BillingFilter.java | 4 +- .../resources/dto/aws/AwsBillingFilter.java | 6 + .../resources/dto/azure/AzureBillingFilter.java | 12 +- .../resources/dto/gcp/GcpBillingFilter.java | 5 + .../webapp/src/dictionary/azure.dictionary.ts | 4 +- .../webapp/src/dictionary/gcp.dictionary.ts | 8 +- 10 files changed, 576 insertions(+), 660 deletions(-) diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java index 297f561..11b45ed 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java @@ -16,13 +16,22 @@ package com.epam.dlab.backendapi.dao; +import com.epam.dlab.MongoKeyWords; import com.epam.dlab.auth.UserInfo; import com.epam.dlab.backendapi.resources.dto.BillingFilter; +import com.epam.dlab.backendapi.roles.RoleType; +import com.epam.dlab.backendapi.roles.UserRoles; +import com.epam.dlab.billing.BillingCalculationUtils; +import com.epam.dlab.billing.DlabResourceType; import com.epam.dlab.dto.UserInstanceStatus; import com.epam.dlab.dto.base.DataEngineType; +import com.epam.dlab.model.aws.ReportLine; import com.google.common.collect.Lists; import com.google.inject.Inject; +import com.mongodb.client.AggregateIterable; import com.mongodb.client.FindIterable; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @@ -37,250 +46,422 @@ import java.util.function.Supplier; import static com.epam.dlab.backendapi.dao.ComputationalDAO.COMPUTATIONAL_ID; import static com.epam.dlab.backendapi.dao.ExploratoryDAO.COMPUTATIONAL_RESOURCES; import static com.epam.dlab.backendapi.dao.ExploratoryDAO.EXPLORATORY_ID; -import static com.epam.dlab.backendapi.dao.MongoCollections.BILLING; -import static com.epam.dlab.backendapi.dao.MongoCollections.USER_INSTANCES; +import static com.epam.dlab.backendapi.dao.MongoCollections.*; +import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE; +import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE; import static com.mongodb.client.model.Accumulators.sum; import static com.mongodb.client.model.Aggregates.group; import static com.mongodb.client.model.Aggregates.match; -import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.*; import static com.mongodb.client.model.Projections.*; import static java.util.Collections.singletonList; @Slf4j public abstract class BaseBillingDAO<T extends BillingFilter> extends BaseDAO implements BillingDAO<T> { - public static final String SHAPE = "shape"; - public static final String SERVICE_BASE_NAME = "service_base_name"; - public static final String ITEMS = "lines"; - public static final String COST_TOTAL = "cost_total"; - public static final String FULL_REPORT = "full_report"; - - private static final String MASTER_NODE_SHAPE = "master_node_shape"; - private static final String SLAVE_NODE_SHAPE = "slave_node_shape"; - private static final String TOTAL_INSTANCE_NUMBER = "total_instance_number"; - - private static final String DATAENGINE_SHAPE = "dataengine_instance_shape"; - private static final String DATAENGINE_INSTANCE_COUNT = "dataengine_instance_count"; - - private static final String DATAENGINE_DOCKER_IMAGE = "image"; - private static final int ONE_HUNDRED = 100; - private static final String TOTAL_FIELD_NAME = "total"; - private static final String COST_FIELD = "$cost"; - public static final String SHARED_RESOURCE_NAME = "Shared resource"; - - @Inject - protected SettingsDAO settings; - @Inject - private UserSettingsDAO userSettingsDAO; - - protected Map<String, ShapeInfo> getShapes(List<String> shapeNames) { - FindIterable<Document> userInstances = getUserInstances(); - final Map<String, ShapeInfo> shapes = new HashMap<>(); - - for (Document d : userInstances) { - getExploratoryShape(shapeNames, d) - .ifPresent(shapeInfo -> shapes.put(d.getString(EXPLORATORY_ID), shapeInfo)); - @SuppressWarnings("unchecked") - List<Document> comp = (List<Document>) d.get(COMPUTATIONAL_RESOURCES); - comp.forEach(computational -> - getComputationalShape(shapeNames, computational) - .ifPresent(shapeInfo -> shapes.put(computational.getString(COMPUTATIONAL_ID), shapeInfo))); - } - - appendSsnAndEdgeNodeType(shapeNames, shapes); - - log.trace("Loaded shapes is {}", shapes); - return shapes; - } - - @Override - public Double getTotalCost() { - return aggregateBillingData(singletonList(group(null, sum(TOTAL_FIELD_NAME, COST_FIELD)))); - } - - @Override - public Double getUserCost(String user) { - final List<Bson> pipeline = Arrays.asList(match(eq(USER, user)), - group(null, sum(TOTAL_FIELD_NAME, COST_FIELD))); - return aggregateBillingData(pipeline); - } - - @Override - public int getBillingQuoteUsed() { - return toPercentage(() -> settings.getMaxBudget(), getTotalCost()); - } - - @Override - public int getBillingUserQuoteUsed(String user) { - return toPercentage(() -> userSettingsDAO.getAllowedBudget(user), getUserCost(user)); - } - - @Override - public boolean isBillingQuoteReached() { - return getBillingQuoteUsed() >= ONE_HUNDRED; - } - - @Override - public boolean isUserQuoteReached(String user) { - final Double userCost = getUserCost(user); - return userSettingsDAO.getAllowedBudget(user) - .filter(allowedBudget -> userCost.intValue() != 0 && allowedBudget <= userCost) - .isPresent(); - } - - protected String getUserOrDefault(String user) { - return StringUtils.isNotBlank(user) ? user : SHARED_RESOURCE_NAME; - } - - private Integer toPercentage(Supplier<Optional<Integer>> allowedBudget, Double totalCost) { - return allowedBudget.get() - .map(userBudget -> (totalCost * ONE_HUNDRED) / userBudget) - .map(Double::intValue) - .orElse(BigDecimal.ZERO.intValue()); - } - - - private Optional<ShapeInfo> getComputationalShape(List<String> shapeNames, Document c) { - return isDataEngine(c.getString(DATAENGINE_DOCKER_IMAGE)) ? getDataEngineShape(shapeNames, c) : - getDataEngineServiceShape(shapeNames, c); - } - - private Double aggregateBillingData(List<Bson> pipeline) { - return Optional.ofNullable(aggregate(BILLING, pipeline).first()) - .map(d -> d.getDouble(TOTAL_FIELD_NAME)) - .orElse(BigDecimal.ZERO.doubleValue()); - } - - private FindIterable<Document> getUserInstances() { - return getCollection(USER_INSTANCES) - .find() - .projection( - fields(excludeId(), - include(SHAPE, EXPLORATORY_ID, STATUS, - COMPUTATIONAL_RESOURCES + "." + COMPUTATIONAL_ID, - COMPUTATIONAL_RESOURCES + "." + MASTER_NODE_SHAPE, - COMPUTATIONAL_RESOURCES + "." + SLAVE_NODE_SHAPE, - COMPUTATIONAL_RESOURCES + "." + TOTAL_INSTANCE_NUMBER, - COMPUTATIONAL_RESOURCES + "." + DATAENGINE_SHAPE, - COMPUTATIONAL_RESOURCES + "." + DATAENGINE_INSTANCE_COUNT, - COMPUTATIONAL_RESOURCES + "." + DATAENGINE_DOCKER_IMAGE, - COMPUTATIONAL_RESOURCES + "." + STATUS - ))); - } - - private Optional<ShapeInfo> getExploratoryShape(List<String> shapeNames, Document d) { - final String shape = d.getString(SHAPE); - if (isShapeAcceptable(shapeNames, shape)) { - return Optional.of(new ShapeInfo(shape, UserInstanceStatus.of(d.getString(STATUS)))); - } - return Optional.empty(); - } - - private boolean isDataEngine(String dockerImage) { - return DataEngineType.fromDockerImageName(dockerImage) == DataEngineType.SPARK_STANDALONE; - } - - private Optional<ShapeInfo> getDataEngineServiceShape(List<String> shapeNames, - Document c) { - final String desMasterShape = c.getString(MASTER_NODE_SHAPE); - final String desSlaveShape = c.getString(SLAVE_NODE_SHAPE); - if (isShapeAcceptable(shapeNames, desMasterShape, desSlaveShape)) { - return Optional.of(new ShapeInfo(desMasterShape, desSlaveShape, c.getString(TOTAL_INSTANCE_NUMBER), - UserInstanceStatus.of(c.getString(STATUS)))); - } - return Optional.empty(); - } - - private Optional<ShapeInfo> getDataEngineShape(List<String> shapeNames, Document c) { - final String dataEngineShape = c.getString(DATAENGINE_SHAPE); - if ((isShapeAcceptable(shapeNames, dataEngineShape)) - && StringUtils.isNotEmpty(c.getString(COMPUTATIONAL_ID))) { - - return Optional.of(new ShapeInfo(dataEngineShape, c.getString(DATAENGINE_INSTANCE_COUNT), - UserInstanceStatus.of(c.getString(STATUS)))); - } - return Optional.empty(); - } - - private boolean isShapeAcceptable(List<String> shapeNames, String... shapes) { - return shapeNames == null || shapeNames.isEmpty() || Arrays.stream(shapes).anyMatch(shapeNames::contains); - } - - protected abstract void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes); - - protected String generateShapeName(ShapeInfo shape) { - return Optional.ofNullable(shape).map(ShapeInfo::getName).orElse(StringUtils.EMPTY); - } - - protected void usersToLowerCase(List<String> users) { - if (users != null) { - users.replaceAll(u -> u != null ? u.toLowerCase() : null); - } - } - - protected void setUserFilter(UserInfo userInfo, BillingFilter filter, boolean isFullReport) { - if (isFullReport) { - usersToLowerCase(filter.getUser()); - } else { - filter.setUser(Lists.newArrayList(userInfo.getName().toLowerCase())); - } - } - - /** - * Store shape info - */ - @Getter - @ToString - protected class ShapeInfo { - private static final String DES_NAME_FORMAT = "Master: %s%sSlave: %d x %s"; - private static final String DE_NAME_FORMAT = "%d x %s"; - private final boolean isDataEngine; - private final String shape; - private final String slaveShape; - private final String slaveCount; - private final boolean isExploratory; - private final UserInstanceStatus status; - - private ShapeInfo(boolean isDataEngine, String shape, String slaveShape, String slaveCount, boolean - isExploratory, UserInstanceStatus status) { - this.isDataEngine = isDataEngine; - this.shape = shape; - this.slaveShape = slaveShape; - this.slaveCount = slaveCount; - this.isExploratory = isExploratory; - this.status = status; - } - - public ShapeInfo(String shape, UserInstanceStatus status) { - this(false, shape, null, null, true, status); - } - - ShapeInfo(String shape, String slaveShape, String slaveCount, UserInstanceStatus status) { - this(false, shape, slaveShape, slaveCount, false, status); - } - - - ShapeInfo(String shape, String slaveCount, UserInstanceStatus status) { - this(true, shape, null, slaveCount, false, status); - } - - public String getName() { - if (isExploratory) { - return shape; - } else { - return clusterName(); - } - } - - private String clusterName() { - try { - final Integer count = Integer.valueOf(slaveCount); - return isDataEngine ? String.format(DE_NAME_FORMAT, count, shape) : - String.format(DES_NAME_FORMAT, shape, System.lineSeparator(), count - 1, slaveShape); - } catch (NumberFormatException e) { - log.error("Cannot parse string {} to integer", slaveCount); - return StringUtils.EMPTY; - } - } - } + public static final String SHAPE = "shape"; + public static final String SERVICE_BASE_NAME = "service_base_name"; + public static final String ITEMS = "lines"; + public static final String COST_TOTAL = "cost_total"; + public static final String FULL_REPORT = "full_report"; + + private static final String MASTER_NODE_SHAPE = "master_node_shape"; + private static final String SLAVE_NODE_SHAPE = "slave_node_shape"; + private static final String TOTAL_INSTANCE_NUMBER = "total_instance_number"; + + private static final String DATAENGINE_SHAPE = "dataengine_instance_shape"; + private static final String DATAENGINE_INSTANCE_COUNT = "dataengine_instance_count"; + + private static final String DATAENGINE_DOCKER_IMAGE = "image"; + private static final int ONE_HUNDRED = 100; + private static final String TOTAL_FIELD_NAME = "total"; + private static final String COST_FIELD = "$cost"; + public static final String SHARED_RESOURCE_NAME = "Shared resource"; + + @Inject + protected SettingsDAO settings; + @Inject + private UserSettingsDAO userSettingsDAO; + + @Override + public Document getReport(UserInfo userInfo, T filter) { + boolean isFullReport = UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing"); + setUserFilter(userInfo, filter, isFullReport); + List<Bson> matchCriteria = matchCriteria(filter); + List<Bson> pipeline = new ArrayList<>(); + if (!matchCriteria.isEmpty()) { + pipeline.add(Aggregates.match(Filters.and(matchCriteria))); + } + pipeline.add(groupCriteria()); + pipeline.add(sortCriteria()); + final Map<String, ShapeInfo> shapes = getShapes(filter.getShapes()); + return prepareReport(filter.getStatuses(), !filter.getShapes().isEmpty(), getCollection(BILLING).aggregate(pipeline), shapes, isFullReport); //TODO add shapes + } + + private Document prepareReport(List<UserInstanceStatus> statuses, boolean filterByShape, + AggregateIterable<Document> agg, + Map<String, ShapeInfo> shapes, boolean fullReport) { + + List<Document> reportItems = new ArrayList<>(); + + Date usageDateStart = null; + Date usageDateEnd = null; + double costTotal = 0D; + + for (Document d : agg) { + Document id = (Document) d.get(MongoKeyWords.MONGO_ID); + String resourceId = id.getString(dlabIdFieldName()); + ShapeInfo shape = shapes.get(resourceId); + final UserInstanceStatus status = Optional.ofNullable(shape).map(ShapeInfo::getStatus).orElse(null); + if ((filterByShape && shape == null) || + (!statuses.isEmpty() && statuses.stream().noneMatch(s -> s.equals(status)))) { + continue; + } + + + final Date dateStart = d.getDate(usageDateFromFieldName()); + if (usageDateStart == null || dateStart.before(usageDateStart)) { + usageDateStart = dateStart; + } + Date dateEnd = d.getDate(usageDateToFieldName()); + if (usageDateEnd == null || dateEnd.after(usageDateEnd)) { + usageDateEnd = dateEnd; + } + + costTotal += d.getDouble(MongoKeyWords.COST); + + final String statusString = Optional + .ofNullable(status) + .map(UserInstanceStatus::toString) + .orElse(StringUtils.EMPTY); + Document item = new Document() + .append(MongoKeyWords.DLAB_USER, getUserOrDefault(id.getString(USER))) + .append(dlabIdFieldName(), resourceId) + .append(shapeFieldName(), generateShapeName(shape)) + .append(FIELD_RESOURCE_TYPE, DlabResourceType.getResourceTypeName(id.getString("dlab_resource_type"))) //todo check on azure!!! + .append(STATUS, + statusString) + .append(productFieldName(), id.getString(productFieldName())) + .append(MongoKeyWords.COST, d.getDouble(MongoKeyWords.COST)) + .append(costFieldName(), BillingCalculationUtils.formatDouble(d.getDouble(MongoKeyWords + .COST))) + .append(currencyCodeFieldName(), id.getString(currencyCodeFieldName())) + .append(usageDateFromFieldName(), dateStart) + .append(usageDateToFieldName(), dateEnd); + + + reportItems.add(item); + } + + return new Document() + .append(SERVICE_BASE_NAME, settings.getServiceBaseName()) + .append(usageDateFromFieldName(), usageDateStart) + .append(usageDateToFieldName(), usageDateEnd) + .append(ITEMS, reportItems) + .append(COST_TOTAL, BillingCalculationUtils.formatDouble(BillingCalculationUtils.round + (costTotal, 2))) + .append(currencyCodeFieldName(), (reportItems.isEmpty() ? null : + reportItems.get(0).getString(currencyCodeFieldName()))) + .append(FULL_REPORT, fullReport); + + } + + protected String currencyCodeFieldName() { + return "currency_code"; + } + + protected String usageDateToFieldName() { + return MongoKeyWords.USAGE_TO; + } + + protected String costFieldName() { + return MongoKeyWords.COST; + } + + protected String productFieldName() { + return ReportLine.FIELD_PRODUCT; + } + + protected String usageDateFromFieldName() { + return MongoKeyWords.USAGE_FROM; + } + + protected String dlabIdFieldName() { + return ReportLine.FIELD_DLAB_ID; + } + + protected String shapeFieldName() { + return SHAPE; + } + + protected abstract Bson sortCriteria(); + + protected abstract Bson groupCriteria(); + + protected Map<String, ShapeInfo> getShapes(List<String> shapeNames) { + FindIterable<Document> userInstances = getUserInstances(); + final Map<String, ShapeInfo> shapes = new HashMap<>(); + + for (Document d : userInstances) { + getExploratoryShape(shapeNames, d) + .ifPresent(shapeInfo -> shapes.put(d.getString(EXPLORATORY_ID), shapeInfo)); + @SuppressWarnings("unchecked") + List<Document> comp = (List<Document>) d.get(COMPUTATIONAL_RESOURCES); + comp.forEach(computational -> + getComputationalShape(shapeNames, computational) + .ifPresent(shapeInfo -> shapes.put(computational.getString(COMPUTATIONAL_ID), shapeInfo))); + } + + appendSsnAndEdgeNodeType(shapeNames, shapes); + + log.trace("Loaded shapes is {}", shapes); + return shapes; + } + + @Override + public Double getTotalCost() { + return aggregateBillingData(singletonList(group(null, sum(TOTAL_FIELD_NAME, COST_FIELD)))); + } + + @Override + public Double getUserCost(String user) { + final List<Bson> pipeline = Arrays.asList(match(eq(USER, user)), + group(null, sum(TOTAL_FIELD_NAME, COST_FIELD))); + return aggregateBillingData(pipeline); + } + + @Override + public int getBillingQuoteUsed() { + return toPercentage(() -> settings.getMaxBudget(), getTotalCost()); + } + + @Override + public int getBillingUserQuoteUsed(String user) { + return toPercentage(() -> userSettingsDAO.getAllowedBudget(user), getUserCost(user)); + } + + @Override + public boolean isBillingQuoteReached() { + return getBillingQuoteUsed() >= ONE_HUNDRED; + } + + @Override + public boolean isUserQuoteReached(String user) { + final Double userCost = getUserCost(user); + return userSettingsDAO.getAllowedBudget(user) + .filter(allowedBudget -> userCost.intValue() != 0 && allowedBudget <= userCost) + .isPresent(); + } + + protected String getUserOrDefault(String user) { + return StringUtils.isNotBlank(user) ? user : SHARED_RESOURCE_NAME; + } + + private Integer toPercentage(Supplier<Optional<Integer>> allowedBudget, Double totalCost) { + return allowedBudget.get() + .map(userBudget -> (totalCost * ONE_HUNDRED) / userBudget) + .map(Double::intValue) + .orElse(BigDecimal.ZERO.intValue()); + } + + private List<Bson> matchCriteria(BillingFilter filter) { + + List<Bson> searchCriteria = new ArrayList<>(); + + if (filter.getUser() != null && !filter.getUser().isEmpty()) { + searchCriteria.add(Filters.in(MongoKeyWords.DLAB_USER, filter.getUser())); + } + + if (filter.getResourceType() != null && !filter.getResourceType().isEmpty()) { + searchCriteria.add(Filters.in("dlab_resource_type", + DlabResourceType.getResourceTypeIds(filter.getResourceType()))); + } + + if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) { + searchCriteria.add(regex(dlabIdFieldName(), filter.getDlabId(), "i")); + } + + if (filter.getDateStart() != null && !filter.getDateStart().isEmpty()) { + searchCriteria.add(gte(MongoKeyWords.USAGE_DAY, filter.getDateStart())); + searchCriteria.add(gte(FIELD_USAGE_DATE, filter.getDateStart())); + } + if (filter.getDateEnd() != null && !filter.getDateEnd().isEmpty()) { + searchCriteria.add(lte(MongoKeyWords.USAGE_DAY, filter.getDateEnd())); + searchCriteria.add(lte(FIELD_USAGE_DATE, filter.getDateEnd())); + } + + searchCriteria.addAll(cloudMatchCriteria((T) filter)); + return searchCriteria; + } + + protected abstract List<Bson> cloudMatchCriteria(T filter); + + + private Optional<ShapeInfo> getComputationalShape(List<String> shapeNames, Document c) { + return isDataEngine(c.getString(DATAENGINE_DOCKER_IMAGE)) ? getDataEngineShape(shapeNames, c) : + getDataEngineServiceShape(shapeNames, c); + } + + private Double aggregateBillingData(List<Bson> pipeline) { + return Optional.ofNullable(aggregate(BILLING, pipeline).first()) + .map(d -> d.getDouble(TOTAL_FIELD_NAME)) + .orElse(BigDecimal.ZERO.doubleValue()); + } + + private FindIterable<Document> getUserInstances() { + return getCollection(USER_INSTANCES) + .find() + .projection( + fields(excludeId(), + include(SHAPE, EXPLORATORY_ID, STATUS, + COMPUTATIONAL_RESOURCES + "." + COMPUTATIONAL_ID, + COMPUTATIONAL_RESOURCES + "." + MASTER_NODE_SHAPE, + COMPUTATIONAL_RESOURCES + "." + SLAVE_NODE_SHAPE, + COMPUTATIONAL_RESOURCES + "." + TOTAL_INSTANCE_NUMBER, + COMPUTATIONAL_RESOURCES + "." + DATAENGINE_SHAPE, + COMPUTATIONAL_RESOURCES + "." + DATAENGINE_INSTANCE_COUNT, + COMPUTATIONAL_RESOURCES + "." + DATAENGINE_DOCKER_IMAGE, + COMPUTATIONAL_RESOURCES + "." + STATUS + ))); + } + + private Optional<ShapeInfo> getExploratoryShape(List<String> shapeNames, Document d) { + final String shape = d.getString(SHAPE); + if (isShapeAcceptable(shapeNames, shape)) { + return Optional.of(new ShapeInfo(shape, UserInstanceStatus.of(d.getString(STATUS)))); + } + return Optional.empty(); + } + + private boolean isDataEngine(String dockerImage) { + return DataEngineType.fromDockerImageName(dockerImage) == DataEngineType.SPARK_STANDALONE; + } + + private Optional<ShapeInfo> getDataEngineServiceShape(List<String> shapeNames, + Document c) { + final String desMasterShape = c.getString(MASTER_NODE_SHAPE); + final String desSlaveShape = c.getString(SLAVE_NODE_SHAPE); + if (isShapeAcceptable(shapeNames, desMasterShape, desSlaveShape)) { + return Optional.of(new ShapeInfo(desMasterShape, desSlaveShape, c.getString(TOTAL_INSTANCE_NUMBER), + UserInstanceStatus.of(c.getString(STATUS)))); + } + return Optional.empty(); + } + + private Optional<ShapeInfo> getDataEngineShape(List<String> shapeNames, Document c) { + final String dataEngineShape = c.getString(DATAENGINE_SHAPE); + if ((isShapeAcceptable(shapeNames, dataEngineShape)) + && StringUtils.isNotEmpty(c.getString(COMPUTATIONAL_ID))) { + + return Optional.of(new ShapeInfo(dataEngineShape, c.getString(DATAENGINE_INSTANCE_COUNT), + UserInstanceStatus.of(c.getString(STATUS)))); + } + return Optional.empty(); + } + + private boolean isShapeAcceptable(List<String> shapeNames, String... shapes) { + return shapeNames == null || shapeNames.isEmpty() || Arrays.stream(shapes).anyMatch(shapeNames::contains); + } + + protected void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes) { + final String ssnShape = getSsnShape(); + if (shapeNames == null || shapeNames.isEmpty() || shapeNames.contains(ssnShape)) { + String serviceBaseName = getServiceBaseName(); + shapes.put(serviceBaseName + "-ssn", new ShapeInfo(ssnShape, UserInstanceStatus.RUNNING)); + FindIterable<Document> docs = getCollection(USER_EDGE) + .find() + .projection(fields(include(ID, EDGE_STATUS))); + for (Document d : docs) { + shapes.put(edgeId(d), + new ShapeInfo(getEdgeSize(), UserInstanceStatus.of(d.getString(EDGE_STATUS)))); + } + } + } + + protected String getServiceBaseName() { + return settings.getServiceBaseName(); + } + + protected abstract String getEdgeSize(); + + protected abstract String edgeId(Document d); + + protected abstract String getSsnShape(); + + + protected String generateShapeName(ShapeInfo shape) { + return Optional.ofNullable(shape).map(ShapeInfo::getName).orElse(StringUtils.EMPTY); + } + + protected void usersToLowerCase(List<String> users) { + if (users != null) { + users.replaceAll(u -> u != null ? u.toLowerCase() : null); + } + } + + protected void setUserFilter(UserInfo userInfo, BillingFilter filter, boolean isFullReport) { + if (isFullReport) { + usersToLowerCase(filter.getUser()); + } else { + filter.setUser(Lists.newArrayList(userInfo.getName().toLowerCase())); + } + } + + /** + * Store shape info + */ + @Getter + @ToString + protected class ShapeInfo { + private static final String DES_NAME_FORMAT = "Master: %s%sSlave: %d x %s"; + private static final String DE_NAME_FORMAT = "%d x %s"; + private final boolean isDataEngine; + private final String shape; + private final String slaveShape; + private final String slaveCount; + private final boolean isExploratory; + private final UserInstanceStatus status; + + private ShapeInfo(boolean isDataEngine, String shape, String slaveShape, String slaveCount, boolean + isExploratory, UserInstanceStatus status) { + this.isDataEngine = isDataEngine; + this.shape = shape; + this.slaveShape = slaveShape; + this.slaveCount = slaveCount; + this.isExploratory = isExploratory; + this.status = status; + } + + public ShapeInfo(String shape, UserInstanceStatus status) { + this(false, shape, null, null, true, status); + } + + ShapeInfo(String shape, String slaveShape, String slaveCount, UserInstanceStatus status) { + this(false, shape, slaveShape, slaveCount, false, status); + } + + + ShapeInfo(String shape, String slaveCount, UserInstanceStatus status) { + this(true, shape, null, slaveCount, false, status); + } + + public String getName() { + if (isExploratory) { + return shape; + } else { + return clusterName(); + } + } + + private String clusterName() { + try { + final Integer count = Integer.valueOf(slaveCount); + return isDataEngine ? String.format(DE_NAME_FORMAT, count, shape) : + String.format(DES_NAME_FORMAT, shape, System.lineSeparator(), count - 1, slaveShape); + } catch (NumberFormatException e) { + log.error("Cannot parse string {} to integer", slaveCount); + return StringUtils.EMPTY; + } + } + } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java index 0e186c0..fa898b5 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/aws/AwsBillingDAO.java @@ -16,188 +16,67 @@ package com.epam.dlab.backendapi.dao.aws; -import com.epam.dlab.auth.UserInfo; import com.epam.dlab.backendapi.dao.BaseBillingDAO; import com.epam.dlab.backendapi.resources.dto.aws.AwsBillingFilter; -import com.epam.dlab.backendapi.roles.RoleType; -import com.epam.dlab.backendapi.roles.UserRoles; -import com.epam.dlab.billing.BillingCalculationUtils; -import com.epam.dlab.billing.DlabResourceType; -import com.epam.dlab.dto.UserInstanceStatus; import com.epam.dlab.util.UsernameUtils; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.FindIterable; -import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Optional; -import static com.epam.dlab.backendapi.dao.MongoCollections.BILLING; -import static com.epam.dlab.backendapi.dao.MongoCollections.USER_EDGE; import static com.epam.dlab.model.aws.ReportLine.*; import static com.mongodb.client.model.Accumulators.*; -import static com.mongodb.client.model.Aggregates.*; -import static com.mongodb.client.model.Filters.*; -import static com.mongodb.client.model.Projections.fields; -import static com.mongodb.client.model.Projections.include; +import static com.mongodb.client.model.Aggregates.group; +import static com.mongodb.client.model.Aggregates.sort; /** * DAO for user billing. */ public class AwsBillingDAO extends BaseBillingDAO<AwsBillingFilter> { - private static final Logger LOGGER = LoggerFactory.getLogger(AwsBillingDAO.class); public static final String DLAB_RESOURCE_TYPE = "dlab_resource_type"; public static final String USAGE_DATE_START = "usage_date_start"; public static final String USAGE_DATE_END = "usage_date_end"; public static final String TAG_RESOURCE_ID = "tag_resource_id"; - /** - * Add the conditions to the list. - * - * @param conditions the list of conditions. - * @param fieldName the name of field. - * @param values the values. - */ - private void addCondition(List<Bson> conditions, String fieldName, List<String> values) { - if (values != null && !values.isEmpty()) { - conditions.add(in(fieldName, values)); - } + @Override + protected Bson sortCriteria() { + return sort(new Document(ID + "." + USER, 1) + .append(ID + "." + FIELD_DLAB_ID, 1) + .append(ID + "." + DLAB_RESOURCE_TYPE, 1) + .append(ID + "." + FIELD_PRODUCT, 1)); } - /** - * Build and returns the billing report. - * - * @param userInfo user info - * @param filter the filter for report data. - * @return billing report - */ - public Document getReport(UserInfo userInfo, AwsBillingFilter filter) { - // Create filter - List<Bson> conditions = new ArrayList<>(); - boolean isFullReport = UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing"); - setUserFilter(userInfo, filter, isFullReport); - addCondition(conditions, USER, filter.getUser()); - addCondition(conditions, FIELD_PRODUCT, filter.getProduct()); - addCondition(conditions, DLAB_RESOURCE_TYPE, DlabResourceType.getResourceTypeIds(filter.getResourceType())); - - addAnotherConditionsIfNecessary(conditions, filter); - - // Create aggregation conditions - - List<Bson> pipeline = new ArrayList<>(); - if (!conditions.isEmpty()) { - LOGGER.trace("Filter conditions is {}", conditions); - pipeline.add(match(and(conditions))); - } - pipeline.add( - group(getGroupingFields(USER, FIELD_DLAB_ID, DLAB_RESOURCE_TYPE, FIELD_PRODUCT, FIELD_RESOURCE_TYPE, - FIELD_CURRENCY_CODE), - sum(FIELD_COST, "$" + FIELD_COST), - min(USAGE_DATE_START, "$" + FIELD_USAGE_DATE), - max(USAGE_DATE_END, "$" + FIELD_USAGE_DATE) - )); - pipeline.add( - sort(new Document(ID + "." + USER, 1) - .append(ID + "." + FIELD_DLAB_ID, 1) - .append(ID + "." + DLAB_RESOURCE_TYPE, 1) - .append(ID + "." + FIELD_PRODUCT, 1)) - ); - - // Get billing report and the list of shape info - AggregateIterable<Document> agg = getCollection(BILLING).aggregate(pipeline); - Map<String, ShapeInfo> shapes = getShapes(filter.getShape()); - - // Build billing report lines - List<Document> reportItems = new ArrayList<>(); - boolean filterByShape = !(filter.getShape() == null || filter.getShape().isEmpty()); - String usageDateStart = null; - String usageDateEnd = null; - double costTotal = 0; - - for (Document d : agg) { - Document id = (Document) d.get(ID); - String resourceId = id.getString(FIELD_DLAB_ID); - ShapeInfo shape = shapes.get(resourceId); - final UserInstanceStatus status = Optional.ofNullable(shape).map(ShapeInfo::getStatus).orElse(null); - if ((filterByShape && shape == null) || (!filter.getStatuses().isEmpty() && filter.getStatuses().stream() - .noneMatch(s -> s.equals(status)))) { - continue; - } - - String resourceTypeId = DlabResourceType.getResourceTypeName(id.getString(DLAB_RESOURCE_TYPE)); - String shapeName = generateShapeName(shape); - String dateStart = d.getString(USAGE_DATE_START); - if (StringUtils.compare(usageDateStart, dateStart, false) > 0) { - usageDateStart = dateStart; - } - String dateEnd = d.getString(USAGE_DATE_END); - if (StringUtils.compare(usageDateEnd, dateEnd) < 0) { - usageDateEnd = dateEnd; - } - double cost = BillingCalculationUtils.round(d.getDouble(FIELD_COST), 2); - costTotal += cost; - - Document item = new Document() - .append(FIELD_USER_ID, getUserOrDefault(id.getString(USER))) - .append(FIELD_DLAB_ID, resourceId) - .append(DLAB_RESOURCE_TYPE, resourceTypeId) - .append(SHAPE, shapeName) - .append(STATUS, - Optional.ofNullable(status).map(UserInstanceStatus::toString).orElse(StringUtils.EMPTY)) - .append(FIELD_PRODUCT, id.getString(FIELD_PRODUCT)) - .append(FIELD_RESOURCE_TYPE, id.getString(FIELD_RESOURCE_TYPE)) - .append(FIELD_COST, BillingCalculationUtils.formatDouble(cost)) - .append(FIELD_CURRENCY_CODE, id.getString(FIELD_CURRENCY_CODE)) - .append(USAGE_DATE_START, dateStart) - .append(USAGE_DATE_END, dateEnd); - reportItems.add(item); - } - - return new Document() - .append(SERVICE_BASE_NAME, settings.getServiceBaseName()) - .append(TAG_RESOURCE_ID, settings.getConfTagResourceId()) - .append(USAGE_DATE_START, usageDateStart) - .append(USAGE_DATE_END, usageDateEnd) - .append(ITEMS, reportItems) - .append(COST_TOTAL, BillingCalculationUtils.formatDouble(BillingCalculationUtils.round(costTotal, 2))) - .append(FIELD_CURRENCY_CODE, (reportItems.isEmpty() ? null : - reportItems.get(0).getString(FIELD_CURRENCY_CODE))) - .append(FULL_REPORT, isFullReport); + @Override + protected Bson groupCriteria() { + return group(getGroupingFields(USER, FIELD_DLAB_ID, DLAB_RESOURCE_TYPE, FIELD_PRODUCT, FIELD_RESOURCE_TYPE, + FIELD_CURRENCY_CODE), + sum(FIELD_COST, "$" + FIELD_COST), + min(USAGE_DATE_START, "$" + FIELD_USAGE_DATE), + max(USAGE_DATE_END, "$" + FIELD_USAGE_DATE)); } - private void addAnotherConditionsIfNecessary(List<Bson> conditions, AwsBillingFilter filter) { - if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) { - conditions.add(regex(FIELD_DLAB_ID, filter.getDlabId(), "i")); - } - - if (filter.getDateStart() != null && !filter.getDateStart().isEmpty()) { - conditions.add(gte(FIELD_USAGE_DATE, filter.getDateStart())); - } - if (filter.getDateEnd() != null && !filter.getDateEnd().isEmpty()) { - conditions.add(lte(FIELD_USAGE_DATE, filter.getDateEnd())); - } + @Override + protected List<Bson> cloudMatchCriteria(AwsBillingFilter filter) { + return Collections.emptyList(); } - protected void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes) { - // Add SSN and EDGE nodes - final String ssnShape = "t2.medium"; - if (shapeNames == null || shapeNames.isEmpty() || shapeNames.contains(ssnShape)) { - String serviceBaseName = settings.getServiceBaseName(); - shapes.put(serviceBaseName + "-ssn", new ShapeInfo(ssnShape, UserInstanceStatus.RUNNING)); - FindIterable<Document> docs = getCollection(USER_EDGE) - .find() - .projection(fields(include(ID, EDGE_STATUS))); - for (Document d : docs) { - shapes.put(String.join("-", serviceBaseName, UsernameUtils.removeDomain(d.getString(ID)), "edge"), - new ShapeInfo(ssnShape, UserInstanceStatus.of(d.getString(EDGE_STATUS)))); - } - } + @Override + protected String getEdgeSize() { + return getSsnShape(); } + + public String edgeId(Document d) { + return String.join("-", settings.getServiceBaseName(), UsernameUtils.removeDomain(d.getString(ID)), "edge"); + } + + + @Override + protected String getSsnShape() { + return "t2.medium"; + } + + + } \ No newline at end of file diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java index 401c83c..c241b06 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/azure/AzureBillingDAO.java @@ -17,156 +17,36 @@ package com.epam.dlab.backendapi.dao.azure; import com.epam.dlab.MongoKeyWords; -import com.epam.dlab.auth.UserInfo; import com.epam.dlab.backendapi.dao.BaseBillingDAO; import com.epam.dlab.backendapi.resources.dto.azure.AzureBillingFilter; -import com.epam.dlab.backendapi.roles.RoleType; -import com.epam.dlab.backendapi.roles.UserRoles; -import com.epam.dlab.billing.BillingCalculationUtils; -import com.epam.dlab.billing.DlabResourceType; -import com.epam.dlab.dto.UserInstanceStatus; import com.google.inject.Singleton; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.FindIterable; import com.mongodb.client.model.Accumulators; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Sorts; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.conversions.Bson; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static com.epam.dlab.backendapi.dao.MongoCollections.USER_EDGE; -import static com.mongodb.client.model.Filters.*; -import static com.mongodb.client.model.Projections.fields; -import static com.mongodb.client.model.Projections.include; @Singleton @Slf4j public class AzureBillingDAO extends BaseBillingDAO<AzureBillingFilter> { public static final String SIZE = "size"; - public Document getReport(UserInfo userInfo, AzureBillingFilter filter) { - - boolean isFullReport = UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing"); - setUserFilter(userInfo, filter, isFullReport); - - List<Bson> matchCriteria = matchCriteria(filter); - List<Bson> pipeline = new ArrayList<>(); - if (!matchCriteria.isEmpty()) { - pipeline.add(Aggregates.match(Filters.and(matchCriteria))); - } - pipeline.add(groupCriteria()); - pipeline.add(sortCriteria()); - - return prepareReport( - filter.getStatuses(), filter.getNodeSize() != null && !filter.getNodeSize().isEmpty(), - getCollection(MongoKeyWords.BILLING_DETAILS).aggregate(pipeline), - getShapes(filter.getNodeSize())) - .append(FULL_REPORT, isFullReport); - } - - private Document prepareReport(List<UserInstanceStatus> statuses, boolean filterByShape, - AggregateIterable<Document> agg, - Map<String, ShapeInfo> shapes) { - - List<Document> reportItems = new ArrayList<>(); - - String usageDateStart = null; - String usageDateEnd = null; - double costTotal = 0D; - - for (Document d : agg) { - Document id = (Document) d.get(MongoKeyWords.MONGO_ID); - String resourceId = id.getString(MongoKeyWords.DLAB_ID); - ShapeInfo shape = shapes.get(resourceId); - final UserInstanceStatus status = Optional.ofNullable(shape).map(ShapeInfo::getStatus).orElse(null); - if ((filterByShape && shape == null) || - (!statuses.isEmpty() && statuses.stream().noneMatch(s -> s.equals(status)))) { - continue; - } - - String dateStart = d.getString(MongoKeyWords.USAGE_FROM); - if (StringUtils.compare(usageDateStart, dateStart, false) > 0) { - usageDateStart = dateStart; - } - String dateEnd = d.getString(MongoKeyWords.USAGE_TO); - if (StringUtils.compare(usageDateEnd, dateEnd) < 0) { - usageDateEnd = dateEnd; - } - - costTotal += d.getDouble(MongoKeyWords.COST); - - Document item = new Document() - .append(MongoKeyWords.DLAB_USER, getUserOrDefault(id.getString(USER))) - .append(MongoKeyWords.DLAB_ID, resourceId) - .append(SIZE, generateShapeName(shape)) - .append(STATUS, - Optional.ofNullable(status).map(UserInstanceStatus::toString).orElse(StringUtils.EMPTY)) - .append(MongoKeyWords.METER_CATEGORY, id.getString(MongoKeyWords.METER_CATEGORY)) - .append(MongoKeyWords.RESOURCE_TYPE, - DlabResourceType.getResourceTypeName(id.getString(MongoKeyWords.RESOURCE_TYPE))) - .append(MongoKeyWords.COST, d.getDouble(MongoKeyWords.COST)) - .append(MongoKeyWords.COST_STRING, BillingCalculationUtils.formatDouble(d.getDouble(MongoKeyWords - .COST))) - .append(MongoKeyWords.CURRENCY_CODE, id.getString(MongoKeyWords.CURRENCY_CODE)) - .append(MongoKeyWords.USAGE_FROM, dateStart) - .append(MongoKeyWords.USAGE_TO, dateEnd); - - - reportItems.add(item); - } - - return new Document() - .append(SERVICE_BASE_NAME, settings.getServiceBaseName()) - .append(MongoKeyWords.USAGE_FROM, usageDateStart) - .append(MongoKeyWords.USAGE_TO, usageDateEnd) - .append(ITEMS, reportItems) - .append(MongoKeyWords.COST_STRING, BillingCalculationUtils.formatDouble(BillingCalculationUtils.round - (costTotal, 2))) - .append(MongoKeyWords.CURRENCY_CODE, (reportItems.isEmpty() ? null : - reportItems.get(0).getString(MongoKeyWords.CURRENCY_CODE))); - - } - - private List<Bson> matchCriteria(AzureBillingFilter filter) { - - List<Bson> searchCriteria = new ArrayList<>(); - - if (filter.getUser() != null && !filter.getUser().isEmpty()) { - searchCriteria.add(Filters.in(MongoKeyWords.DLAB_USER, filter.getUser())); - } - - if (filter.getCategory() != null && !filter.getCategory().isEmpty()) { - searchCriteria.add(Filters.in(MongoKeyWords.METER_CATEGORY, filter.getCategory())); - } - - if (filter.getResourceType() != null && !filter.getResourceType().isEmpty()) { - searchCriteria.add(Filters.in(MongoKeyWords.RESOURCE_TYPE, - DlabResourceType.getResourceTypeIds(filter.getResourceType()))); - } - - if (filter.getDlabId() != null && !filter.getDlabId().isEmpty()) { - searchCriteria.add(regex(MongoKeyWords.DLAB_ID, filter.getDlabId(), "i")); - } - - if (filter.getDateStart() != null && !filter.getDateStart().isEmpty()) { - searchCriteria.add(gte(MongoKeyWords.USAGE_DAY, filter.getDateStart())); - } - if (filter.getDateEnd() != null && !filter.getDateEnd().isEmpty()) { - searchCriteria.add(lte(MongoKeyWords.USAGE_DAY, filter.getDateEnd())); + @Override + protected List<Bson> cloudMatchCriteria(AzureBillingFilter filter) { + if (!filter.getCategory().isEmpty()) { + return Collections.singletonList(Filters.in(MongoKeyWords.METER_CATEGORY, filter.getCategory())); + } else { + return Collections.emptyList(); } - - return searchCriteria; } - private Bson groupCriteria() { + @Override + protected Bson groupCriteria() { return Aggregates.group(getGroupingFields( MongoKeyWords.DLAB_USER, MongoKeyWords.DLAB_ID, @@ -179,7 +59,8 @@ public class AzureBillingDAO extends BaseBillingDAO<AzureBillingFilter> { ); } - private Bson sortCriteria() { + @Override + protected Bson sortCriteria() { return Aggregates.sort(Sorts.ascending( MongoKeyWords.prependId(MongoKeyWords.DLAB_USER), MongoKeyWords.prependId(MongoKeyWords.DLAB_ID), @@ -188,25 +69,57 @@ public class AzureBillingDAO extends BaseBillingDAO<AzureBillingFilter> { } @Override - protected void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes) { + protected String getServiceBaseName() { + return settings.getServiceBaseName().replace("_", "-").toLowerCase(); + } + + @Override + protected String getEdgeSize() { + return settings.getAzureEdgeInstanceSize(); + } - String serviceBaseName = settings.getServiceBaseName().replace("_", "-").toLowerCase(); + @Override + protected String edgeId(Document d) { + return d.getString(INSTANCE_ID); + } - final String ssnSize = settings.getAzureSsnInstanceSize(); - if (shapeNames == null || shapeNames.isEmpty() || shapeNames.contains(ssnSize)) { - shapes.put(serviceBaseName + "-ssn", new BaseBillingDAO.ShapeInfo(ssnSize, UserInstanceStatus.RUNNING)); - } + @Override + protected String getSsnShape() { + return settings.getAzureSsnInstanceSize(); + } + @Override + protected String shapeFieldName() { + return SIZE; + } - final String edgeSize = settings.getAzureEdgeInstanceSize(); - if (shapeNames == null || shapeNames.isEmpty() || shapeNames.contains(edgeSize)) { - FindIterable<Document> docs = getCollection(USER_EDGE) - .find() - .projection(fields(include(INSTANCE_ID, EDGE_STATUS))); - for (Document d : docs) { - shapes.put(d.getString(INSTANCE_ID), - new BaseBillingDAO.ShapeInfo(edgeSize, UserInstanceStatus.of(d.getString(EDGE_STATUS)))); - } - } + @Override + protected String dlabIdFieldName() { + return MongoKeyWords.DLAB_ID; + } + + @Override + protected String productFieldName() { + return MongoKeyWords.METER_CATEGORY; + } + + @Override + protected String costFieldName() { + return MongoKeyWords.COST_STRING; + } + + @Override + protected String usageDateFromFieldName() { + return MongoKeyWords.USAGE_FROM; + } + + @Override + protected String usageDateToFieldName() { + return MongoKeyWords.USAGE_TO; + } + + @Override + protected String currencyCodeFieldName() { + return MongoKeyWords.CURRENCY_CODE; } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/gcp/GcpBillingDao.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/gcp/GcpBillingDao.java index 4a51272..ed7f6f6 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/gcp/GcpBillingDao.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/gcp/GcpBillingDao.java @@ -18,36 +18,42 @@ package com.epam.dlab.backendapi.dao.gcp; -import com.epam.dlab.auth.UserInfo; import com.epam.dlab.backendapi.dao.BaseBillingDAO; import com.epam.dlab.backendapi.resources.dto.gcp.GcpBillingFilter; -import com.epam.dlab.backendapi.roles.RoleType; -import com.epam.dlab.backendapi.roles.UserRoles; -import com.epam.dlab.billing.BillingCalculationUtils; -import com.epam.dlab.billing.DlabResourceType; -import com.epam.dlab.dto.UserInstanceStatus; -import com.mongodb.client.AggregateIterable; -import org.apache.commons.lang3.StringUtils; +import com.epam.dlab.util.UsernameUtils; import org.bson.Document; import org.bson.conversions.Bson; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Optional; -import static com.epam.dlab.MongoKeyWords.*; -import static com.epam.dlab.backendapi.dao.MongoCollections.BILLING; +import static com.epam.dlab.MongoKeyWords.USAGE_FROM; +import static com.epam.dlab.MongoKeyWords.USAGE_TO; import static com.epam.dlab.backendapi.dao.aws.AwsBillingDAO.DLAB_RESOURCE_TYPE; -import static com.epam.dlab.backendapi.dao.aws.AwsBillingDAO.TAG_RESOURCE_ID; import static com.epam.dlab.model.aws.ReportLine.*; import static com.mongodb.client.model.Accumulators.*; -import static com.mongodb.client.model.Aggregates.*; -import static com.mongodb.client.model.Filters.and; -import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Aggregates.group; +import static com.mongodb.client.model.Aggregates.sort; public class GcpBillingDao extends BaseBillingDAO<GcpBillingFilter> { @Override + protected Bson sortCriteria() { + return sort(new Document(ID + "." + USER, 1) + .append(ID + "." + FIELD_DLAB_ID, 1) + .append(ID + "." + FIELD_PRODUCT, 1)); + } + + @Override + protected Bson groupCriteria() { + return group(getGroupingFields(USER, FIELD_DLAB_ID, DLAB_RESOURCE_TYPE, FIELD_PRODUCT, + currencyCodeFieldName()), + sum(FIELD_COST, "$" + FIELD_COST), + min(USAGE_FROM, "$" + FIELD_USAGE_DATE), + max(USAGE_TO, "$" + FIELD_USAGE_DATE) + ); + } + + @Override public Double getTotalCost() { return null; } @@ -78,104 +84,22 @@ public class GcpBillingDao extends BaseBillingDAO<GcpBillingFilter> { } @Override - protected void appendSsnAndEdgeNodeType(List<String> shapeNames, Map<String, ShapeInfo> shapes) { - + protected List<Bson> cloudMatchCriteria(GcpBillingFilter filter) { + return Collections.emptyList(); } - public Document getReport(UserInfo userInfo, GcpBillingFilter filter) { - // Create filter - List<Bson> conditions = new ArrayList<>(); - boolean isFullReport = UserRoles.checkAccess(userInfo, RoleType.PAGE, "/api/infrastructure_provision/billing"); - setUserFilter(userInfo, filter, isFullReport); - addCondition(conditions, USER, filter.getUser()); - addCondition(conditions, FIELD_PRODUCT, filter.getProduct()); - - // Create aggregation conditions - - List<Bson> pipeline = new ArrayList<>(); - if (!conditions.isEmpty()) { - pipeline.add(match(and(conditions))); - } - pipeline.add( - group(getGroupingFields(USER, FIELD_DLAB_ID, DLAB_RESOURCE_TYPE, FIELD_PRODUCT, FIELD_RESOURCE_TYPE, - FIELD_CURRENCY_CODE), - sum(FIELD_COST, "$" + FIELD_COST), - min(USAGE_FROM, "$" + FIELD_USAGE_DATE), - max(USAGE_TO, "$" + FIELD_USAGE_DATE) - )); - pipeline.add( - sort(new Document(ID + "." + USER, 1) - .append(ID + "." + FIELD_DLAB_ID, 1) - .append(ID + "." + RESOURCE_TYPE, 1) - .append(ID + "." + FIELD_PRODUCT, 1)) - ); - - // Get billing report and the list of shape info - AggregateIterable<Document> agg = getCollection(BILLING).aggregate(pipeline); - Map<String, ShapeInfo> shapes = getShapes(filter.getShape()); - - // Build billing report lines - List<Document> reportItems = new ArrayList<>(); - boolean filterByShape = !(filter.getShape() == null || filter.getShape().isEmpty()); - String usageDateStart = null; - String usageDateEnd = null; - double costTotal = 0; - - for (Document d : agg) { - Document id = (Document) d.get(ID); - String resourceId = id.getString(FIELD_DLAB_ID); - ShapeInfo shape = shapes.get(resourceId); - final UserInstanceStatus status = Optional.ofNullable(shape).map(ShapeInfo::getStatus).orElse(null); - if ((filterByShape && shape == null) || (!filter.getStatuses().isEmpty() && filter.getStatuses().stream() - .noneMatch(s -> s.equals(status)))) { - continue; - } - - String resourceTypeId = DlabResourceType.getResourceTypeName(id.getString(DLAB_RESOURCE_TYPE)); - String shapeName = generateShapeName(shape); - String dateStart = d.getString(USAGE_FROM); - if (StringUtils.compare(usageDateStart, dateStart, false) > 0) { - usageDateStart = dateStart; - } - String dateEnd = d.getString(USAGE_TO); - if (StringUtils.compare(usageDateEnd, dateEnd) < 0) { - usageDateEnd = dateEnd; - } - double cost = BillingCalculationUtils.round(d.getDouble(FIELD_COST), 2); - costTotal += cost; - - Document item = new Document() - .append(FIELD_USER_ID, getUserOrDefault(id.getString(USER))) - .append(FIELD_DLAB_ID, resourceId) - .append(DLAB_RESOURCE_TYPE, resourceTypeId) - .append(SHAPE, shapeName) - .append(STATUS, - Optional.ofNullable(status).map(UserInstanceStatus::toString).orElse(StringUtils.EMPTY)) - .append(FIELD_PRODUCT, id.getString(FIELD_PRODUCT)) - .append(FIELD_RESOURCE_TYPE, id.getString(FIELD_RESOURCE_TYPE)) - .append(FIELD_COST, BillingCalculationUtils.formatDouble(cost)) - .append(FIELD_CURRENCY_CODE, id.getString(FIELD_CURRENCY_CODE)) - .append(USAGE_FROM, dateStart) - .append(USAGE_TO, dateEnd); - reportItems.add(item); - } - - return new Document() - .append(SERVICE_BASE_NAME, settings.getServiceBaseName()) - .append(TAG_RESOURCE_ID, settings.getConfTagResourceId()) - .append(USAGE_FROM, usageDateStart) - .append(USAGE_TO, usageDateEnd) - .append(ITEMS, reportItems) - .append(COST_TOTAL, BillingCalculationUtils.formatDouble(BillingCalculationUtils.round(costTotal, 2))) - .append(FIELD_CURRENCY_CODE, (reportItems.isEmpty() ? null : - reportItems.get(0).getString(FIELD_CURRENCY_CODE))) - .append(FULL_REPORT, isFullReport); + @Override + protected String getEdgeSize() { + return getSsnShape(); } - private void addCondition(List<Bson> conditions, String fieldName, List<String> values) { - if (values != null && !values.isEmpty()) { - conditions.add(in(fieldName, values)); - } + public String edgeId(Document d) { + return String.join("-", settings.getServiceBaseName(), UsernameUtils.removeDomain(d.getString(ID)), "edge"); } + + @Override + protected String getSsnShape() { + return "t2.medium"; + } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java index 8c65c5a..d26722b 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/BillingFilter.java @@ -8,7 +8,7 @@ import java.util.Collections; import java.util.List; @Data -public class BillingFilter { +public abstract class BillingFilter { @JsonProperty protected List<String> user; @JsonProperty("dlab_id") @@ -21,4 +21,6 @@ public class BillingFilter { protected String dateEnd; @JsonProperty("status") protected List<UserInstanceStatus> statuses = Collections.emptyList(); + + public abstract List<String> getShapes(); } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/aws/AwsBillingFilter.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/aws/AwsBillingFilter.java index 1371754..00b2017 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/aws/AwsBillingFilter.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/aws/AwsBillingFilter.java @@ -31,4 +31,10 @@ public class AwsBillingFilter extends BillingFilter { private List<String> product; @JsonProperty private List<String> shape; + + + @Override + public List<String> getShapes() { + return shape; + } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/azure/AzureBillingFilter.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/azure/AzureBillingFilter.java index d724459..0de5818 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/azure/AzureBillingFilter.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/azure/AzureBillingFilter.java @@ -21,12 +21,18 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.Collections; import java.util.List; @Data @JsonIgnoreProperties(ignoreUnknown = true) public class AzureBillingFilter extends BillingFilter { - @JsonProperty("size") - private List<String> nodeSize; - private List<String> category; + @JsonProperty("size") + private List<String> nodeSize; + private List<String> category = Collections.emptyList(); + + @Override + public List<String> getShapes() { + return nodeSize; + } } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/gcp/GcpBillingFilter.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/gcp/GcpBillingFilter.java index 2966146..3d855f2 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/gcp/GcpBillingFilter.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/gcp/GcpBillingFilter.java @@ -34,4 +34,9 @@ public class GcpBillingFilter extends BillingFilter { private List<String> shape; @JsonProperty private List<String> product; + + @Override + public List<String> getShapes() { + return shape; + } } diff --git a/services/self-service/src/main/resources/webapp/src/dictionary/azure.dictionary.ts b/services/self-service/src/main/resources/webapp/src/dictionary/azure.dictionary.ts index f040e97..00379fc 100644 --- a/services/self-service/src/main/resources/webapp/src/dictionary/azure.dictionary.ts +++ b/services/self-service/src/main/resources/webapp/src/dictionary/azure.dictionary.ts @@ -36,7 +36,7 @@ export const NAMING_CONVENTION = { 'billing': { 'resourceName': 'resourceName', 'cost': 'costString', - 'costTotal': 'costString', + 'costTotal': 'cost_total', 'currencyCode': 'currencyCode', 'dateFrom': 'from', 'dateTo': 'to', @@ -107,4 +107,4 @@ export class ReportingConfigModel { this.date_end = ''; this.dlab_id = ''; } -} \ No newline at end of file +} diff --git a/services/self-service/src/main/resources/webapp/src/dictionary/gcp.dictionary.ts b/services/self-service/src/main/resources/webapp/src/dictionary/gcp.dictionary.ts index 7c7f10d..1bf1111 100644 --- a/services/self-service/src/main/resources/webapp/src/dictionary/gcp.dictionary.ts +++ b/services/self-service/src/main/resources/webapp/src/dictionary/gcp.dictionary.ts @@ -39,12 +39,12 @@ export const NAMING_CONVENTION = { 'cost': 'cost', 'costTotal': 'cost_total', 'currencyCode': 'currency_code', - 'dateFrom': 'usage_date_start', - 'dateTo': 'usage_date_end', + 'dateFrom': 'from', + 'dateTo': 'to', 'service': 'product', 'service_filter_key': 'product', - 'type': 'resource_type', - 'resourceType': 'dlab_resource_type', + 'type': 'dlab_resource_type', + 'resourceType': 'resource_type', 'instance_size': 'shape', 'dlabId': 'dlab_id' }, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
