This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch DLAB-1571 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 3ba048723faf6bb0f3529ddda3dfc0526a54c98b Author: Oleh Fuks <[email protected]> AuthorDate: Tue Mar 17 15:07:09 2020 +0200 Added more changes to Azure remote billing --- .../azure/AzureBillableResourcesService.java | 315 --------------------- .../billing/azure/AzureBillingDetailsService.java | 194 ------------- .../azure/AzureInvoiceCalculationService.java | 60 ++-- .../dlab/billing/azure/BillingSchedulerAzure.java | 26 +- .../azure/model/AzureDailyResourceInvoice.java | 34 +-- 5 files changed, 23 insertions(+), 606 deletions(-) diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java deleted file mode 100644 index c4ee5b4..0000000 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.epam.dlab.billing.azure; - -import com.epam.dlab.MongoKeyWords; -import com.epam.dlab.billing.DlabResourceType; -import com.epam.dlab.billing.azure.model.AzureDlabBillableResource; -import com.epam.dlab.dto.UserInstanceDTO; -import com.epam.dlab.dto.azure.edge.EdgeInfoAzure; -import com.epam.dlab.dto.computational.UserComputationalResource; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; -import com.mongodb.client.FindIterable; -import com.mongodb.client.model.Filters; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; - -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import static com.mongodb.client.model.Projections.exclude; -import static com.mongodb.client.model.Projections.fields; - -/** - * Helps to retrieve billable resources that are created in scope of DLab usage. Uses MongoDB as data source - * for created resources - */ -@Slf4j -public class AzureBillableResourcesService { - private static final String SHARED_RESOURCE = "Shared resource"; - private static final String[] USER_INSTANCES_EXCLUDED_FIELDS = {"scheduler_data", "last_activity", - "computational_resources.scheduler_data", "computational_resources.last_activity"}; - private final ObjectMapper objectMapper = new ObjectMapper(); - - private MongoDbBillingClient mongoDbBillingClient; - private String serviceBaseName; - private String sharedStorageAccountTagName; - private String ssnStorageAccountTagName; - private String azureDataLakeTagName; - - /** - * Constructs the service class - * - * @param mongoDbBillingClient mongodb client to retrieve all billable resources - * @param sharedStorageAccountTagName shared storage account tag name - * @param ssnStorageAccountTagName ssn storage account tag name - * @param azureDataLakeTagName azure DataLake tag name - */ - public AzureBillableResourcesService(MongoDbBillingClient mongoDbBillingClient, String sharedStorageAccountTagName, - String ssnStorageAccountTagName, String azureDataLakeTagName) { - this.mongoDbBillingClient = mongoDbBillingClient; - - this.serviceBaseName = getConfigurationSettingValue(MongoKeyWords.SERVICE_BASE_NAME_KEY) - .replace('_', '-').toLowerCase(); - - this.sharedStorageAccountTagName = sharedStorageAccountTagName; - this.ssnStorageAccountTagName = ssnStorageAccountTagName; - this.azureDataLakeTagName = azureDataLakeTagName; - } - - - /** - * Collects billable resources - * - * @return set of all billable resources that were created in scope by DLab from its installation to current time - */ - public Set<AzureDlabBillableResource> getBillableResources() { - - Set<AzureDlabBillableResource> billableResources = new HashSet<>(); - - billableResources.addAll(getSsn()); - billableResources.addAll(getDataLake()); - billableResources.addAll(getEdgeAndStorageAccount()); - billableResources.addAll(getNotebooksAndClusters()); - - List<AzureDlabBillableResource> list = new ArrayList<>(billableResources); - list.sort(Comparator.comparing(AzureDlabBillableResource::getId)); - - try { - log.debug("Billable resources is \n {}", objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString - (list)); - } catch (JsonProcessingException e) { - log.debug("Error during pretty printing. Show simple list", e); - log.debug("Billable resources is {}", list); - } - - return billableResources; - } - - private String getConfigurationSettingValue(String key) { - - Document document = mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.SETTINGS_COLLECTION) - .find(Filters.eq(MongoKeyWords.MONGO_ID, key)).first(); - - if (document != null) { - String value = document.getString("value"); - if (StringUtils.isEmpty(value)) { - throw new IllegalStateException("Configuration " + key + " does not have value in settings"); - } - log.info("Key {} has value {}", key, value); - return value; - } else { - throw new IllegalStateException("Configuration " + key + " is not present in settings"); - } - - } - - private String getConfigurationSettingValueOrEmpty(String key) { - try { - return getConfigurationSettingValue(key); - } catch (IllegalStateException e) { - log.warn("key {} is not found", key, e); - return null; - } - } - - private Set<AzureDlabBillableResource> getSsn() { - - return Sets.newHashSet( - AzureDlabBillableResource.builder().id(serviceBaseName + "-ssn").type(DlabResourceType.SSN).build(), - AzureDlabBillableResource.builder().id(ssnStorageAccountTagName).type(DlabResourceType - .SSN_STORAGE_ACCOUNT).build(), - AzureDlabBillableResource.builder().id(sharedStorageAccountTagName).type(DlabResourceType - .COLLABORATION_STORAGE_ACCOUNT).build(), - AzureDlabBillableResource.builder().id(serviceBaseName + "-ssn-volume-primary") - .type(DlabResourceType.VOLUME).build() - ); - } - - private Set<AzureDlabBillableResource> getDataLake() { - - if (azureDataLakeTagName != null) { - return Sets.newHashSet(AzureDlabBillableResource.builder().id(azureDataLakeTagName) - .type(DlabResourceType.DATA_LAKE_STORE).build()); - } - - return Sets.newHashSet(); - } - - private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() { - Map<String, List<Document>> projectEndpoints = StreamSupport.stream(mongoDbBillingClient.getDatabase() - .getCollection("Projects").find().spliterator(), false) - .collect(Collectors.toMap(key -> key.getString("name").toLowerCase(), - value -> (List<Document>) value.get("endpoints"))); - - return projectEndpoints.entrySet() - .stream() - .flatMap(projectEndpoint -> getEdgeAndStoragePerProject(projectEndpoint.getKey(), projectEndpoint.getValue())) - .collect(Collectors.toSet()); - } - - private Stream<AzureDlabBillableResource> getEdgeAndStoragePerProject(String projectName, List<Document> endpoints) { - return endpoints - .stream() - .flatMap(endpoint -> { - try { - return getEdgeAndStorageAccount(projectName, objectMapper.readValue( - objectMapper.writeValueAsString(endpoint.get("edgeInfo")), - new com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() { - })).stream(); - } catch (IOException e) { - log.error("Error during preparation of billable resources", e); - } - return Stream.empty(); - }); - } - - private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(String projectName, EdgeInfoAzure edgeInfoAzure) { - Set<AzureDlabBillableResource> billableResources = new HashSet<>(); - - if (StringUtils.isNotEmpty(edgeInfoAzure.getUserContainerName())) { - billableResources.add(AzureDlabBillableResource.builder() - .id(edgeInfoAzure.getUserStorageAccountTagName()) - .type(DlabResourceType.EDGE_STORAGE_ACCOUNT) - .user(SHARED_RESOURCE) - .project(projectName) - .build()); - } - - if (StringUtils.isNotEmpty(edgeInfoAzure.getInstanceId())) { - billableResources.add(AzureDlabBillableResource.builder() - .id(edgeInfoAzure.getInstanceId()) - .type(DlabResourceType.EDGE) - .user(SHARED_RESOURCE) - .project(projectName) - .build()); - - billableResources.add(AzureDlabBillableResource.builder() - .id(edgeInfoAzure.getInstanceId() + "-volume-primary") - .type(DlabResourceType.VOLUME) - .user(SHARED_RESOURCE) - .project(projectName) - .build()); - } - - return billableResources; - } - - private Set<AzureDlabBillableResource> getNotebooksAndClusters() { - - Set<AzureDlabBillableResource> billableResources = new HashSet<>(); - - try { - final FindIterable<Document> userInstanceDocuments = mongoDbBillingClient.getDatabase() - .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION) - .find() - .projection(fields(exclude(USER_INSTANCES_EXCLUDED_FIELDS))); - List<UserInstanceDTO> userInstanceDTOS = objectMapper.readValue( - objectMapper.writeValueAsString(userInstanceDocuments), - new com.fasterxml.jackson.core.type.TypeReference<List<UserInstanceDTO>>() { - }); - - if (userInstanceDTOS != null && !userInstanceDTOS.isEmpty()) { - userInstanceDTOS.forEach(e -> billableResources.addAll(getNotebookAndClusters(e))); - } - - } catch (IOException e) { - log.error("Error during preparation of billable resources", e); - } - - return billableResources; - } - - private Set<AzureDlabBillableResource> getNotebookAndClusters(UserInstanceDTO userInstanceDTO) { - Set<AzureDlabBillableResource> notebookResources = new HashSet<>(); - - if (StringUtils.isNotEmpty(userInstanceDTO.getExploratoryId())) { - notebookResources.add(AzureDlabBillableResource.builder() - .id(userInstanceDTO.getExploratoryId()) - .type(DlabResourceType.EXPLORATORY) - .user(userInstanceDTO.getUser()) - .project(userInstanceDTO.getProject()) - .notebookId(userInstanceDTO.getExploratoryId()) - .resourceName(userInstanceDTO.getExploratoryName()) - .build()); - notebookResources.addAll(getVolumes(userInstanceDTO, userInstanceDTO.getExploratoryId(), "Volume primary", - "Volume secondary")); - - if (userInstanceDTO.getResources() != null && !userInstanceDTO.getResources().isEmpty()) { - for (UserComputationalResource userComputationalResource : userInstanceDTO.getResources()) { - if (StringUtils.isNotEmpty(userComputationalResource.getComputationalId())) { - - notebookResources.add(AzureDlabBillableResource.builder() - .id(userComputationalResource.getComputationalId()) - .type(DlabResourceType.COMPUTATIONAL) - .user(userInstanceDTO.getUser()) - .project(userInstanceDTO.getProject()) - .notebookId(userInstanceDTO.getExploratoryId()) - .resourceName(userComputationalResource.getComputationalName()) - .build()); - final List<AzureDlabBillableResource> volumes = getVolumes(userInstanceDTO, - userComputationalResource.getComputationalId(), - userComputationalResource.getComputationalName() + " volume primary", - userComputationalResource.getComputationalName() + " volume secondary"); - notebookResources.addAll(volumes); - - } else { - log.error("Computational with empty id {} is found in notebook {}. Skip it.", - userComputationalResource, userInstanceDTO); - } - } - } - - } else { - log.error("Notebook {} with empty id id found. Skip it.", userInstanceDTO); - } - - return notebookResources; - } - - private List<AzureDlabBillableResource> getVolumes(UserInstanceDTO userInstanceDTO, String exploratoryId, String - primaryVolumeName, String secondaryVolumeName) { - - return Arrays.asList( - AzureDlabBillableResource.builder() - .id(exploratoryId + "-volume-primary") - .type(DlabResourceType.VOLUME) - .user(userInstanceDTO.getUser()) - .project(userInstanceDTO.getProject()) - .notebookId(userInstanceDTO.getExploratoryId()) - .resourceName(primaryVolumeName) - .build(), - AzureDlabBillableResource.builder() - .id(exploratoryId + "-volume-secondary") - .type(DlabResourceType.VOLUME) - .user(userInstanceDTO.getUser()) - .project(userInstanceDTO.getProject()) - .notebookId(userInstanceDTO.getExploratoryId()) - .resourceName(secondaryVolumeName) - .build() - ); - } -} diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java deleted file mode 100644 index 6d84eb8..0000000 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.epam.dlab.billing.azure; - -import com.epam.dlab.MongoKeyWords; -import com.epam.dlab.billing.BillingCalculationUtils; -import com.epam.dlab.billing.DlabResourceType; -import com.google.common.collect.Lists; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.model.*; -import com.mongodb.client.result.UpdateResult; -import lombok.extern.slf4j.Slf4j; -import org.bson.Document; -import org.bson.conversions.Bson; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.stream.StreamSupport; - -@Slf4j -public class AzureBillingDetailsService { - private MongoDbBillingClient mongoDbBillingClient; - private String currencyCode; - - public AzureBillingDetailsService(MongoDbBillingClient mongoDbBillingClient, String currencyCode) { - this.mongoDbBillingClient = mongoDbBillingClient; - this.currencyCode = currencyCode; - } - - public void updateBillingDetails() { - StreamSupport.stream(mongoDbBillingClient.getDatabase() - .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION).find().spliterator(), false) - .map(a -> a.getString(MongoKeyWords.DLAB_USER)) - .forEach(this::updateBillingDetails); - } - - public void updateBillingDetails(String user) { - log.debug("Updating billing details for user {}", user); - - try { - AggregateIterable<Document> aggregateIterable = mongoDbBillingClient.getDatabase() - .getCollection(MongoKeyWords.BILLING_DETAILS) - .aggregate(Lists.newArrayList( - Aggregates.match( - Filters.and( - Filters.eq(MongoKeyWords.DLAB_USER, user), - Filters.in(MongoKeyWords.RESOURCE_TYPE, - DlabResourceType.EXPLORATORY.toString(), - DlabResourceType.COMPUTATIONAL.toString(), - DlabResourceType.VOLUME.toString()) - ) - ), - - Aggregates.group(getGroupingFields( - MongoKeyWords.DLAB_ID, - MongoKeyWords.DLAB_USER, - MongoKeyWords.EXPLORATORY_ID, - MongoKeyWords.RESOURCE_TYPE, - MongoKeyWords.RESOURCE_NAME, - MongoKeyWords.COMPUTATIONAL_ID, - MongoKeyWords.METER_CATEGORY), - Accumulators.sum(MongoKeyWords.COST, MongoKeyWords.prepend$(MongoKeyWords.COST)), - Accumulators.min(MongoKeyWords.USAGE_FROM, MongoKeyWords.prepend$(MongoKeyWords - .USAGE_DAY)), - Accumulators.max(MongoKeyWords.USAGE_TO, MongoKeyWords.prepend$(MongoKeyWords - .USAGE_DAY)) - ), - - Aggregates.sort(Sorts.ascending( - MongoKeyWords.prependId(MongoKeyWords.RESOURCE_NAME), - MongoKeyWords.prependId(MongoKeyWords.METER_CATEGORY))) - ) - ); - - updateBillingDetails(user, mapToDetails(aggregateIterable)); - } catch (RuntimeException e) { - log.error("Updating billing details for user {} is failed", user, e); - } - } - - private List<Document> mapToDetails(AggregateIterable<Document> aggregateIterable) { - List<Document> billingDetails = new ArrayList<>(); - for (Document document : aggregateIterable) { - Document oldRef = (Document) document.get(MongoKeyWords.MONGO_ID); - Document newDocument = new Document(); - - newDocument.append(MongoKeyWords.USAGE_FROM, document.getString(MongoKeyWords.USAGE_FROM)); - newDocument.append(MongoKeyWords.USAGE_TO, document.getString(MongoKeyWords.USAGE_TO)); - newDocument.append(MongoKeyWords.COST, document.getDouble(MongoKeyWords.COST)); - - newDocument.append(MongoKeyWords.METER_CATEGORY, oldRef.getString(MongoKeyWords.METER_CATEGORY)); - newDocument.append(MongoKeyWords.RESOURCE_NAME, oldRef.getString(MongoKeyWords.RESOURCE_NAME)); - newDocument.append(MongoKeyWords.EXPLORATORY_ID, oldRef.getString(MongoKeyWords.EXPLORATORY_ID)); - newDocument.append(MongoKeyWords.RESOURCE_TYPE, oldRef.getString(MongoKeyWords.RESOURCE_TYPE)); - newDocument.append(MongoKeyWords.CURRENCY_CODE, currencyCode); - - billingDetails.add(newDocument); - } - - return billingDetails; - } - - - private void updateBillingDetails(String user, List<Document> billingDetails) { - if (!billingDetails.isEmpty()) { - Map<String, List<Document>> info = new HashMap<>(); - - Consumer<Document> aggregator = e -> { - - String notebookId = e.getString(MongoKeyWords.EXPLORATORY_ID); - List<Document> documents = info.get(notebookId); - if (documents == null) { - documents = new ArrayList<>(); - } - - documents.add(e); - info.put(notebookId, documents); - }; - - billingDetails.stream() - .filter(e -> DlabResourceType.EXPLORATORY.toString().equals(e.getString(MongoKeyWords - .RESOURCE_TYPE))) - .forEach(aggregator); - - billingDetails.stream() - .filter(e -> DlabResourceType.COMPUTATIONAL.toString().equals(e.getString(MongoKeyWords - .RESOURCE_TYPE))) - .forEach(aggregator); - - billingDetails.stream() - .filter(e -> DlabResourceType.VOLUME.toString().equals(e.getString(MongoKeyWords.RESOURCE_TYPE))) - .forEach(aggregator); - - - for (Map.Entry<String, List<Document>> entry : info.entrySet()) { - double sum = entry.getValue().stream().mapToDouble(e -> e.getDouble(MongoKeyWords.COST)).sum(); - - entry.getValue().forEach(e -> e.put(MongoKeyWords.COST_STRING, - BillingCalculationUtils.formatDouble(e.getDouble(MongoKeyWords.COST)))); - - log.debug("Update billing for notebook {}, cost is {} {}", entry.getKey(), sum, currencyCode); - - Bson updates = Updates.combine( - Updates.set(MongoKeyWords.COST_STRING, BillingCalculationUtils.formatDouble(sum)), - Updates.set(MongoKeyWords.COST, sum), - Updates.set(MongoKeyWords.CURRENCY_CODE, currencyCode), - Updates.set(MongoKeyWords.BILLING_DETAILS, entry.getValue())); - - UpdateResult updateResult = mongoDbBillingClient.getDatabase() - .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION) - .updateOne( - Filters.and( - Filters.eq(MongoKeyWords.DLAB_USER, user), - Filters.eq(MongoKeyWords.EXPLORATORY_ID_OLD, entry.getKey()) - ), - updates - ); - - log.debug("Update result for {}/{} is {}", user, entry.getKey(), updateResult); - } - } else { - log.warn("No billing details found for notebooks for user {}", user); - } - } - - - private Document getGroupingFields(String... fieldNames) { - Document d = new Document(); - for (String name : fieldNames) { - d.put(name, "$" + name); - } - return d; - } -} diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java index 0fd6098..02ec569 100644 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java @@ -37,7 +37,10 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.net.URISyntaxException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** @@ -61,13 +64,9 @@ public class AzureInvoiceCalculationService { * Constructs service class * * @param billingConfigurationAzure contains <code>billing-azure</code> module configuration - * @param billableResources resources that invoices should be calculated for */ - public AzureInvoiceCalculationService(BillingConfigurationAzure billingConfigurationAzure, - Set<AzureDlabBillableResource> billableResources) { + public AzureInvoiceCalculationService(BillingConfigurationAzure billingConfigurationAzure) { this.billingConfigurationAzure = billingConfigurationAzure; - this.billableResources = billableResources.stream().collect(Collectors.toMap(AzureDlabBillableResource::getId, - e -> e)); } /** @@ -135,8 +134,6 @@ public class AzureInvoiceCalculationService { if (usageAggregateRecordList != null && !usageAggregateRecordList.isEmpty()) { log.info("Processing {} usage records", usageAggregateRecordList.size()); - - usageAggregateRecordList = usageAggregateRecordList.stream().filter(e -> matchProperStructure(e) && isBillableDlabResource(e)) .collect(Collectors.toList()); @@ -164,7 +161,6 @@ public class AzureInvoiceCalculationService { } private boolean matchProperStructure(UsageAggregateRecord record) { - if (record.getProperties() == null) { return false; } @@ -181,13 +177,10 @@ public class AzureInvoiceCalculationService { private boolean isBillableDlabResource(UsageAggregateRecord record) { String dlabId = record.getProperties().getParsedInstanceData().getMicrosoftResources().getTags().get("Name"); - return dlabId != null && !dlabId.isEmpty() && billableResources.containsKey(dlabId); + return dlabId != null && !dlabId.isEmpty(); } - private AzureDailyResourceInvoice calculateInvoice(Map<String, Meter> rates, UsageAggregateRecord record, - String dlabId) { - - AzureDlabBillableResource azureDlabBillableResource = billableResources.get(dlabId); + private AzureDailyResourceInvoice calculateInvoice(Map<String, Meter> rates, UsageAggregateRecord record, String dlabId) { String meterId = record.getProperties().getMeterId(); Meter rateCard = rates.get(meterId); @@ -196,34 +189,25 @@ public class AzureInvoiceCalculationService { if (meterRates != null) { Double rate = meterRates.get(AzureRateCardClient.MAIN_RATE_KEY); if (rate != null) { - - AzureDailyResourceInvoice azureDailyResourceInvoice = new AzureDailyResourceInvoice - (azureDlabBillableResource); - azureDailyResourceInvoice.setUsageStartDate(record.getProperties().getUsageStartTime()); - azureDailyResourceInvoice.setUsageEndDate(record.getProperties().getUsageEndTime()); - azureDailyResourceInvoice.setMeterCategory(record.getProperties().getMeterCategory()); - azureDailyResourceInvoice.setCost( - BillingCalculationUtils.round(rate * record.getProperties().getQuantity(), 2)); - azureDailyResourceInvoice.setDay(getDay(record.getProperties().getUsageStartTime())); - azureDailyResourceInvoice.setCurrencyCode(billingConfigurationAzure.getCurrency()); - - log.trace("Generated invoice for azure resource {}", azureDailyResourceInvoice); - - return azureDailyResourceInvoice; - + return AzureDailyResourceInvoice.builder() + .dlabId(dlabId) + .usageStartDate(record.getProperties().getUsageStartTime()) + .usageEndDate(record.getProperties().getUsageEndTime()) + .meterCategory(record.getProperties().getMeterCategory()) + .cost(BillingCalculationUtils.round(rate * record.getProperties().getQuantity(), 3)) + .day(getDay(record.getProperties().getUsageStartTime())) + .currencyCode(billingConfigurationAzure.getCurrency()) + .build(); } else { - log.error("Rate Card {} has no rate for meter id {} and rate id {}. Skip record {}. Azure resource" + - " {}", - rateCard, meterId, AzureRateCardClient.MAIN_RATE_KEY, record, azureDlabBillableResource); + log.error("Rate Card {} has no rate for meter id {} and rate id {}. Skip record {}.", + rateCard, meterId, AzureRateCardClient.MAIN_RATE_KEY, record); } } else { - log.error("Rate Card {} has no meter rates fro meter id {}. Skip record {}. Azure resource {}", - rateCard, meterId, record, azureDlabBillableResource); + log.error("Rate Card {} has no meter rates fro meter id {}. Skip record {}", + rateCard, meterId, record); } } else { - log.error("Meter rate {} form usage aggregate is not found in rate card. Skip record {}. Azure resource " + - "{}", - meterId, record, azureDlabBillableResource); + log.error("Meter rate {} form usage aggregate is not found in rate card. Skip record {}.", meterId, record); } return null; @@ -231,7 +215,6 @@ public class AzureInvoiceCalculationService { private String getNewToken() { try { - log.info("Requesting authentication token ... "); ApplicationTokenCredentials applicationTokenCredentials = new ApplicationTokenCredentials( billingConfigurationAzure.getClientId(), @@ -247,7 +230,6 @@ public class AzureInvoiceCalculationService { } private String getDay(String dateTime) { - if (dateTime != null) { String[] parts = dateTime.split("T"); if (parts.length == 2) { diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java index 0e11a1f..f81fe2e 100644 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java @@ -23,7 +23,6 @@ import com.epam.dlab.MongoKeyWords; import com.epam.dlab.billing.azure.config.AzureAuthFile; import com.epam.dlab.billing.azure.config.BillingConfigurationAzure; import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice; -import com.epam.dlab.billing.azure.model.AzureDlabBillableResource; import com.epam.dlab.billing.azure.model.BillingPeriod; import com.epam.dlab.exceptions.DlabException; import com.epam.dlab.util.mongo.modules.IsoDateModule; @@ -47,7 +46,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -122,7 +120,6 @@ public class BillingSchedulerAzure { private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern ("yyyy-MM-dd'T'HH:mm:ss.SSS'Z"); private static final String SCHEDULER_ID = "azureBillingScheduler"; - private AzureBillingDetailsService azureBillingDetailsService; private BillingConfigurationAzure billingConfigurationAzure; private MongoDbBillingClient client; private ObjectMapper objectMapper = new ObjectMapper().registerModule(new IsoDateModule()); @@ -131,8 +128,6 @@ public class BillingSchedulerAzure { public CalculateBilling(BillingConfigurationAzure billingConfigurationAzure, MongoDbBillingClient client) { this.billingConfigurationAzure = billingConfigurationAzure; this.client = client; - this.azureBillingDetailsService = new AzureBillingDetailsService(client, - billingConfigurationAzure.getCurrency()); } @Override @@ -146,14 +141,7 @@ public class BillingSchedulerAzure { log.info("Billing period from db is {}", billingPeriod); if (shouldTriggerJobByTime(currentTime, billingPeriod)) { - boolean hasNew = run(billingPeriod); - if (hasNew) { - log.info("Updating billing details"); - azureBillingDetailsService.updateBillingDetails(); - } - - updateBillingPeriod(billingPeriod, currentTime, hasNew); } } @@ -193,32 +181,20 @@ public class BillingSchedulerAzure { } private boolean run(BillingPeriod billingPeriod) { - - AzureBillableResourcesService azureBillableResourcesService = new AzureBillableResourcesService(client, - billingConfigurationAzure.getSharedStorageAccountTagName(), - billingConfigurationAzure.getSsnStorageAccountTagName(), - billingConfigurationAzure.getDatalakeTagName()); - Set<AzureDlabBillableResource> billableResources = azureBillableResourcesService.getBillableResources(); - AzureInvoiceCalculationService azureInvoiceCalculationService - = new AzureInvoiceCalculationService(billingConfigurationAzure, billableResources); + = new AzureInvoiceCalculationService(billingConfigurationAzure); List<AzureDailyResourceInvoice> dailyInvoices = azureInvoiceCalculationService.generateInvoiceData( DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC)), DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC))); - if (!dailyInvoices.isEmpty()) { - client.getDatabase().getCollection(MongoKeyWords.BILLING_DETAILS) .insertMany(dailyInvoices.stream().map(AzureDailyResourceInvoice::to) .collect(Collectors.toList())); - return true; - } else { log.warn("Daily invoices is empty for period {}", billingPeriod); - return false; } } diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java index ff132a2..486ddd5 100644 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java @@ -19,35 +19,21 @@ package com.epam.dlab.billing.azure.model; -import com.epam.dlab.billing.DlabResourceType; import com.epam.dlab.billing.azure.MongoDocument; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; @Data -@NoArgsConstructor +@Builder @EqualsAndHashCode(callSuper = true) @JsonIgnoreProperties(ignoreUnknown = true) public class AzureDailyResourceInvoice extends MongoDocument<AzureDailyResourceInvoice> { @JsonProperty private String dlabId; @JsonProperty - private String user; - @JsonProperty - private String project; - @JsonProperty - private String exploratoryId; - @JsonProperty - private String computationalId; - @JsonProperty - private DlabResourceType resourceType; - @JsonProperty - private String resourceName; - @JsonProperty private String meterCategory; @JsonProperty private String usageStartDate; @@ -59,22 +45,4 @@ public class AzureDailyResourceInvoice extends MongoDocument<AzureDailyResourceI private double cost; @JsonProperty private String currencyCode; - - @Builder - public AzureDailyResourceInvoice(AzureDlabBillableResource azureDlabBillableResource) { - this.dlabId = azureDlabBillableResource.getId(); - this.user = azureDlabBillableResource.getUser(); - this.project = azureDlabBillableResource.getProject(); - this.resourceType = azureDlabBillableResource.getType(); - this.resourceName = azureDlabBillableResource.getResourceName(); - - if (resourceType == DlabResourceType.EXPLORATORY) { - this.exploratoryId = azureDlabBillableResource.getId(); - } else if (resourceType == DlabResourceType.COMPUTATIONAL) { - this.computationalId = azureDlabBillableResource.getId(); - this.exploratoryId = azureDlabBillableResource.getNotebookId(); - } else if (resourceType == DlabResourceType.VOLUME) { - this.exploratoryId = azureDlabBillableResource.getNotebookId(); - } - } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
