This is an automated email from the ASF dual-hosted git repository.

ofuks pushed a commit to branch DLAB-1571
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git

commit 3ba048723faf6bb0f3529ddda3dfc0526a54c98b
Author: Oleh Fuks <[email protected]>
AuthorDate: Tue Mar 17 15:07:09 2020 +0200

    Added more changes to Azure remote billing
---
 .../azure/AzureBillableResourcesService.java       | 315 ---------------------
 .../billing/azure/AzureBillingDetailsService.java  | 194 -------------
 .../azure/AzureInvoiceCalculationService.java      |  60 ++--
 .../dlab/billing/azure/BillingSchedulerAzure.java  |  26 +-
 .../azure/model/AzureDailyResourceInvoice.java     |  34 +--
 5 files changed, 23 insertions(+), 606 deletions(-)

diff --git 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
deleted file mode 100644
index c4ee5b4..0000000
--- 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure;
-
-import com.epam.dlab.MongoKeyWords;
-import com.epam.dlab.billing.DlabResourceType;
-import com.epam.dlab.billing.azure.model.AzureDlabBillableResource;
-import com.epam.dlab.dto.UserInstanceDTO;
-import com.epam.dlab.dto.azure.edge.EdgeInfoAzure;
-import com.epam.dlab.dto.computational.UserComputationalResource;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
-import com.mongodb.client.FindIterable;
-import com.mongodb.client.model.Filters;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.bson.Document;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-import static com.mongodb.client.model.Projections.exclude;
-import static com.mongodb.client.model.Projections.fields;
-
-/**
- * Helps to retrieve billable resources that are created in scope of DLab 
usage. Uses MongoDB as data source
- * for created resources
- */
-@Slf4j
-public class AzureBillableResourcesService {
-       private static final String SHARED_RESOURCE = "Shared resource";
-       private static final String[] USER_INSTANCES_EXCLUDED_FIELDS = 
{"scheduler_data", "last_activity",
-                       "computational_resources.scheduler_data", 
"computational_resources.last_activity"};
-       private final ObjectMapper objectMapper = new ObjectMapper();
-
-       private MongoDbBillingClient mongoDbBillingClient;
-       private String serviceBaseName;
-       private String sharedStorageAccountTagName;
-       private String ssnStorageAccountTagName;
-       private String azureDataLakeTagName;
-
-       /**
-        * Constructs the service class
-        *
-        * @param mongoDbBillingClient        mongodb client to retrieve all 
billable resources
-        * @param sharedStorageAccountTagName shared storage account tag name
-        * @param ssnStorageAccountTagName    ssn storage account tag name
-        * @param azureDataLakeTagName        azure DataLake tag name
-        */
-       public AzureBillableResourcesService(MongoDbBillingClient 
mongoDbBillingClient, String sharedStorageAccountTagName,
-                                                                               
 String ssnStorageAccountTagName, String azureDataLakeTagName) {
-               this.mongoDbBillingClient = mongoDbBillingClient;
-
-               this.serviceBaseName = 
getConfigurationSettingValue(MongoKeyWords.SERVICE_BASE_NAME_KEY)
-                               .replace('_', '-').toLowerCase();
-
-               this.sharedStorageAccountTagName = sharedStorageAccountTagName;
-               this.ssnStorageAccountTagName = ssnStorageAccountTagName;
-               this.azureDataLakeTagName = azureDataLakeTagName;
-       }
-
-
-       /**
-        * Collects billable resources
-        *
-        * @return set of all billable resources that were created in scope by 
DLab from its installation to current time
-        */
-       public Set<AzureDlabBillableResource> getBillableResources() {
-
-               Set<AzureDlabBillableResource> billableResources = new 
HashSet<>();
-
-               billableResources.addAll(getSsn());
-               billableResources.addAll(getDataLake());
-               billableResources.addAll(getEdgeAndStorageAccount());
-               billableResources.addAll(getNotebooksAndClusters());
-
-               List<AzureDlabBillableResource> list = new 
ArrayList<>(billableResources);
-               
list.sort(Comparator.comparing(AzureDlabBillableResource::getId));
-
-               try {
-                       log.debug("Billable resources is \n {}", 
objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString
-                                       (list));
-               } catch (JsonProcessingException e) {
-                       log.debug("Error during pretty printing. Show simple 
list", e);
-                       log.debug("Billable resources is {}", list);
-               }
-
-               return billableResources;
-       }
-
-       private String getConfigurationSettingValue(String key) {
-
-               Document document = 
mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.SETTINGS_COLLECTION)
-                               .find(Filters.eq(MongoKeyWords.MONGO_ID, 
key)).first();
-
-               if (document != null) {
-                       String value = document.getString("value");
-                       if (StringUtils.isEmpty(value)) {
-                               throw new IllegalStateException("Configuration 
" + key + " does not have value in settings");
-                       }
-                       log.info("Key {} has value {}", key, value);
-                       return value;
-               } else {
-                       throw new IllegalStateException("Configuration " + key 
+ " is not present in settings");
-               }
-
-       }
-
-       private String getConfigurationSettingValueOrEmpty(String key) {
-               try {
-                       return getConfigurationSettingValue(key);
-               } catch (IllegalStateException e) {
-                       log.warn("key {} is not found", key, e);
-                       return null;
-               }
-       }
-
-       private Set<AzureDlabBillableResource> getSsn() {
-
-               return Sets.newHashSet(
-                               
AzureDlabBillableResource.builder().id(serviceBaseName + 
"-ssn").type(DlabResourceType.SSN).build(),
-                               
AzureDlabBillableResource.builder().id(ssnStorageAccountTagName).type(DlabResourceType
-                                               .SSN_STORAGE_ACCOUNT).build(),
-                               
AzureDlabBillableResource.builder().id(sharedStorageAccountTagName).type(DlabResourceType
-                                               
.COLLABORATION_STORAGE_ACCOUNT).build(),
-                               
AzureDlabBillableResource.builder().id(serviceBaseName + "-ssn-volume-primary")
-                                               
.type(DlabResourceType.VOLUME).build()
-               );
-       }
-
-       private Set<AzureDlabBillableResource> getDataLake() {
-
-               if (azureDataLakeTagName != null) {
-                       return 
Sets.newHashSet(AzureDlabBillableResource.builder().id(azureDataLakeTagName)
-                                       
.type(DlabResourceType.DATA_LAKE_STORE).build());
-               }
-
-               return Sets.newHashSet();
-       }
-
-       private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() {
-               Map<String, List<Document>> projectEndpoints = 
StreamSupport.stream(mongoDbBillingClient.getDatabase()
-                               
.getCollection("Projects").find().spliterator(), false)
-                               .collect(Collectors.toMap(key -> 
key.getString("name").toLowerCase(),
-                                               value -> (List<Document>) 
value.get("endpoints")));
-
-               return projectEndpoints.entrySet()
-                               .stream()
-                               .flatMap(projectEndpoint -> 
getEdgeAndStoragePerProject(projectEndpoint.getKey(), 
projectEndpoint.getValue()))
-                               .collect(Collectors.toSet());
-       }
-
-       private Stream<AzureDlabBillableResource> 
getEdgeAndStoragePerProject(String projectName, List<Document> endpoints) {
-               return endpoints
-                               .stream()
-                               .flatMap(endpoint -> {
-                                       try {
-                                               return 
getEdgeAndStorageAccount(projectName, objectMapper.readValue(
-                                                               
objectMapper.writeValueAsString(endpoint.get("edgeInfo")),
-                                                               new 
com.fasterxml.jackson.core.type.TypeReference<EdgeInfoAzure>() {
-                                                               })).stream();
-                                       } catch (IOException e) {
-                                               log.error("Error during 
preparation of billable resources", e);
-                                       }
-                                       return Stream.empty();
-                               });
-       }
-
-       private Set<AzureDlabBillableResource> getEdgeAndStorageAccount(String 
projectName, EdgeInfoAzure edgeInfoAzure) {
-               Set<AzureDlabBillableResource> billableResources = new 
HashSet<>();
-
-               if 
(StringUtils.isNotEmpty(edgeInfoAzure.getUserContainerName())) {
-                       
billableResources.add(AzureDlabBillableResource.builder()
-                                       
.id(edgeInfoAzure.getUserStorageAccountTagName())
-                                       
.type(DlabResourceType.EDGE_STORAGE_ACCOUNT)
-                                       .user(SHARED_RESOURCE)
-                                       .project(projectName)
-                                       .build());
-               }
-
-               if (StringUtils.isNotEmpty(edgeInfoAzure.getInstanceId())) {
-                       
billableResources.add(AzureDlabBillableResource.builder()
-                                       .id(edgeInfoAzure.getInstanceId())
-                                       .type(DlabResourceType.EDGE)
-                                       .user(SHARED_RESOURCE)
-                                       .project(projectName)
-                                       .build());
-
-                       
billableResources.add(AzureDlabBillableResource.builder()
-                                       .id(edgeInfoAzure.getInstanceId() + 
"-volume-primary")
-                                       .type(DlabResourceType.VOLUME)
-                                       .user(SHARED_RESOURCE)
-                                       .project(projectName)
-                                       .build());
-               }
-
-               return billableResources;
-       }
-
-       private Set<AzureDlabBillableResource> getNotebooksAndClusters() {
-
-               Set<AzureDlabBillableResource> billableResources = new 
HashSet<>();
-
-               try {
-                       final FindIterable<Document> userInstanceDocuments = 
mongoDbBillingClient.getDatabase()
-                                       
.getCollection(MongoKeyWords.NOTEBOOK_COLLECTION)
-                                       .find()
-                                       
.projection(fields(exclude(USER_INSTANCES_EXCLUDED_FIELDS)));
-                       List<UserInstanceDTO> userInstanceDTOS = 
objectMapper.readValue(
-                                       
objectMapper.writeValueAsString(userInstanceDocuments),
-                                       new 
com.fasterxml.jackson.core.type.TypeReference<List<UserInstanceDTO>>() {
-                                       });
-
-                       if (userInstanceDTOS != null && 
!userInstanceDTOS.isEmpty()) {
-                               userInstanceDTOS.forEach(e -> 
billableResources.addAll(getNotebookAndClusters(e)));
-                       }
-
-               } catch (IOException e) {
-                       log.error("Error during preparation of billable 
resources", e);
-               }
-
-               return billableResources;
-       }
-
-       private Set<AzureDlabBillableResource> 
getNotebookAndClusters(UserInstanceDTO userInstanceDTO) {
-               Set<AzureDlabBillableResource> notebookResources = new 
HashSet<>();
-
-               if (StringUtils.isNotEmpty(userInstanceDTO.getExploratoryId())) 
{
-                       
notebookResources.add(AzureDlabBillableResource.builder()
-                                       .id(userInstanceDTO.getExploratoryId())
-                                       .type(DlabResourceType.EXPLORATORY)
-                                       .user(userInstanceDTO.getUser())
-                                       .project(userInstanceDTO.getProject())
-                                       
.notebookId(userInstanceDTO.getExploratoryId())
-                                       
.resourceName(userInstanceDTO.getExploratoryName())
-                                       .build());
-                       notebookResources.addAll(getVolumes(userInstanceDTO, 
userInstanceDTO.getExploratoryId(), "Volume primary",
-                                       "Volume secondary"));
-
-                       if (userInstanceDTO.getResources() != null && 
!userInstanceDTO.getResources().isEmpty()) {
-                               for (UserComputationalResource 
userComputationalResource : userInstanceDTO.getResources()) {
-                                       if 
(StringUtils.isNotEmpty(userComputationalResource.getComputationalId())) {
-
-                                               
notebookResources.add(AzureDlabBillableResource.builder()
-                                                               
.id(userComputationalResource.getComputationalId())
-                                                               
.type(DlabResourceType.COMPUTATIONAL)
-                                                               
.user(userInstanceDTO.getUser())
-                                                               
.project(userInstanceDTO.getProject())
-                                                               
.notebookId(userInstanceDTO.getExploratoryId())
-                                                               
.resourceName(userComputationalResource.getComputationalName())
-                                                               .build());
-                                               final 
List<AzureDlabBillableResource> volumes = getVolumes(userInstanceDTO,
-                                                               
userComputationalResource.getComputationalId(),
-                                                               
userComputationalResource.getComputationalName() + " volume primary",
-                                                               
userComputationalResource.getComputationalName() + " volume secondary");
-                                               
notebookResources.addAll(volumes);
-
-                                       } else {
-                                               log.error("Computational with 
empty id {} is found in notebook {}. Skip it.",
-                                                               
userComputationalResource, userInstanceDTO);
-                                       }
-                               }
-                       }
-
-               } else {
-                       log.error("Notebook {} with empty id id found. Skip 
it.", userInstanceDTO);
-               }
-
-               return notebookResources;
-       }
-
-       private List<AzureDlabBillableResource> getVolumes(UserInstanceDTO 
userInstanceDTO, String exploratoryId, String
-                       primaryVolumeName, String secondaryVolumeName) {
-
-               return Arrays.asList(
-                               AzureDlabBillableResource.builder()
-                                               .id(exploratoryId + 
"-volume-primary")
-                                               .type(DlabResourceType.VOLUME)
-                                               .user(userInstanceDTO.getUser())
-                                               
.project(userInstanceDTO.getProject())
-                                               
.notebookId(userInstanceDTO.getExploratoryId())
-                                               .resourceName(primaryVolumeName)
-                                               .build(),
-                               AzureDlabBillableResource.builder()
-                                               .id(exploratoryId + 
"-volume-secondary")
-                                               .type(DlabResourceType.VOLUME)
-                                               .user(userInstanceDTO.getUser())
-                                               
.project(userInstanceDTO.getProject())
-                                               
.notebookId(userInstanceDTO.getExploratoryId())
-                                               
.resourceName(secondaryVolumeName)
-                                               .build()
-               );
-       }
-}
diff --git 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java
 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java
deleted file mode 100644
index 6d84eb8..0000000
--- 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillingDetailsService.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure;
-
-import com.epam.dlab.MongoKeyWords;
-import com.epam.dlab.billing.BillingCalculationUtils;
-import com.epam.dlab.billing.DlabResourceType;
-import com.google.common.collect.Lists;
-import com.mongodb.client.AggregateIterable;
-import com.mongodb.client.model.*;
-import com.mongodb.client.result.UpdateResult;
-import lombok.extern.slf4j.Slf4j;
-import org.bson.Document;
-import org.bson.conversions.Bson;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.stream.StreamSupport;
-
-@Slf4j
-public class AzureBillingDetailsService {
-       private MongoDbBillingClient mongoDbBillingClient;
-       private String currencyCode;
-
-       public AzureBillingDetailsService(MongoDbBillingClient 
mongoDbBillingClient, String currencyCode) {
-               this.mongoDbBillingClient = mongoDbBillingClient;
-               this.currencyCode = currencyCode;
-       }
-
-       public void updateBillingDetails() {
-               StreamSupport.stream(mongoDbBillingClient.getDatabase()
-                               
.getCollection(MongoKeyWords.NOTEBOOK_COLLECTION).find().spliterator(), false)
-                               .map(a -> a.getString(MongoKeyWords.DLAB_USER))
-                               .forEach(this::updateBillingDetails);
-       }
-
-       public void updateBillingDetails(String user) {
-               log.debug("Updating billing details for user {}", user);
-
-               try {
-                       AggregateIterable<Document> aggregateIterable = 
mongoDbBillingClient.getDatabase()
-                                       
.getCollection(MongoKeyWords.BILLING_DETAILS)
-                                       .aggregate(Lists.newArrayList(
-                                                       Aggregates.match(
-                                                                       
Filters.and(
-                                                                               
        Filters.eq(MongoKeyWords.DLAB_USER, user),
-                                                                               
        Filters.in(MongoKeyWords.RESOURCE_TYPE,
-                                                                               
                        DlabResourceType.EXPLORATORY.toString(),
-                                                                               
                        DlabResourceType.COMPUTATIONAL.toString(),
-                                                                               
                        DlabResourceType.VOLUME.toString())
-                                                                       )
-                                                       ),
-
-                                                       
Aggregates.group(getGroupingFields(
-                                                                       
MongoKeyWords.DLAB_ID,
-                                                                       
MongoKeyWords.DLAB_USER,
-                                                                       
MongoKeyWords.EXPLORATORY_ID,
-                                                                       
MongoKeyWords.RESOURCE_TYPE,
-                                                                       
MongoKeyWords.RESOURCE_NAME,
-                                                                       
MongoKeyWords.COMPUTATIONAL_ID,
-                                                                       
MongoKeyWords.METER_CATEGORY),
-                                                                       
Accumulators.sum(MongoKeyWords.COST, 
MongoKeyWords.prepend$(MongoKeyWords.COST)),
-                                                                       
Accumulators.min(MongoKeyWords.USAGE_FROM, MongoKeyWords.prepend$(MongoKeyWords
-                                                                               
        .USAGE_DAY)),
-                                                                       
Accumulators.max(MongoKeyWords.USAGE_TO, MongoKeyWords.prepend$(MongoKeyWords
-                                                                               
        .USAGE_DAY))
-                                                       ),
-
-                                                       
Aggregates.sort(Sorts.ascending(
-                                                                       
MongoKeyWords.prependId(MongoKeyWords.RESOURCE_NAME),
-                                                                       
MongoKeyWords.prependId(MongoKeyWords.METER_CATEGORY)))
-                                                       )
-                                       );
-
-                       updateBillingDetails(user, 
mapToDetails(aggregateIterable));
-               } catch (RuntimeException e) {
-                       log.error("Updating billing details for user {} is 
failed", user, e);
-               }
-       }
-
-       private List<Document> mapToDetails(AggregateIterable<Document> 
aggregateIterable) {
-               List<Document> billingDetails = new ArrayList<>();
-               for (Document document : aggregateIterable) {
-                       Document oldRef = (Document) 
document.get(MongoKeyWords.MONGO_ID);
-                       Document newDocument = new Document();
-
-                       newDocument.append(MongoKeyWords.USAGE_FROM, 
document.getString(MongoKeyWords.USAGE_FROM));
-                       newDocument.append(MongoKeyWords.USAGE_TO, 
document.getString(MongoKeyWords.USAGE_TO));
-                       newDocument.append(MongoKeyWords.COST, 
document.getDouble(MongoKeyWords.COST));
-
-                       newDocument.append(MongoKeyWords.METER_CATEGORY, 
oldRef.getString(MongoKeyWords.METER_CATEGORY));
-                       newDocument.append(MongoKeyWords.RESOURCE_NAME, 
oldRef.getString(MongoKeyWords.RESOURCE_NAME));
-                       newDocument.append(MongoKeyWords.EXPLORATORY_ID, 
oldRef.getString(MongoKeyWords.EXPLORATORY_ID));
-                       newDocument.append(MongoKeyWords.RESOURCE_TYPE, 
oldRef.getString(MongoKeyWords.RESOURCE_TYPE));
-                       newDocument.append(MongoKeyWords.CURRENCY_CODE, 
currencyCode);
-
-                       billingDetails.add(newDocument);
-               }
-
-               return billingDetails;
-       }
-
-
-       private void updateBillingDetails(String user, List<Document> 
billingDetails) {
-               if (!billingDetails.isEmpty()) {
-                       Map<String, List<Document>> info = new HashMap<>();
-
-                       Consumer<Document> aggregator = e -> {
-
-                               String notebookId = 
e.getString(MongoKeyWords.EXPLORATORY_ID);
-                               List<Document> documents = info.get(notebookId);
-                               if (documents == null) {
-                                       documents = new ArrayList<>();
-                               }
-
-                               documents.add(e);
-                               info.put(notebookId, documents);
-                       };
-
-                       billingDetails.stream()
-                                       .filter(e -> 
DlabResourceType.EXPLORATORY.toString().equals(e.getString(MongoKeyWords
-                                                       .RESOURCE_TYPE)))
-                                       .forEach(aggregator);
-
-                       billingDetails.stream()
-                                       .filter(e -> 
DlabResourceType.COMPUTATIONAL.toString().equals(e.getString(MongoKeyWords
-                                                       .RESOURCE_TYPE)))
-                                       .forEach(aggregator);
-
-                       billingDetails.stream()
-                                       .filter(e -> 
DlabResourceType.VOLUME.toString().equals(e.getString(MongoKeyWords.RESOURCE_TYPE)))
-                                       .forEach(aggregator);
-
-
-                       for (Map.Entry<String, List<Document>> entry : 
info.entrySet()) {
-                               double sum = 
entry.getValue().stream().mapToDouble(e -> 
e.getDouble(MongoKeyWords.COST)).sum();
-
-                               entry.getValue().forEach(e -> 
e.put(MongoKeyWords.COST_STRING,
-                                               
BillingCalculationUtils.formatDouble(e.getDouble(MongoKeyWords.COST))));
-
-                               log.debug("Update billing for notebook {}, cost 
is {} {}", entry.getKey(), sum, currencyCode);
-
-                               Bson updates = Updates.combine(
-                                               
Updates.set(MongoKeyWords.COST_STRING, 
BillingCalculationUtils.formatDouble(sum)),
-                                               Updates.set(MongoKeyWords.COST, 
sum),
-                                               
Updates.set(MongoKeyWords.CURRENCY_CODE, currencyCode),
-                                               
Updates.set(MongoKeyWords.BILLING_DETAILS, entry.getValue()));
-
-                               UpdateResult updateResult = 
mongoDbBillingClient.getDatabase()
-                                               
.getCollection(MongoKeyWords.NOTEBOOK_COLLECTION)
-                                               .updateOne(
-                                                               Filters.and(
-                                                                               
Filters.eq(MongoKeyWords.DLAB_USER, user),
-                                                                               
Filters.eq(MongoKeyWords.EXPLORATORY_ID_OLD, entry.getKey())
-                                                               ),
-                                                               updates
-                                               );
-
-                               log.debug("Update result for {}/{} is {}", 
user, entry.getKey(), updateResult);
-                       }
-               } else {
-                       log.warn("No billing details found for notebooks for 
user {}", user);
-               }
-       }
-
-
-       private Document getGroupingFields(String... fieldNames) {
-               Document d = new Document();
-               for (String name : fieldNames) {
-                       d.put(name, "$" + name);
-               }
-               return d;
-       }
-}
diff --git 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
index 0fd6098..02ec569 100644
--- 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
+++ 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureInvoiceCalculationService.java
@@ -37,7 +37,10 @@ import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -61,13 +64,9 @@ public class AzureInvoiceCalculationService {
         * Constructs service class
         *
         * @param billingConfigurationAzure contains <code>billing-azure</code> 
module configuration
-        * @param billableResources         resources that invoices should be 
calculated for
         */
-       public AzureInvoiceCalculationService(BillingConfigurationAzure 
billingConfigurationAzure,
-                                                                               
  Set<AzureDlabBillableResource> billableResources) {
+       public AzureInvoiceCalculationService(BillingConfigurationAzure 
billingConfigurationAzure) {
                this.billingConfigurationAzure = billingConfigurationAzure;
-               this.billableResources = 
billableResources.stream().collect(Collectors.toMap(AzureDlabBillableResource::getId,
-                e -> e));
        }
 
        /**
@@ -135,8 +134,6 @@ public class AzureInvoiceCalculationService {
 
                if (usageAggregateRecordList != null && 
!usageAggregateRecordList.isEmpty()) {
                        log.info("Processing {} usage records", 
usageAggregateRecordList.size());
-
-
                        usageAggregateRecordList = 
usageAggregateRecordList.stream().filter(e ->
                                        matchProperStructure(e) && 
isBillableDlabResource(e))
                                        .collect(Collectors.toList());
@@ -164,7 +161,6 @@ public class AzureInvoiceCalculationService {
        }
 
        private boolean matchProperStructure(UsageAggregateRecord record) {
-
                if (record.getProperties() == null) {
                        return false;
                }
@@ -181,13 +177,10 @@ public class AzureInvoiceCalculationService {
 
        private boolean isBillableDlabResource(UsageAggregateRecord record) {
                String dlabId = 
record.getProperties().getParsedInstanceData().getMicrosoftResources().getTags().get("Name");
-               return dlabId != null && !dlabId.isEmpty() && 
billableResources.containsKey(dlabId);
+               return dlabId != null && !dlabId.isEmpty();
        }
 
-       private AzureDailyResourceInvoice calculateInvoice(Map<String, Meter> 
rates, UsageAggregateRecord record,
-                                                                               
                           String dlabId) {
-
-               AzureDlabBillableResource azureDlabBillableResource = 
billableResources.get(dlabId);
+       private AzureDailyResourceInvoice calculateInvoice(Map<String, Meter> 
rates, UsageAggregateRecord record, String dlabId) {
                String meterId = record.getProperties().getMeterId();
                Meter rateCard = rates.get(meterId);
 
@@ -196,34 +189,25 @@ public class AzureInvoiceCalculationService {
                        if (meterRates != null) {
                                Double rate = 
meterRates.get(AzureRateCardClient.MAIN_RATE_KEY);
                                if (rate != null) {
-
-                                       AzureDailyResourceInvoice 
azureDailyResourceInvoice = new AzureDailyResourceInvoice
-                            (azureDlabBillableResource);
-                                       
azureDailyResourceInvoice.setUsageStartDate(record.getProperties().getUsageStartTime());
-                                       
azureDailyResourceInvoice.setUsageEndDate(record.getProperties().getUsageEndTime());
-                                       
azureDailyResourceInvoice.setMeterCategory(record.getProperties().getMeterCategory());
-                                       azureDailyResourceInvoice.setCost(
-                                                       
BillingCalculationUtils.round(rate * record.getProperties().getQuantity(), 2));
-                                       
azureDailyResourceInvoice.setDay(getDay(record.getProperties().getUsageStartTime()));
-                                       
azureDailyResourceInvoice.setCurrencyCode(billingConfigurationAzure.getCurrency());
-
-                                       log.trace("Generated invoice for azure 
resource {}", azureDailyResourceInvoice);
-
-                                       return azureDailyResourceInvoice;
-
+                                       return 
AzureDailyResourceInvoice.builder()
+                                                       .dlabId(dlabId)
+                                                       
.usageStartDate(record.getProperties().getUsageStartTime())
+                                                       
.usageEndDate(record.getProperties().getUsageEndTime())
+                                                       
.meterCategory(record.getProperties().getMeterCategory())
+                                                       
.cost(BillingCalculationUtils.round(rate * 
record.getProperties().getQuantity(), 3))
+                                                       
.day(getDay(record.getProperties().getUsageStartTime()))
+                                                       
.currencyCode(billingConfigurationAzure.getCurrency())
+                                                       .build();
                                } else {
-                                       log.error("Rate Card {} has no rate for 
meter id {} and rate id {}. Skip record {}. Azure resource" +
-                                    " {}",
-                                                       rateCard, meterId, 
AzureRateCardClient.MAIN_RATE_KEY, record, azureDlabBillableResource);
+                                       log.error("Rate Card {} has no rate for 
meter id {} and rate id {}. Skip record {}.",
+                                                       rateCard, meterId, 
AzureRateCardClient.MAIN_RATE_KEY, record);
                                }
                        } else {
-                               log.error("Rate Card {} has no meter rates fro 
meter id {}. Skip record {}. Azure resource {}",
-                                               rateCard, meterId, record, 
azureDlabBillableResource);
+                               log.error("Rate Card {} has no meter rates fro 
meter id {}. Skip record {}",
+                                               rateCard, meterId, record);
                        }
                } else {
-                       log.error("Meter rate {} form usage aggregate is not 
found in rate card. Skip record {}.  Azure resource " +
-                            "{}",
-                                       meterId, record, 
azureDlabBillableResource);
+                       log.error("Meter rate {} form usage aggregate is not 
found in rate card. Skip record {}.", meterId, record);
                }
 
                return null;
@@ -231,7 +215,6 @@ public class AzureInvoiceCalculationService {
 
        private String getNewToken() {
                try {
-
                        log.info("Requesting authentication token ... ");
                        ApplicationTokenCredentials applicationTokenCredentials 
= new ApplicationTokenCredentials(
                                        billingConfigurationAzure.getClientId(),
@@ -247,7 +230,6 @@ public class AzureInvoiceCalculationService {
        }
 
        private String getDay(String dateTime) {
-
                if (dateTime != null) {
                        String[] parts = dateTime.split("T");
                        if (parts.length == 2) {
diff --git 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
index 0e11a1f..f81fe2e 100644
--- 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
+++ 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
@@ -23,7 +23,6 @@ import com.epam.dlab.MongoKeyWords;
 import com.epam.dlab.billing.azure.config.AzureAuthFile;
 import com.epam.dlab.billing.azure.config.BillingConfigurationAzure;
 import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice;
-import com.epam.dlab.billing.azure.model.AzureDlabBillableResource;
 import com.epam.dlab.billing.azure.model.BillingPeriod;
 import com.epam.dlab.exceptions.DlabException;
 import com.epam.dlab.util.mongo.modules.IsoDateModule;
@@ -47,7 +46,6 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -122,7 +120,6 @@ public class BillingSchedulerAzure {
                private static final DateTimeFormatter DATE_TIME_FORMATTER = 
DateTimeFormat.forPattern
                                ("yyyy-MM-dd'T'HH:mm:ss.SSS'Z");
                private static final String SCHEDULER_ID = 
"azureBillingScheduler";
-               private AzureBillingDetailsService azureBillingDetailsService;
                private BillingConfigurationAzure billingConfigurationAzure;
                private MongoDbBillingClient client;
                private ObjectMapper objectMapper = new 
ObjectMapper().registerModule(new IsoDateModule());
@@ -131,8 +128,6 @@ public class BillingSchedulerAzure {
                public CalculateBilling(BillingConfigurationAzure 
billingConfigurationAzure, MongoDbBillingClient client) {
                        this.billingConfigurationAzure = 
billingConfigurationAzure;
                        this.client = client;
-                       this.azureBillingDetailsService = new 
AzureBillingDetailsService(client,
-                                       
billingConfigurationAzure.getCurrency());
                }
 
                @Override
@@ -146,14 +141,7 @@ public class BillingSchedulerAzure {
                                        log.info("Billing period from db is 
{}", billingPeriod);
 
                                        if (shouldTriggerJobByTime(currentTime, 
billingPeriod)) {
-
                                                boolean hasNew = 
run(billingPeriod);
-                                               if (hasNew) {
-                                                       log.info("Updating 
billing details");
-                                                       
azureBillingDetailsService.updateBillingDetails();
-                                               }
-
-
                                                
updateBillingPeriod(billingPeriod, currentTime, hasNew);
                                        }
                                }
@@ -193,32 +181,20 @@ public class BillingSchedulerAzure {
                }
 
                private boolean run(BillingPeriod billingPeriod) {
-
-                       AzureBillableResourcesService 
azureBillableResourcesService = new AzureBillableResourcesService(client,
-                                       
billingConfigurationAzure.getSharedStorageAccountTagName(),
-                                       
billingConfigurationAzure.getSsnStorageAccountTagName(),
-                                       
billingConfigurationAzure.getDatalakeTagName());
-                       Set<AzureDlabBillableResource> billableResources = 
azureBillableResourcesService.getBillableResources();
-
                        AzureInvoiceCalculationService 
azureInvoiceCalculationService
-                                       = new 
AzureInvoiceCalculationService(billingConfigurationAzure, billableResources);
+                                       = new 
AzureInvoiceCalculationService(billingConfigurationAzure);
 
                        List<AzureDailyResourceInvoice> dailyInvoices = 
azureInvoiceCalculationService.generateInvoiceData(
                                        DATE_TIME_FORMATTER.print(new 
DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC)),
                                        DATE_TIME_FORMATTER.print(new 
DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC)));
 
-
                        if (!dailyInvoices.isEmpty()) {
-
                                
client.getDatabase().getCollection(MongoKeyWords.BILLING_DETAILS)
                                                
.insertMany(dailyInvoices.stream().map(AzureDailyResourceInvoice::to)
                                                                
.collect(Collectors.toList()));
-
                                return true;
-
                        } else {
                                log.warn("Daily invoices is empty for period 
{}", billingPeriod);
-
                                return false;
                        }
                }
diff --git 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
index ff132a2..486ddd5 100644
--- 
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
+++ 
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/model/AzureDailyResourceInvoice.java
@@ -19,35 +19,21 @@
 
 package com.epam.dlab.billing.azure.model;
 
-import com.epam.dlab.billing.DlabResourceType;
 import com.epam.dlab.billing.azure.MongoDocument;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
 
 @Data
-@NoArgsConstructor
+@Builder
 @EqualsAndHashCode(callSuper = true)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class AzureDailyResourceInvoice extends 
MongoDocument<AzureDailyResourceInvoice> {
        @JsonProperty
        private String dlabId;
        @JsonProperty
-       private String user;
-       @JsonProperty
-       private String project;
-       @JsonProperty
-       private String exploratoryId;
-       @JsonProperty
-       private String computationalId;
-       @JsonProperty
-       private DlabResourceType resourceType;
-       @JsonProperty
-       private String resourceName;
-       @JsonProperty
        private String meterCategory;
        @JsonProperty
        private String usageStartDate;
@@ -59,22 +45,4 @@ public class AzureDailyResourceInvoice extends 
MongoDocument<AzureDailyResourceI
        private double cost;
        @JsonProperty
        private String currencyCode;
-
-       @Builder
-       public AzureDailyResourceInvoice(AzureDlabBillableResource 
azureDlabBillableResource) {
-               this.dlabId = azureDlabBillableResource.getId();
-               this.user = azureDlabBillableResource.getUser();
-               this.project = azureDlabBillableResource.getProject();
-               this.resourceType = azureDlabBillableResource.getType();
-               this.resourceName = azureDlabBillableResource.getResourceName();
-
-               if (resourceType == DlabResourceType.EXPLORATORY) {
-                       this.exploratoryId = azureDlabBillableResource.getId();
-               } else if (resourceType == DlabResourceType.COMPUTATIONAL) {
-                       this.computationalId = 
azureDlabBillableResource.getId();
-                       this.exploratoryId = 
azureDlabBillableResource.getNotebookId();
-               } else if (resourceType == DlabResourceType.VOLUME) {
-                       this.exploratoryId = 
azureDlabBillableResource.getNotebookId();
-               }
-       }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to