This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch inactivity_integration
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/inactivity_integration by this
push:
new 47979d7 DLAB-527 fixed bug connected with azure billing
47979d7 is described below
commit 47979d7ff473326c179b3e126be8871860e15716
Author: bhliva <[email protected]>
AuthorDate: Wed Mar 13 17:35:13 2019 +0200
DLAB-527 fixed bug connected with azure billing
---
.../azure/AzureBillableResourcesService.java | 492 ++++++++++-----------
1 file changed, 246 insertions(+), 246 deletions(-)
diff --git
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
index 1630ac7..b6d88fd 100644
---
a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
+++
b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/AzureBillableResourcesService.java
@@ -43,250 +43,250 @@ import static com.mongodb.client.model.Projections.fields;
*/
@Slf4j
public class AzureBillableResourcesService {
- private static final String[] USER_INSTANCES_EXCLUDED_FIELDS =
{"scheduler_data", "computational_resources" +
- ".scheduler_data"};
- private final ObjectMapper objectMapper = new ObjectMapper();
-
- private MongoDbBillingClient mongoDbBillingClient;
- private String serviceBaseName;
- private String sharedStorageAccountTagName;
- private String ssnStorageAccountTagName;
- private String azureDataLakeTagName;
-
- /**
- * Constructs the service class
- *
- * @param mongoDbBillingClient mongodb client to retrieve all billable
resources
- */
- public AzureBillableResourcesService(MongoDbBillingClient
mongoDbBillingClient) {
- this.mongoDbBillingClient = mongoDbBillingClient;
-
- this.serviceBaseName =
getConfigurationSettingValue(MongoKeyWords.SERVICE_BASE_NAME_KEY)
- .replace('_', '-').toLowerCase();
-
- this.sharedStorageAccountTagName =
getConfigurationSettingValue(MongoKeyWords.SHARED_STORAGE_ACCOUNT_TAG_KEY);
- this.ssnStorageAccountTagName =
getConfigurationSettingValue(MongoKeyWords.SSN_STORAGE_ACCOUNT_TAG_KEY);
- this.azureDataLakeTagName =
getConfigurationSettingValueOrEmpty(MongoKeyWords.DATA_LAKE_TAG_NAME);
-
- }
-
-
- /**
- * Collects billable resources
- *
- * @return set of all billable resources that were created in scope by
DLab from its installation to current time
- */
- public Set<AzureDlabBillableResource> getBillableResources() {
-
- Set<AzureDlabBillableResource> billableResources = new
HashSet<>();
-
- billableResources.addAll(getSsn());
- billableResources.addAll(getDataLake());
- billableResources.addAll(getEdgeAndStorageAccount());
- billableResources.addAll(getNotebooksAndClusters());
-
- List<AzureDlabBillableResource> list = new
ArrayList<>(billableResources);
-
list.sort(Comparator.comparing(AzureDlabBillableResource::getId));
-
- try {
- log.debug("Billable resources is \n {}",
objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString
- (list));
- } catch (JsonProcessingException e) {
- log.debug("Error during pretty printing. Show simple
list", e);
- log.debug("Billable resources is {}", list);
- }
-
- return billableResources;
- }
-
- private String getConfigurationSettingValue(String key) {
-
- Document document =
mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.SETTINGS_COLLECTION)
- .find(Filters.eq(MongoKeyWords.MONGO_ID,
key)).first();
-
- if (document != null) {
- String value = document.getString("value");
- if (StringUtils.isEmpty(value)) {
- throw new IllegalStateException("Configuration
" + key + " does not have value in settings");
- }
- log.info("Key {} has value {}", key, value);
- return value;
- } else {
- throw new IllegalStateException("Configuration " + key
+ " is not present in settings");
- }
-
- }
-
- private String getConfigurationSettingValueOrEmpty(String key) {
- try {
- return getConfigurationSettingValue(key);
- } catch (IllegalStateException e) {
- log.warn("key {} is not found", key, e);
- return null;
- }
- }
-
- private Set<AzureDlabBillableResource> getSsn() {
-
- return Sets.newHashSet(
-
AzureDlabBillableResource.builder().id(serviceBaseName +
"-ssn").type(DlabResourceType.SSN).build(),
-
AzureDlabBillableResource.builder().id(ssnStorageAccountTagName).type(DlabResourceType
- .SSN_STORAGE_ACCOUNT).build(),
-
AzureDlabBillableResource.builder().id(sharedStorageAccountTagName).type(DlabResourceType
-
.COLLABORATION_STORAGE_ACCOUNT).build(),
-
AzureDlabBillableResource.builder().id(serviceBaseName + "-ssn-volume-primary")
-
.type(DlabResourceType.VOLUME).build()
- );
- }
-
- private Set<AzureDlabBillableResource> getDataLake() {
-
- if (azureDataLakeTagName != null) {
- return
Sets.newHashSet(AzureDlabBillableResource.builder().id(azureDataLakeTagName)
-
.type(DlabResourceType.DATA_LAKE_STORE).build());
- }
-
- return Sets.newHashSet();
- }
-
- private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() {
- Set<AzureDlabBillableResource> billableResources = new
HashSet<>();
-
- try {
-
- List<EdgeInfoAzure> edgeInfoList =
objectMapper.readValue(
-
objectMapper.writeValueAsString(mongoDbBillingClient.getDatabase()
-
.getCollection(MongoKeyWords.EDGE_COLLECTION).find()),
- new
com.fasterxml.jackson.core.type.TypeReference<List<EdgeInfoAzure>>() {
- });
-
- if (edgeInfoList != null && !edgeInfoList.isEmpty()) {
- for (EdgeInfoAzure edgeInfoAzure :
edgeInfoList) {
-
billableResources.addAll(getEdgeAndStorageAccount(edgeInfoAzure));
- }
- }
-
- return billableResources;
- } catch (IOException e) {
- log.error("Error during preparation of billable
resources", e);
- }
- return billableResources;
- }
-
- private Set<AzureDlabBillableResource>
getEdgeAndStorageAccount(EdgeInfoAzure edgeInfoAzure) {
-
- Set<AzureDlabBillableResource> billableResources = new
HashSet<>();
-
- if
(StringUtils.isNotEmpty(edgeInfoAzure.getUserContainerName())) {
-
billableResources.add(AzureDlabBillableResource.builder()
-
.id(edgeInfoAzure.getUserStorageAccountTagName())
-
.type(DlabResourceType.EDGE_STORAGE_ACCOUNT)
- .user(edgeInfoAzure.getId()).build());
- }
-
- if (StringUtils.isNotEmpty(edgeInfoAzure.getInstanceId())) {
-
billableResources.add(AzureDlabBillableResource.builder()
- .id(edgeInfoAzure.getInstanceId())
- .type(DlabResourceType.EDGE)
- .user(edgeInfoAzure.getId()).build());
-
-
billableResources.add(AzureDlabBillableResource.builder()
- .id(serviceBaseName + "-" +
edgeUserSimpleName(edgeInfoAzure) + "-edge-volume-primary")
- .type(DlabResourceType.VOLUME)
- .user(edgeInfoAzure.getId()).build());
- }
-
- return billableResources;
- }
-
- private String edgeUserSimpleName(EdgeInfoAzure edgeInfoAzure) {
- return edgeInfoAzure.getId().replaceAll("@.*", "");
- }
-
- private Set<AzureDlabBillableResource> getNotebooksAndClusters() {
-
- Set<AzureDlabBillableResource> billableResources = new
HashSet<>();
-
- try {
- final FindIterable<Document> userInstanceDocuments =
mongoDbBillingClient.getDatabase()
-
.getCollection(MongoKeyWords.NOTEBOOK_COLLECTION)
- .find()
-
.projection(fields(exclude(USER_INSTANCES_EXCLUDED_FIELDS)));
- List<UserInstanceDTO> userInstanceDTOS =
objectMapper.readValue(
-
objectMapper.writeValueAsString(userInstanceDocuments),
- new
com.fasterxml.jackson.core.type.TypeReference<List<UserInstanceDTO>>() {
- });
-
- if (userInstanceDTOS != null &&
!userInstanceDTOS.isEmpty()) {
- userInstanceDTOS.forEach(e ->
billableResources.addAll(getNotebookAndClusters(e)));
- }
-
- } catch (IOException e) {
- log.error("Error during preparation of billable
resources", e);
- }
-
- return billableResources;
- }
-
- private Set<AzureDlabBillableResource>
getNotebookAndClusters(UserInstanceDTO userInstanceDTO) {
- Set<AzureDlabBillableResource> notebookResources = new
HashSet<>();
-
- if (StringUtils.isNotEmpty(userInstanceDTO.getExploratoryId()))
{
-
notebookResources.add(AzureDlabBillableResource.builder()
- .id(userInstanceDTO.getExploratoryId())
- .type(DlabResourceType.EXPLORATORY)
- .user(userInstanceDTO.getUser())
-
.notebookId(userInstanceDTO.getExploratoryId())
-
.resourceName(userInstanceDTO.getExploratoryName()).build());
- notebookResources.addAll(getVolumes(userInstanceDTO,
userInstanceDTO.getExploratoryId(), "Volume primary",
- "Volume secondary"));
-
- if (userInstanceDTO.getResources() != null &&
!userInstanceDTO.getResources().isEmpty()) {
- for (UserComputationalResource
userComputationalResource : userInstanceDTO.getResources()) {
- if
(StringUtils.isNotEmpty(userComputationalResource.getComputationalId())) {
-
-
notebookResources.add(AzureDlabBillableResource.builder()
-
.id(userComputationalResource.getComputationalId())
-
.type(DlabResourceType.COMPUTATIONAL)
-
.user(userInstanceDTO.getUser())
-
.notebookId(userInstanceDTO.getExploratoryId())
-
.resourceName(userComputationalResource.getComputationalName()).build());
- final
List<AzureDlabBillableResource> volumes = getVolumes(userInstanceDTO,
-
userComputationalResource.getComputationalId(),
-
userComputationalResource.getComputationalName() + " volume primary",
-
userComputationalResource.getComputationalName() + " volume secondary");
-
notebookResources.addAll(volumes);
-
- } else {
- log.error("Computational with
empty id {} is found in notebook {}. Skip it.",
-
userComputationalResource, userInstanceDTO);
- }
- }
- }
-
- } else {
- log.error("Notebook {} with empty id id found. Skip
it.", userInstanceDTO);
- }
-
- return notebookResources;
- }
-
- private List<AzureDlabBillableResource> getVolumes(UserInstanceDTO
userInstanceDTO, String exploratoryId, String
- primaryVolumeName, String secondaryVolumeName) {
-
- return Arrays.asList(
- AzureDlabBillableResource.builder()
- .id(exploratoryId +
"-volume-primary")
- .type(DlabResourceType.VOLUME)
- .user(userInstanceDTO.getUser())
-
.notebookId(userInstanceDTO.getExploratoryId())
-
.resourceName(primaryVolumeName).build(),
- AzureDlabBillableResource.builder()
- .id(exploratoryId +
"-volume-secondary")
- .type(DlabResourceType.VOLUME)
- .user(userInstanceDTO.getUser())
-
.notebookId(userInstanceDTO.getExploratoryId())
-
.resourceName(secondaryVolumeName).build()
- );
- }
+ private static final String[] USER_INSTANCES_EXCLUDED_FIELDS =
{"scheduler_data", "last_activity",
+ "computational_resources.scheduler_data",
"computational_resources.last_activity"};
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
+ private MongoDbBillingClient mongoDbBillingClient;
+ private String serviceBaseName;
+ private String sharedStorageAccountTagName;
+ private String ssnStorageAccountTagName;
+ private String azureDataLakeTagName;
+
+ /**
+ * Constructs the service class
+ *
+ * @param mongoDbBillingClient mongodb client to retrieve all billable
resources
+ */
+ public AzureBillableResourcesService(MongoDbBillingClient
mongoDbBillingClient) {
+ this.mongoDbBillingClient = mongoDbBillingClient;
+
+ this.serviceBaseName =
getConfigurationSettingValue(MongoKeyWords.SERVICE_BASE_NAME_KEY)
+ .replace('_', '-').toLowerCase();
+
+ this.sharedStorageAccountTagName =
getConfigurationSettingValue(MongoKeyWords.SHARED_STORAGE_ACCOUNT_TAG_KEY);
+ this.ssnStorageAccountTagName =
getConfigurationSettingValue(MongoKeyWords.SSN_STORAGE_ACCOUNT_TAG_KEY);
+ this.azureDataLakeTagName =
getConfigurationSettingValueOrEmpty(MongoKeyWords.DATA_LAKE_TAG_NAME);
+
+ }
+
+
+ /**
+ * Collects billable resources
+ *
+ * @return set of all billable resources that were created in scope by
DLab from its installation to current time
+ */
+ public Set<AzureDlabBillableResource> getBillableResources() {
+
+ Set<AzureDlabBillableResource> billableResources = new HashSet<>();
+
+ billableResources.addAll(getSsn());
+ billableResources.addAll(getDataLake());
+ billableResources.addAll(getEdgeAndStorageAccount());
+ billableResources.addAll(getNotebooksAndClusters());
+
+ List<AzureDlabBillableResource> list = new
ArrayList<>(billableResources);
+ list.sort(Comparator.comparing(AzureDlabBillableResource::getId));
+
+ try {
+ log.debug("Billable resources is \n {}",
objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString
+ (list));
+ } catch (JsonProcessingException e) {
+ log.debug("Error during pretty printing. Show simple list", e);
+ log.debug("Billable resources is {}", list);
+ }
+
+ return billableResources;
+ }
+
+ private String getConfigurationSettingValue(String key) {
+
+ Document document =
mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.SETTINGS_COLLECTION)
+ .find(Filters.eq(MongoKeyWords.MONGO_ID, key)).first();
+
+ if (document != null) {
+ String value = document.getString("value");
+ if (StringUtils.isEmpty(value)) {
+ throw new IllegalStateException("Configuration " + key + "
does not have value in settings");
+ }
+ log.info("Key {} has value {}", key, value);
+ return value;
+ } else {
+ throw new IllegalStateException("Configuration " + key + " is not
present in settings");
+ }
+
+ }
+
+ private String getConfigurationSettingValueOrEmpty(String key) {
+ try {
+ return getConfigurationSettingValue(key);
+ } catch (IllegalStateException e) {
+ log.warn("key {} is not found", key, e);
+ return null;
+ }
+ }
+
+ private Set<AzureDlabBillableResource> getSsn() {
+
+ return Sets.newHashSet(
+ AzureDlabBillableResource.builder().id(serviceBaseName +
"-ssn").type(DlabResourceType.SSN).build(),
+
AzureDlabBillableResource.builder().id(ssnStorageAccountTagName).type(DlabResourceType
+ .SSN_STORAGE_ACCOUNT).build(),
+
AzureDlabBillableResource.builder().id(sharedStorageAccountTagName).type(DlabResourceType
+ .COLLABORATION_STORAGE_ACCOUNT).build(),
+ AzureDlabBillableResource.builder().id(serviceBaseName +
"-ssn-volume-primary")
+ .type(DlabResourceType.VOLUME).build()
+ );
+ }
+
+ private Set<AzureDlabBillableResource> getDataLake() {
+
+ if (azureDataLakeTagName != null) {
+ return
Sets.newHashSet(AzureDlabBillableResource.builder().id(azureDataLakeTagName)
+ .type(DlabResourceType.DATA_LAKE_STORE).build());
+ }
+
+ return Sets.newHashSet();
+ }
+
+ private Set<AzureDlabBillableResource> getEdgeAndStorageAccount() {
+ Set<AzureDlabBillableResource> billableResources = new HashSet<>();
+
+ try {
+
+ List<EdgeInfoAzure> edgeInfoList = objectMapper.readValue(
+
objectMapper.writeValueAsString(mongoDbBillingClient.getDatabase()
+
.getCollection(MongoKeyWords.EDGE_COLLECTION).find()),
+ new
com.fasterxml.jackson.core.type.TypeReference<List<EdgeInfoAzure>>() {
+ });
+
+ if (edgeInfoList != null && !edgeInfoList.isEmpty()) {
+ for (EdgeInfoAzure edgeInfoAzure : edgeInfoList) {
+
billableResources.addAll(getEdgeAndStorageAccount(edgeInfoAzure));
+ }
+ }
+
+ return billableResources;
+ } catch (IOException e) {
+ log.error("Error during preparation of billable resources", e);
+ }
+ return billableResources;
+ }
+
+ private Set<AzureDlabBillableResource>
getEdgeAndStorageAccount(EdgeInfoAzure edgeInfoAzure) {
+
+ Set<AzureDlabBillableResource> billableResources = new HashSet<>();
+
+ if (StringUtils.isNotEmpty(edgeInfoAzure.getUserContainerName())) {
+ billableResources.add(AzureDlabBillableResource.builder()
+ .id(edgeInfoAzure.getUserStorageAccountTagName())
+ .type(DlabResourceType.EDGE_STORAGE_ACCOUNT)
+ .user(edgeInfoAzure.getId()).build());
+ }
+
+ if (StringUtils.isNotEmpty(edgeInfoAzure.getInstanceId())) {
+ billableResources.add(AzureDlabBillableResource.builder()
+ .id(edgeInfoAzure.getInstanceId())
+ .type(DlabResourceType.EDGE)
+ .user(edgeInfoAzure.getId()).build());
+
+ billableResources.add(AzureDlabBillableResource.builder()
+ .id(serviceBaseName + "-" +
edgeUserSimpleName(edgeInfoAzure) + "-edge-volume-primary")
+ .type(DlabResourceType.VOLUME)
+ .user(edgeInfoAzure.getId()).build());
+ }
+
+ return billableResources;
+ }
+
+ private String edgeUserSimpleName(EdgeInfoAzure edgeInfoAzure) {
+ return edgeInfoAzure.getId().replaceAll("@.*", "");
+ }
+
+ private Set<AzureDlabBillableResource> getNotebooksAndClusters() {
+
+ Set<AzureDlabBillableResource> billableResources = new HashSet<>();
+
+ try {
+ final FindIterable<Document> userInstanceDocuments =
mongoDbBillingClient.getDatabase()
+ .getCollection(MongoKeyWords.NOTEBOOK_COLLECTION)
+ .find()
+
.projection(fields(exclude(USER_INSTANCES_EXCLUDED_FIELDS)));
+ List<UserInstanceDTO> userInstanceDTOS = objectMapper.readValue(
+ objectMapper.writeValueAsString(userInstanceDocuments),
+ new
com.fasterxml.jackson.core.type.TypeReference<List<UserInstanceDTO>>() {
+ });
+
+ if (userInstanceDTOS != null && !userInstanceDTOS.isEmpty()) {
+ userInstanceDTOS.forEach(e ->
billableResources.addAll(getNotebookAndClusters(e)));
+ }
+
+ } catch (IOException e) {
+ log.error("Error during preparation of billable resources", e);
+ }
+
+ return billableResources;
+ }
+
+ private Set<AzureDlabBillableResource>
getNotebookAndClusters(UserInstanceDTO userInstanceDTO) {
+ Set<AzureDlabBillableResource> notebookResources = new HashSet<>();
+
+ if (StringUtils.isNotEmpty(userInstanceDTO.getExploratoryId())) {
+ notebookResources.add(AzureDlabBillableResource.builder()
+ .id(userInstanceDTO.getExploratoryId())
+ .type(DlabResourceType.EXPLORATORY)
+ .user(userInstanceDTO.getUser())
+ .notebookId(userInstanceDTO.getExploratoryId())
+
.resourceName(userInstanceDTO.getExploratoryName()).build());
+ notebookResources.addAll(getVolumes(userInstanceDTO,
userInstanceDTO.getExploratoryId(), "Volume primary",
+ "Volume secondary"));
+
+ if (userInstanceDTO.getResources() != null &&
!userInstanceDTO.getResources().isEmpty()) {
+ for (UserComputationalResource userComputationalResource :
userInstanceDTO.getResources()) {
+ if
(StringUtils.isNotEmpty(userComputationalResource.getComputationalId())) {
+
+
notebookResources.add(AzureDlabBillableResource.builder()
+
.id(userComputationalResource.getComputationalId())
+ .type(DlabResourceType.COMPUTATIONAL)
+ .user(userInstanceDTO.getUser())
+ .notebookId(userInstanceDTO.getExploratoryId())
+
.resourceName(userComputationalResource.getComputationalName()).build());
+ final List<AzureDlabBillableResource> volumes =
getVolumes(userInstanceDTO,
+ userComputationalResource.getComputationalId(),
+
userComputationalResource.getComputationalName() + " volume primary",
+
userComputationalResource.getComputationalName() + " volume secondary");
+ notebookResources.addAll(volumes);
+
+ } else {
+ log.error("Computational with empty id {} is found in
notebook {}. Skip it.",
+ userComputationalResource, userInstanceDTO);
+ }
+ }
+ }
+
+ } else {
+ log.error("Notebook {} with empty id id found. Skip it.",
userInstanceDTO);
+ }
+
+ return notebookResources;
+ }
+
+ private List<AzureDlabBillableResource> getVolumes(UserInstanceDTO
userInstanceDTO, String exploratoryId, String
+ primaryVolumeName, String secondaryVolumeName) {
+
+ return Arrays.asList(
+ AzureDlabBillableResource.builder()
+ .id(exploratoryId + "-volume-primary")
+ .type(DlabResourceType.VOLUME)
+ .user(userInstanceDTO.getUser())
+ .notebookId(userInstanceDTO.getExploratoryId())
+ .resourceName(primaryVolumeName).build(),
+ AzureDlabBillableResource.builder()
+ .id(exploratoryId + "-volume-secondary")
+ .type(DlabResourceType.VOLUME)
+ .user(userInstanceDTO.getUser())
+ .notebookId(userInstanceDTO.getExploratoryId())
+ .resourceName(secondaryVolumeName).build()
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
