This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch DLAB-1571 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 4b791b9c16b71f15a4a70649e76fda82851e1ece Author: Oleh Fuks <[email protected]> AuthorDate: Fri Mar 13 12:02:08 2020 +0200 [DLAB-643] Conveyed billing for remote aws endpoint --- pom.xml | 2 +- services/billing-aws/billing.yml | 3 +- services/billing-aws/pom.xml | 128 +++++---- .../java/com/epam/dlab/BillingAwsApplication.java | 37 +++ .../main/java/com/epam/dlab/BillingScheduler.java | 4 +- .../java/com/epam/dlab/conf/SecurityConfig.java | 66 +++++ .../epam/dlab/controller/BillingController.java | 50 ++++ .../main/java/com/epam/dlab/dao/BillingDAO.java | 31 +++ .../com/epam/dlab/dao/impl/BillingDAOImpl.java | 109 ++++++++ .../java/com/epam/dlab/mongo/AdapterMongoDb.java | 9 +- .../com/epam/dlab/mongo/DlabResourceTypeDAO.java | 297 +-------------------- .../billing-aws/src/main/resources/application.yml | 34 +++ 12 files changed, 407 insertions(+), 363 deletions(-) diff --git a/pom.xml b/pom.xml index 853ef15..b6808a8 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ <com.google.inject.version>4.2.0</com.google.inject.version> <dropwizard-template-config.version>1.4.0</dropwizard-template-config.version> <com.aegisql.conveyor.version>1.1.7</com.aegisql.conveyor.version> - <org.mongodb.version>3.3.0</org.mongodb.version> + <org.mongodb.version>3.8.2</org.mongodb.version> <junit.version>4.12</junit.version> <org.mockito.version>1.10.19</org.mockito.version> <java.version>1.8</java.version> diff --git a/services/billing-aws/billing.yml b/services/billing-aws/billing.yml index ee70487..510411f 100644 --- a/services/billing-aws/billing.yml +++ b/services/billing-aws/billing.yml @@ -34,7 +34,8 @@ database: dlabdb scheduler: # Schedule is comma separated values of time in format hh[:mm[:ss]]. hh - in the 24-hour clock, at 8:15PM is 20:15. - schedule: 0:00, 3:00, 6:00, 9:00, 12:00, 15:00, 18:00, 21:00 + schedule: 0:00, 1:00, 2:00, 3:00, 4:00, 5:00, 6:00, 7:00, 8:00, 9:00, 10:00, 11:00, 12:00, 13:00, 14:00, 15:00, 16:00, + 17:00, 18:00, 19:00, 20:00, 21:00, 22:00, 23:00 # Adapter for reading source data. Known types: file, s3file adapterIn: diff --git a/services/billing-aws/pom.xml b/services/billing-aws/pom.xml index a411c20..ec4c830 100644 --- a/services/billing-aws/pom.xml +++ b/services/billing-aws/pom.xml @@ -28,7 +28,7 @@ <version>1.0</version> <relativePath>../../pom.xml</relativePath> </parent> - + <artifactId>billing-aws</artifactId> <properties> @@ -37,6 +37,25 @@ <org.freemarker.version>2.3.22</org.freemarker.version> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>2.1.3.RELEASE</version> + <type>pom</type> + <scope>import</scope> + </dependency> + <dependency> + <groupId>org.keycloak.bom</groupId> + <artifactId>keycloak-adapter-bom</artifactId> + <version>4.8.3.Final</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <dependency> <groupId>com.epam.dlab</groupId> @@ -64,11 +83,6 @@ <version>${org.freemarker.version}</version> </dependency> <dependency> - <groupId>com.epam.dlab</groupId> - <artifactId>dlab-model</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> <groupId>javax.validation</groupId> <artifactId>validation-api</artifactId> <version>2.0.0.Final</version> @@ -119,80 +133,62 @@ <artifactId>guava</artifactId> <version>24.1-jre</version> </dependency> + + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-configuration-processor</artifactId> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-mongodb</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-security</artifactId> + </dependency> + <dependency> + <groupId>org.keycloak</groupId> + <artifactId>keycloak-spring-boot-starter</artifactId> + </dependency> + <dependency> + <groupId>org.keycloak</groupId> + <artifactId>keycloak-spring-security-adapter</artifactId> + <version>4.8.3.Final</version> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <version>${org.mockito.version}</version> <scope>test</scope> </dependency> - + <dependency> + <groupId>com.epam.dlab</groupId> + <artifactId>dlab-model</artifactId> + <version>${project.parent.version}</version> + </dependency> </dependencies> <build> <plugins> - <plugin> - <artifactId>maven-shade-plugin</artifactId> - <version>${maven-shade-plugin.version}</version> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> <executions> <execution> - <phase>package</phase> <goals> - <goal>shade</goal> + <goal>repackage</goal> </goals> - <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - <minimizeJar>false</minimizeJar> - <filters> - <filter> - <artifact>org.hibernate:hibernate-validator</artifact> - <includes>**</includes> - </filter> - <filter> - <artifact>org.glassfish.web:javax.el</artifact> - <includes>**</includes> - </filter> - <filter> - <artifact>org.jboss.logging:jboss-logging</artifact> - <includes>**</includes> - </filter> - <filter> - <artifact>com.fasterxml:classmate</artifact> - <includes>**</includes> - </filter> - - <filter> - <artifact>javax.validation:validation-api</artifact> - <includes>**</includes> - </filter> - <filter> - <artifact>commons-logging:commons-logging</artifact> - <includes>**</includes> - </filter> - <filter> - <artifact>com.fasterxml.jackson.core:jackson-databind</artifact> - <includes>**</includes> - </filter> - </filters> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>com.epam.dlab.BillingScheduler</mainClass> - <manifestEntries> - <Created-By><EPAM> Systems</Created-By> - <Name>com/epam/dlab</Name> - <Implementation-Title>DLab Billing Tool</Implementation-Title> - <Implementation-Version>${dlab.version}</Implementation-Version> - <Implementation-Vendor><EPAM> Systems</Implementation-Vendor> - <Build-Time>${maven.build.timestamp}</Build-Time> - <Build-OS>${os.name}</Build-OS> - <GIT-Branch>${scmBranch}</GIT-Branch> - <GIT-Commit>${buildNumber}</GIT-Commit> - </manifestEntries> - </transformer> - </transformers> - </configuration> </execution> </executions> </plugin> diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java new file mode 100644 index 0000000..cede034 --- /dev/null +++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab; + +import com.epam.dlab.exceptions.InitializationException; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +@SpringBootApplication +@EnableMongoRepositories +@EnableConfigurationProperties +public class BillingAwsApplication { + + public static void main(String[] args) throws InitializationException { + SpringApplication.run(BillingAwsApplication.class, args); + BillingScheduler.startScheduler(args); + } +} diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java index 5db8269..7a7d5ff 100644 --- a/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java +++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java @@ -229,7 +229,7 @@ public class BillingScheduler implements Runnable { * @param args the arguments of command line. * @throws InitializationException */ - public static void main(String[] args) throws InitializationException { + public static void startScheduler(String[] args) throws InitializationException { if (ServiceUtils.printAppVersion(BillingTool.class, args)) { return; } @@ -247,8 +247,6 @@ public class BillingScheduler implements Runnable { } else { throw new InitializationException("Missing the name of configuration file"); } - } else { - throw new InitializationException("Unknow argument: " + args[i]); } } diff --git a/services/billing-aws/src/main/java/com/epam/dlab/conf/SecurityConfig.java b/services/billing-aws/src/main/java/com/epam/dlab/conf/SecurityConfig.java new file mode 100644 index 0000000..dba4086 --- /dev/null +++ b/services/billing-aws/src/main/java/com/epam/dlab/conf/SecurityConfig.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.conf; + +import org.keycloak.adapters.KeycloakConfigResolver; +import org.keycloak.adapters.springboot.KeycloakSpringBootConfigResolver; +import org.keycloak.adapters.springsecurity.KeycloakConfiguration; +import org.keycloak.adapters.springsecurity.authentication.KeycloakAuthenticationProvider; +import org.keycloak.adapters.springsecurity.config.KeycloakWebSecurityConfigurerAdapter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.core.authority.mapping.SimpleAuthorityMapper; +import org.springframework.security.core.session.SessionRegistryImpl; +import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy; +import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy; + +@KeycloakConfiguration +class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter { + + @Autowired + public void configureGlobal(AuthenticationManagerBuilder auth) { + KeycloakAuthenticationProvider keycloakAuthenticationProvider = keycloakAuthenticationProvider(); + keycloakAuthenticationProvider.setGrantedAuthoritiesMapper(new SimpleAuthorityMapper()); + auth.authenticationProvider(keycloakAuthenticationProvider); + } + + @Bean + public KeycloakConfigResolver KeycloakConfigResolver() { + return new KeycloakSpringBootConfigResolver(); + } + + @Bean + @Override + protected SessionAuthenticationStrategy sessionAuthenticationStrategy() { + return new RegisterSessionAuthenticationStrategy(new SessionRegistryImpl()); + } + + @Override + protected void configure(HttpSecurity http) throws Exception { + super.configure(http); + http + .anonymous().disable() + .authorizeRequests() + .anyRequest() + .authenticated(); + } +} \ No newline at end of file diff --git a/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java b/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java new file mode 100644 index 0000000..82d6d16 --- /dev/null +++ b/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.controller; + +import com.epam.dlab.dao.BillingDAO; +import com.epam.dlab.dto.billing.BillingData; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +public class BillingController { + + private final BillingDAO billingDAO; + + public BillingController(BillingDAO billingDAO) { + this.billingDAO = billingDAO; + } + + @GetMapping + public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) { + return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK); + } + + @GetMapping("/report") + public ResponseEntity<List<BillingData>> getBilling() { + return new ResponseEntity<>(billingDAO.getBillingReport(), HttpStatus.OK); + } +} diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java b/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java new file mode 100644 index 0000000..8ecaddf --- /dev/null +++ b/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.dao; + +import com.epam.dlab.dto.billing.BillingData; + +import java.util.List; + +public interface BillingDAO { + + List<BillingData> getBillingReport(); + + List<BillingData> getBillingReport(List<String> dlabIds); +} diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java b/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java new file mode 100644 index 0000000..4fe4606 --- /dev/null +++ b/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.dao.impl; + +import com.epam.dlab.dao.BillingDAO; +import com.epam.dlab.dto.billing.BillingData; +import com.epam.dlab.exceptions.DlabException; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.GroupOperation; +import org.springframework.data.mongodb.core.aggregation.MatchOperation; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.epam.dlab.model.aws.ReportLine.FIELD_COST; +import static com.epam.dlab.model.aws.ReportLine.FIELD_CURRENCY_CODE; +import static com.epam.dlab.model.aws.ReportLine.FIELD_DLAB_ID; +import static com.epam.dlab.model.aws.ReportLine.FIELD_PRODUCT; +import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE; +import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE; +import static org.springframework.data.mongodb.core.aggregation.Aggregation.group; +import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation; + +@Component +@Slf4j +public class BillingDAOImpl implements BillingDAO { + private final MongoTemplate mongoTemplate; + + public BillingDAOImpl(MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @Override + public List<BillingData> getBillingReport() { + try { + GroupOperation groupOperation = getGroupOperation(); + Aggregation aggregation = newAggregation(groupOperation); + + return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults() + .stream() + .map(this::toBillingData) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Cannot retrieve billing information ", e); + throw new DlabException("Cannot retrieve billing information", e); + } + } + + @Override + public List<BillingData> getBillingReport(List<String> dlabIds) { + try { + GroupOperation groupOperation = getGroupOperation(); + MatchOperation matchOperation = Aggregation.match(Criteria.where("dlab_id").in(dlabIds)); + Aggregation aggregation = newAggregation(matchOperation, groupOperation); + + return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults() + .stream() + .map(this::toBillingData) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Cannot retrieve billing information ", e); + throw new DlabException("Cannot retrieve billing information", e); + } + } + + private GroupOperation getGroupOperation() { + return group(FIELD_PRODUCT, FIELD_CURRENCY_CODE, FIELD_RESOURCE_TYPE, FIELD_DLAB_ID) + .min(FIELD_USAGE_DATE).as("from") + .max(FIELD_USAGE_DATE).as("to") + .sum(FIELD_COST).as(FIELD_COST); + } + + private BillingData toBillingData(Document billingData) { + return BillingData.builder() + .tag(billingData.getString(FIELD_DLAB_ID)) + .usageDateFrom(Optional.ofNullable(billingData.getString("from")).map(LocalDate::parse).orElse(null)) + .usageDateTo(Optional.ofNullable(billingData.getString("to")).map(LocalDate::parse).orElse(null)) + .product(billingData.getString(FIELD_PRODUCT)) + .usageType(billingData.getString(FIELD_RESOURCE_TYPE)) + .cost(BigDecimal.valueOf(billingData.getDouble(FIELD_COST)).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue()) + .currency(billingData.getString(FIELD_CURRENCY_CODE)) + .build(); + } +} diff --git a/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java b/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java index a13ed05..e0af4e6 100644 --- a/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java +++ b/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java @@ -191,7 +191,7 @@ public class AdapterMongoDb extends DBAdapterBase { } @Override - public void writeHeader(List<String> header) throws AdapterException { + public void writeHeader(List<String> header) { // Nothing to do } @@ -235,13 +235,6 @@ public class AdapterMongoDb extends DBAdapterBase { } catch (Exception e) { throw new AdapterException("Cannot update total monthly cost. " + e.getLocalizedMessage(), e); } - try { - if (months.size() > 0) { - resourceTypeDAO.updateExploratoryCost(); - } - } catch (Exception e) { - throw new AdapterException("Cannot update total cost of exploratory. " + e.getLocalizedMessage(), e); - } } @Override diff --git a/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java b/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java index dcfe369..0c95605 100644 --- a/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java +++ b/services/billing-aws/src/main/java/com/epam/dlab/mongo/DlabResourceTypeDAO.java @@ -19,28 +19,28 @@ package com.epam.dlab.mongo; -import com.epam.dlab.billing.BillingCalculationUtils; -import com.epam.dlab.billing.DlabResourceType; -import com.epam.dlab.dto.base.DataEngineType; import com.epam.dlab.exceptions.InitializationException; import com.epam.dlab.exceptions.ParseException; import com.epam.dlab.model.aws.ReportLine; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Updates; -import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.text.SimpleDateFormat; -import java.util.*; - -import static com.mongodb.client.model.Accumulators.*; -import static com.mongodb.client.model.Aggregates.*; -import static com.mongodb.client.model.Filters.*; -import static com.mongodb.client.model.Projections.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static com.mongodb.client.model.Accumulators.sum; +import static com.mongodb.client.model.Aggregates.group; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Filters.and; +import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.gte; +import static com.mongodb.client.model.Filters.lte; import static org.apache.commons.lang3.StringUtils.EMPTY; /** @@ -48,8 +48,6 @@ import static org.apache.commons.lang3.StringUtils.EMPTY; */ public class DlabResourceTypeDAO implements MongoConstants { private static final Logger LOGGER = LoggerFactory.getLogger(DlabResourceTypeDAO.class); - private static final String VOLUME_PRIMARY_SUFFIX = "-volume-primary"; - private static final String VOLUME_SECONDARY_SUFFIX = "-volume-secondary"; /** * Mongo database connection. @@ -63,11 +61,6 @@ public class DlabResourceTypeDAO implements MongoConstants { private String serviceBaseNameId; /** - * Describe all DLab resources: SSN, EDGE, exploratory, computational and buckets. - */ - private ResourceItemList resourceList; - - /** * Instantiate DAO for billing resources. * * @param connection the connection to Mongo DB. @@ -76,7 +69,6 @@ public class DlabResourceTypeDAO implements MongoConstants { public DlabResourceTypeDAO(MongoDbConnection connection) throws InitializationException { this.connection = connection; setServiceBaseName(); - setResourceList(); } /** @@ -110,131 +102,6 @@ public class DlabResourceTypeDAO implements MongoConstants { } /** - * Return DLab resources from Mongo DB. - * - * @throws InitializationException - */ - public ResourceItemList getResourceList() { - return resourceList; - } - - /** - * Load and return DLab resources from Mongo DB. - * - * @throws InitializationException - */ - private void setResourceList() { - resourceList = new ResourceItemList(); - - // Add SSN - String sbName = getServiceBaseName(); - resourceList.append(sbName + "-ssn", "SSN", DlabResourceType.SSN); - resourceList.append(sbName + "-ssn-volume-primary", "SSN volume", DlabResourceType.VOLUME); - resourceList.append(sbName + "-ssn-bucket", "SSN bucket", DlabResourceType.SSN_BUCKET); - - // collaboration bucket - resourceList.append(sbName + "-shared-bucket", "Collaboration bucket", DlabResourceType - .COLLABORATION_BUCKET); - - // Add PROJECTS - Bson projection = fields(include("name", "endpoints")); - Iterable<Document> docs = connection.getCollection("Projects").find().projection(projection); - for (Document d : docs) { - String projectName = d.getString("name"); - ((List<Document>) d.get("endpoints")) - .stream() - .map(endpoint -> endpoint.getString("name")) - .forEach(endpoint -> { - resourceList.append(sbName + "-" + endpoint + "-shared-bucket", "Shared endpoint bucket", - DlabResourceType.COLLABORATION_BUCKET); - resourceList.append(sbName + "-" + projectName + "-" + endpoint + "-bucket", "Project bucket", - DlabResourceType.COLLABORATION_BUCKET, null, null, projectName); - resourceList.append(sbName + "-" + projectName + "-" + endpoint + "-edge", "EDGE Node", - DlabResourceType.EDGE, null, null, projectName); - resourceList.append(sbName + "-" + projectName+ "-" + endpoint + "-edge-volume-primary", - "EDGE Volume", DlabResourceType.VOLUME, null, null, projectName); - }); - } - - // Add exploratory - projection = fields(include(FIELD_USER, - FIELD_EXPLORATORY_NAME, - FIELD_EXPLORATORY_ID, - FIELD_PROJECT, - FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_COMPUTATIONAL_ID, - FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_COMPUTATIONAL_NAME, - FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_IMAGE, - FIELD_COMPUTATIONAL_RESOURCES + "." + FIELD_DATAENGINE_INSTANCE_COUNT), - excludeId()); - docs = connection.getCollection(COLLECTION_USER_INSTANCES).find().projection(projection); - for (Document exp : docs) { - String username = exp.getString(FIELD_USER); - String exploratoryName = exp.getString(FIELD_EXPLORATORY_NAME); - String exploratoryId = exp.getString(FIELD_EXPLORATORY_ID); - String project = exp.getString(FIELD_PROJECT); - resourceList.append(exploratoryId, exploratoryName, DlabResourceType.EXPLORATORY, username, - exploratoryName, project); - appendExploratoryVolumes(username, exploratoryName, exploratoryId, project); - - // Add computational - @SuppressWarnings("unchecked") - List<Document> compList = (List<Document>) exp.get(FIELD_COMPUTATIONAL_RESOURCES); - if (compList == null) { - continue; - } - for (Document comp : compList) { - String computationalId = comp.getString(FIELD_COMPUTATIONAL_ID); - String computationalName = comp.getString(FIELD_COMPUTATIONAL_NAME); - final DataEngineType dataEngineType = DataEngineType.fromDockerImageName(comp.getString(FIELD_IMAGE)); - resourceList.append(computationalId, computationalName, DlabResourceType.COMPUTATIONAL, username, - exploratoryName, project); - if (DataEngineType.CLOUD_SERVICE == dataEngineType) { - appendDataengineServiceVolumes(username, exploratoryName, computationalId, computationalName, - project); - } else { - appendDataengineVolumes(username, exploratoryName, comp, computationalId, computationalName, - project); - } - } - } - LOGGER.debug("resourceList is {}", resourceList); - } - - private void appendExploratoryVolumes(String username, String exploratoryName, String exploratoryId, - String project) { - resourceList.append(exploratoryId + VOLUME_PRIMARY_SUFFIX, "Volume primary", DlabResourceType.VOLUME, - username, exploratoryName, project); - resourceList.append(exploratoryId + VOLUME_SECONDARY_SUFFIX, "Volume secondary", DlabResourceType.VOLUME, - username, exploratoryName, project); - } - - private void appendDataengineServiceVolumes(String username, String exploratoryName, String computationalId, - String computationalName, String project) { - resourceList.append(computationalId + VOLUME_PRIMARY_SUFFIX, computationalName + " volume primary", - DlabResourceType.VOLUME, username, exploratoryName, project); - resourceList.append(computationalId + VOLUME_SECONDARY_SUFFIX, computationalName + " volume secondary", - DlabResourceType.VOLUME, username, exploratoryName, project); - } - - private void appendDataengineVolumes(String username, String exploratoryName, Document comp, String - computationalId, String computationalName, String project) { - resourceList.append(computationalId + "-m-volume-primary", computationalName + " master volume primary", - DlabResourceType.VOLUME, username, exploratoryName, project); - resourceList.append(computationalId + "-m-volume-secondary", computationalName + " master volume secondary", - DlabResourceType.VOLUME, username, exploratoryName, project); - final Integer instanceCount = Integer.valueOf(comp.getString(FIELD_DATAENGINE_INSTANCE_COUNT)); - for (int i = instanceCount - 1; i > 0; i--) { - final String slaveId = computationalId + "-s" + i; - final String slaveName = computationalName + "-s" + i; - resourceList.append(slaveId + VOLUME_PRIMARY_SUFFIX, slaveName + " volume primary", DlabResourceType - .VOLUME, username, exploratoryName, project); - resourceList.append(slaveId + VOLUME_SECONDARY_SUFFIX, slaveName + " volume secondary", DlabResourceType - .VOLUME, username, exploratoryName, project); - } - } - - - /** * Convert and return the report line of billing to Mongo document. * * @param row report line. @@ -248,21 +115,7 @@ public class DlabResourceTypeDAO implements MongoConstants { resourceId); } resourceId = resourceId.substring(serviceBaseNameId.length()); - - ResourceItem resource = resourceList.getById(resourceId); Document d = new Document(ReportLine.FIELD_DLAB_ID, resourceId); - if (resource == null) { - d.put(FIELD_DLAB_RESOURCE_ID, null); - d.put(FIELD_DLAB_RESOURCE_TYPE, null); - d.put(ReportLine.FIELD_USER_ID, null); - d.put(FIELD_EXPLORATORY_NAME, null); - } else { - d.put(FIELD_DLAB_RESOURCE_ID, resource.getResourceId()); - d.put(FIELD_DLAB_RESOURCE_TYPE, resource.getType().toString()); - d.put(ReportLine.FIELD_USER_ID, resource.getUser()); - d.put(FIELD_EXPLORATORY_NAME, resource.getExploratoryName()); - d.put(FIELD_PROJECT, resource.getProject()); - } return d.append(ReportLine.FIELD_USAGE_DATE, row.getUsageDate()) .append(ReportLine.FIELD_PRODUCT, row.getProduct()) .append(ReportLine.FIELD_USAGE_TYPE, row.getUsageType()) @@ -280,7 +133,7 @@ public class DlabResourceTypeDAO implements MongoConstants { * * @param fieldNames the list of field names. */ - private Document getGrouppingFields(String... fieldNames) { + private Document getGroupingFields(String... fieldNames) { Document d = new Document(); for (String name : fieldNames) { d.put(name, "$" + name); @@ -307,7 +160,7 @@ public class DlabResourceTypeDAO implements MongoConstants { List<? extends Bson> pipeline = Arrays.asList( match(and(gte(ReportLine.FIELD_USAGE_DATE, month + "-01"), lte(ReportLine.FIELD_USAGE_DATE, month + "-31"))), - group(getGrouppingFields(FIELD_DLAB_RESOURCE_ID, + group(getGroupingFields(FIELD_DLAB_RESOURCE_ID, FIELD_DLAB_RESOURCE_TYPE, FIELD_USER, FIELD_EXPLORATORY_NAME, @@ -333,128 +186,4 @@ public class DlabResourceTypeDAO implements MongoConstants { collection.insertMany(totals); } } - - /** - * Comparator to sort billing exploratory details. - */ - class BillingComparator implements Comparator<Document> { - @Override - public int compare(Document d1, Document d2) { - int result = StringUtils.compare(d1.getString(FIELD_RESOURCE_NAME), d2.getString(FIELD_RESOURCE_NAME)); - if (result == 0) { - result = StringUtils.compare(d1.getString(ReportLine.FIELD_PRODUCT), d2.getString(ReportLine - .FIELD_PRODUCT)); - if (result == 0) { - return StringUtils.compare(d1.getString(ReportLine.FIELD_RESOURCE_TYPE), d2.getString(ReportLine - .FIELD_RESOURCE_TYPE)); - } - } - return result; - } - } - - /** - * Update exploratory cost in Mongo DB. - * - * @param user the name of user. - * @param exploratoryName id of exploratory. - */ - private void updateExploratoryCost(String user, String exploratoryName) { - LOGGER.debug("Update explorartory {} cost for user {}", exploratoryName, user); - List<? extends Bson> pipeline = Arrays.asList( - match(and(eq(FIELD_USER, user), - eq(FIELD_EXPLORATORY_NAME, exploratoryName))), - group(getGrouppingFields(FIELD_DLAB_RESOURCE_ID, - ReportLine.FIELD_PRODUCT, - ReportLine.FIELD_RESOURCE_TYPE, - ReportLine.FIELD_CURRENCY_CODE), - sum(ReportLine.FIELD_COST, "$" + ReportLine.FIELD_COST), - min(FIELD_USAGE_DATE_START, "$" + ReportLine.FIELD_USAGE_DATE), - max(FIELD_USAGE_DATE_END, "$" + ReportLine.FIELD_USAGE_DATE) - ), - sort(new Document(FIELD_ID + "." + FIELD_DLAB_RESOURCE_ID, 1).append(FIELD_ID + "." + ReportLine - .FIELD_PRODUCT, 1)) - ); - AggregateIterable<Document> docs = connection.getCollection(COLLECTION_BILLING) - .aggregate(pipeline); - LinkedList<Document> billing = new LinkedList<>(); - ResourceItemList resources = getResourceList(); - Double costTotal = null; - String currencyCode = null; - for (Document d : docs) { - Document id = (Document) d.get(FIELD_ID); - double cost = BillingCalculationUtils.round(d.getDouble(ReportLine.FIELD_COST), 2); - costTotal = (costTotal == null ? cost : costTotal + cost); - if (currencyCode == null) { - currencyCode = id.getString(ReportLine.FIELD_CURRENCY_CODE); - } - - Document total = new Document() - .append(FIELD_RESOURCE_NAME, resources.getById(id.getString(FIELD_DLAB_RESOURCE_ID)) - .getResourceName()) - .append(ReportLine.FIELD_PRODUCT, id.getString(ReportLine.FIELD_PRODUCT)) - .append(ReportLine.FIELD_RESOURCE_TYPE, id.getString(ReportLine.FIELD_RESOURCE_TYPE)) - .append(ReportLine.FIELD_COST, BillingCalculationUtils.formatDouble(cost)) - .append(ReportLine.FIELD_CURRENCY_CODE, id.getString(ReportLine.FIELD_CURRENCY_CODE)) - .append(FIELD_USAGE_DATE_START, d.getString(FIELD_USAGE_DATE_START)) - .append(FIELD_USAGE_DATE_END, d.getString(FIELD_USAGE_DATE_END)); - billing.add(total); - } - - LOGGER.debug("Total explorartory {} cost for user {} is {} {}, detail count is {}", - exploratoryName, user, costTotal, currencyCode, billing.size()); - billing.sort(new BillingComparator()); - - MongoCollection<Document> cExploratory = connection.getCollection(COLLECTION_USER_INSTANCES); - Bson values = Updates.combine( - Updates.set(ReportLine.FIELD_COST, BillingCalculationUtils.formatDouble(costTotal)), - Updates.set(FIELD_CURRENCY_CODE, currencyCode), - Updates.set(COLLECTION_BILLING, (!billing.isEmpty() ? billing : null))); - cExploratory.updateOne( - and(and(eq(FIELD_USER, user), - eq(FIELD_EXPLORATORY_NAME, exploratoryName))), - values); - } - - /** - * Update EDGE cost in Mongo DB. - * - * @param user the name of user. - */ - private void updateEdgeCost(String user) { - List<? extends Bson> pipeline = Arrays.asList( - match(and(eq(FIELD_USER, user), - eq(FIELD_EXPLORATORY_NAME, null))), - group(getGrouppingFields(ReportLine.FIELD_CURRENCY_CODE), - sum(ReportLine.FIELD_COST, "$" + ReportLine.FIELD_COST)) - ); - AggregateIterable<Document> docs = connection.getCollection(COLLECTION_BILLING_TOTAL) - .aggregate(pipeline); - - MongoCollection<Document> cEdge = connection.getCollection(COLLECTION_USER_EDGE); - for (Document d : docs) { - Document id = (Document) d.get(FIELD_ID); - Bson values = Updates.combine( - Updates.set(ReportLine.FIELD_COST, BillingCalculationUtils.round(d.getDouble(ReportLine - .FIELD_COST), 2)), - Updates.set(FIELD_CURRENCY_CODE, id.get(ReportLine.FIELD_CURRENCY_CODE))); - cEdge.updateOne( - eq(FIELD_ID, user), - values); - } - } - - /** - * Update the cost of exploratory environment for all users in Mongo DB. - */ - public void updateExploratoryCost() { - for (int i = 0; i < resourceList.size(); i++) { - ResourceItem item = resourceList.get(i); - if (item.getType() == DlabResourceType.EXPLORATORY) { - updateExploratoryCost(item.getUser(), item.getExploratoryName()); - } else if (item.getType() == DlabResourceType.EDGE) { - updateEdgeCost(item.getUser()); - } - } - } } diff --git a/services/billing-aws/src/main/resources/application.yml b/services/billing-aws/src/main/resources/application.yml new file mode 100644 index 0000000..b31f91a --- /dev/null +++ b/services/billing-aws/src/main/resources/application.yml @@ -0,0 +1,34 @@ +spring: + main: + allow-bean-definition-overriding: true + data: + mongodb: + username: admin + password: admin + database: dlabdb + port: 27017 + host: localhost + +server: + port: 8088 + servlet: + contextPath: /api/billing + +server.ssl.key-store-type: JKS +server.ssl.key-store: /Users/ofuks/keys/dlabcert/billing.jks +server.ssl.key-store-password: KEYSTORE_PASSWORD +server.ssl.key-alias: billing + +logging: + file: /var/opt/dlab/log/ssn/billing.log + level: + com: + epam: trace + +keycloak: + bearer-only: true + realm: DLAB_bhliva + resource: sss + credentials.secret: cf5a484b-039b-4161-8707-ad65c0f25962 + ssl-required: none + auth-server-url: http://52.11.45.11:8080/auth \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
