This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch DLAB-1571 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 667524ffdfb44984b441081bb6c40bec03408914 Author: Oleh Fuks <[email protected]> AuthorDate: Mon Mar 16 13:33:15 2020 +0200 Conveyed billing for Azure remote endpoint --- services/billing-azure/pom.xml | 109 ++++++++++++--------- .../billing/azure/BillingAzureApplication.java | 36 +++++++ .../billing/azure/config/DlabConfiguration.java | 47 +++++++++ .../dlab/billing/azure/config/SecurityConfig.java | 66 +++++++++++++ .../azure/controller/BillingController.java | 50 ++++++++++ .../epam/dlab/billing/azure/dao/BillingDAO.java | 31 ++++++ .../billing/azure/dao/impl/BillingDAOImpl.java | 109 +++++++++++++++++++++ .../src/main/resources/application.yml | 36 +++++++ 8 files changed, 438 insertions(+), 46 deletions(-) diff --git a/services/billing-azure/pom.xml b/services/billing-azure/pom.xml index 41edc3c..75c8e35 100644 --- a/services/billing-azure/pom.xml +++ b/services/billing-azure/pom.xml @@ -28,16 +28,65 @@ <relativePath>../../pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>billing-azure</artifactId> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-dependencies</artifactId> + <version>2.1.3.RELEASE</version> + <type>pom</type> + <scope>import</scope> + </dependency> + <dependency> + <groupId>org.keycloak.bom</groupId> + <artifactId>keycloak-adapter-bom</artifactId> + <version>4.8.3.Final</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <dependency> - <groupId>com.microsoft.azure</groupId> - <artifactId>azure-client-authentication</artifactId> - <version>1.2.1</version> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-configuration-processor</artifactId> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-data-mongodb</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-security</artifactId> + </dependency> + <dependency> + <groupId>org.keycloak</groupId> + <artifactId>keycloak-spring-boot-starter</artifactId> + </dependency> + <dependency> + <groupId>org.keycloak</groupId> + <artifactId>keycloak-spring-security-adapter</artifactId> + <version>4.8.3.Final</version> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>${org.mockito.version}</version> + <scope>test</scope> </dependency> - <dependency> <groupId>com.epam.dlab</groupId> <artifactId>dlab-model</artifactId> @@ -45,6 +94,12 @@ </dependency> <dependency> + <groupId>com.microsoft.azure</groupId> + <artifactId>azure-client-authentication</artifactId> + <version>1.2.1</version> + </dependency> + + <dependency> <groupId>org.mongodb</groupId> <artifactId>mongo-java-driver</artifactId> <version>${org.mongodb.version}</version> @@ -82,51 +137,13 @@ <build> <plugins> <plugin> - <artifactId>maven-shade-plugin</artifactId> - <version>${maven-shade-plugin.version}</version> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> <executions> <execution> - <phase>package</phase> <goals> - <goal>shade</goal> + <goal>repackage</goal> </goals> - <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - <minimizeJar>true</minimizeJar> - <filters> - <filter> - <artifact>commons-logging:commons-logging</artifact> - <includes>**</includes> - </filter> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>com.epam.dlab.billing.azure.BillingSchedulerAzure</mainClass> - <manifestEntries> - <Created-By><EPAM> Systems</Created-By> - <Name>com/epam/dlab</Name> - <Implementation-Title>DLab Azure Billing Module</Implementation-Title> - <Implementation-Version>${dlab.version}</Implementation-Version> - <Implementation-Vendor><EPAM> Systems</Implementation-Vendor> - <Build-Time>${maven.build.timestamp}</Build-Time> - <Build-OS>${os.name}</Build-OS> - <GIT-Branch>${scmBranch}</GIT-Branch> - <GIT-Commit>${buildNumber}</GIT-Commit> - </manifestEntries> - </transformer> - </transformers> - </configuration> </execution> </executions> </plugin> diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingAzureApplication.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingAzureApplication.java new file mode 100644 index 0000000..1a40767 --- /dev/null +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingAzureApplication.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.billing.azure; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +@SpringBootApplication +@EnableMongoRepositories +@EnableConfigurationProperties +public class BillingAzureApplication { + + public static void main(String[] args) { + SpringApplication.run(BillingAzureApplication.class, args); + } + +} diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/DlabConfiguration.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/DlabConfiguration.java new file mode 100644 index 0000000..c1163c0 --- /dev/null +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/DlabConfiguration.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.billing.azure.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties("dlab") +@Data +public class DlabConfiguration { + private long initialDelay; + private long period; + private String clientId; + private String clientSecret; + private String tenantId; + private String subscriptionId; + private String authenticationFile; + private String offerNumber; + private String currency; + private String locale; + private String regionInfo; + private boolean billingEnabled; + private String ssnStorageAccountTagName; + private String sharedStorageAccountTagName; + private String datalakeTagName; + private LoggingConfigurationFactory logging; + private AggregationOutputMongoDataSource aggregationOutputMongoDataSource; +} diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/SecurityConfig.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/SecurityConfig.java new file mode 100644 index 0000000..35e341c --- /dev/null +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/SecurityConfig.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.billing.azure.config; + +import org.keycloak.adapters.KeycloakConfigResolver; +import org.keycloak.adapters.springboot.KeycloakSpringBootConfigResolver; +import org.keycloak.adapters.springsecurity.KeycloakConfiguration; +import org.keycloak.adapters.springsecurity.authentication.KeycloakAuthenticationProvider; +import org.keycloak.adapters.springsecurity.config.KeycloakWebSecurityConfigurerAdapter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.core.authority.mapping.SimpleAuthorityMapper; +import org.springframework.security.core.session.SessionRegistryImpl; +import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy; +import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy; + +@KeycloakConfiguration +class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter { + + @Autowired + public void configureGlobal(AuthenticationManagerBuilder auth) { + KeycloakAuthenticationProvider keycloakAuthenticationProvider = keycloakAuthenticationProvider(); + keycloakAuthenticationProvider.setGrantedAuthoritiesMapper(new SimpleAuthorityMapper()); + auth.authenticationProvider(keycloakAuthenticationProvider); + } + + @Bean + public KeycloakConfigResolver KeycloakConfigResolver() { + return new KeycloakSpringBootConfigResolver(); + } + + @Bean + @Override + protected SessionAuthenticationStrategy sessionAuthenticationStrategy() { + return new RegisterSessionAuthenticationStrategy(new SessionRegistryImpl()); + } + + @Override + protected void configure(HttpSecurity http) throws Exception { + super.configure(http); + http + .anonymous().disable() + .authorizeRequests() + .anyRequest() + .authenticated(); + } +} \ No newline at end of file diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java new file mode 100644 index 0000000..53227a2 --- /dev/null +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.billing.azure.controller; + +import com.epam.dlab.billing.azure.dao.BillingDAO; +import com.epam.dlab.dto.billing.BillingData; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +public class BillingController { + + private final BillingDAO billingDAO; + + public BillingController(BillingDAO billingDAO) { + this.billingDAO = billingDAO; + } + + @GetMapping + public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) { + return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK); + } + + @GetMapping("/report") + public ResponseEntity<List<BillingData>> getBilling() { + return new ResponseEntity<>(billingDAO.getBillingReport(), HttpStatus.OK); + } +} diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java new file mode 100644 index 0000000..1f578b1 --- /dev/null +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.billing.azure.dao; + +import com.epam.dlab.dto.billing.BillingData; + +import java.util.List; + +public interface BillingDAO { + + List<BillingData> getBillingReport(); + + List<BillingData> getBillingReport(List<String> dlabIds); +} diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java new file mode 100644 index 0000000..34901d0 --- /dev/null +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.epam.dlab.billing.azure.dao.impl; + +import com.epam.dlab.billing.azure.dao.BillingDAO; +import com.epam.dlab.dto.billing.BillingData; +import com.epam.dlab.exceptions.DlabException; +import lombok.extern.slf4j.Slf4j; +import org.bson.Document; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.aggregation.Aggregation; +import org.springframework.data.mongodb.core.aggregation.GroupOperation; +import org.springframework.data.mongodb.core.aggregation.MatchOperation; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.epam.dlab.model.aws.ReportLine.FIELD_COST; +import static com.epam.dlab.model.aws.ReportLine.FIELD_CURRENCY_CODE; +import static com.epam.dlab.model.aws.ReportLine.FIELD_DLAB_ID; +import static com.epam.dlab.model.aws.ReportLine.FIELD_PRODUCT; +import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE; +import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE; +import static org.springframework.data.mongodb.core.aggregation.Aggregation.group; +import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation; + +@Component +@Slf4j +public class BillingDAOImpl implements BillingDAO { + private final MongoTemplate mongoTemplate; + + public BillingDAOImpl(MongoTemplate mongoTemplate) { + this.mongoTemplate = mongoTemplate; + } + + @Override + public List<BillingData> getBillingReport() { + try { + GroupOperation groupOperation = getGroupOperation(); + Aggregation aggregation = newAggregation(groupOperation); + + return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults() + .stream() + .map(this::toBillingData) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Cannot retrieve billing information ", e); + throw new DlabException("Cannot retrieve billing information", e); + } + } + + @Override + public List<BillingData> getBillingReport(List<String> dlabIds) { + try { + GroupOperation groupOperation = getGroupOperation(); + MatchOperation matchOperation = Aggregation.match(Criteria.where("dlab_id").in(dlabIds)); + Aggregation aggregation = newAggregation(matchOperation, groupOperation); + + return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults() + .stream() + .map(this::toBillingData) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Cannot retrieve billing information ", e); + throw new DlabException("Cannot retrieve billing information", e); + } + } + + private GroupOperation getGroupOperation() { + return group(FIELD_PRODUCT, FIELD_CURRENCY_CODE, FIELD_RESOURCE_TYPE, FIELD_DLAB_ID) + .min(FIELD_USAGE_DATE).as("from") + .max(FIELD_USAGE_DATE).as("to") + .sum(FIELD_COST).as(FIELD_COST); + } + + private BillingData toBillingData(Document billingData) { + return BillingData.builder() + .tag(billingData.getString(FIELD_DLAB_ID)) + .usageDateFrom(Optional.ofNullable(billingData.getString("from")).map(LocalDate::parse).orElse(null)) + .usageDateTo(Optional.ofNullable(billingData.getString("to")).map(LocalDate::parse).orElse(null)) + .product(billingData.getString(FIELD_PRODUCT)) + .usageType(billingData.getString(FIELD_RESOURCE_TYPE)) + .cost(BigDecimal.valueOf(billingData.getDouble(FIELD_COST)).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue()) + .currency(billingData.getString(FIELD_CURRENCY_CODE)) + .build(); + } +} diff --git a/services/billing-azure/src/main/resources/application.yml b/services/billing-azure/src/main/resources/application.yml new file mode 100644 index 0000000..52e6862 --- /dev/null +++ b/services/billing-azure/src/main/resources/application.yml @@ -0,0 +1,36 @@ +spring: + main: + allow-bean-definition-overriding: true + data: + mongodb: + username: admin + password: admin + database: dlabdb + port: 27017 + host: localhost +dlab: + + +server: + port: 8088 + servlet: + contextPath: /api/billing + +server.ssl.key-store-type: JKS +server.ssl.key-store: /Users/ofuks/keys/dlabcert/billing.jks +server.ssl.key-store-password: KEYSTORE_PASSWORD +server.ssl.key-alias: billing + +logging: + file: /var/opt/dlab/log/ssn/billing.log + level: + com: + epam: trace + +keycloak: + bearer-only: true + realm: DLAB_bhliva + resource: sss + credentials.secret: cf5a484b-039b-4161-8707-ad65c0f25962 + ssl-required: none + auth-server-url: http://52.11.45.11:8080/auth \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
