This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-1565-aws in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit b270d012d09e813f077b703b25b41212a2712bba Author: bodnarmykola <[email protected]> AuthorDate: Wed Mar 18 13:43:14 2020 +0200 [DLAB-1565]: [AWS] - billing on remote endpoint implementation --- .../terraform/bin/deploy/billing_app_aws.yml | 34 ++++++++ .../terraform/bin/deploy/billing_aws.yml | 98 ++++++++++++++++++++++ .../terraform/bin/deploy/endpoint_fab.py | 85 ++++++++++++++++++- .../terraform/bin/deploy/supervisor_svc.conf | 2 +- 4 files changed, 217 insertions(+), 2 deletions(-) diff --git a/infrastructure-provisioning/terraform/bin/deploy/billing_app_aws.yml b/infrastructure-provisioning/terraform/bin/deploy/billing_app_aws.yml new file mode 100644 index 0000000..be04937 --- /dev/null +++ b/infrastructure-provisioning/terraform/bin/deploy/billing_app_aws.yml @@ -0,0 +1,34 @@ +spring: + main: + allow-bean-definition-overriding: true + data: + mongodb: + username: admin + password: MONGO_PASSWORD + database: dlabdb + port: 27017 + host: MONGO_HOST + +server: + port: 8088 + servlet: + contextPath: /api/billing + +server.ssl.key-store-type: JKS +server.ssl.key-store: /home/OS_USER/keys/endpoint.keystore.jks +server.ssl.key-store-password: KEY_STORE_PASSWORD +server.ssl.key-alias: billing + +logging: + file: /var/opt/dlab/log/ssn/billing.log + level: + com: + epam: trace + +keycloak: + bearer-only: true + realm: dlab + resource: KEYCLOAK_CLIENT_ID + credentials.secret: CLIENT_SECRET + ssl-required: none + auth-server-url: KEYCLOAK_AUTH_SERVER_URL \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml b/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml new file mode 100644 index 0000000..3531846 --- /dev/null +++ b/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml @@ -0,0 +1,98 @@ +# ***************************************************************************** +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ****************************************************************************** + +# Specify the parameters enclosed in angle brackets. +# Run the follows command to get help for details of configuration: +# java -jar billing-1.0.jar --help conf +# java -jar billing-1.0.jar --help {adapter | parser | filter | logappender} [name] + +billingEnabled: true + +host: MONGO_HOST +port: 27017 +username: admin +password: MONGO_PASSWORD +database: dlabdb + +scheduler: +# Schedule is comma separated values of time in format hh[:mm[:ss]]. hh - in the 24-hour clock, at 8:15PM is 20:15. + schedule: 0:00, 1:00, 2:00, 3:00, 4:00, 5:00, 6:00, 7:00, 8:00, 9:00, 10:00, 11:00, 12:00, 13:00, 14:00, 15:00, 16:00, + 17:00, 18:00, 19:00, 20:00, 21:00, 22:00, 23:00 + +# Adapter for reading source data. Known types: file, s3file +adapterIn: + - type: s3file + bucket: BILLING_BUCKET_NAME + path: REPORT_PATH + awsJobEnabled: AWS_JOB_ENABLED + accountId: ACCOUNT_ID + accessKeyId: ACCESS_KEY_ID + secretAccessKey: SECRET_ACCESS_KEY + +# Adapter for writing converted data. Known types: console, file, s3file, mongodb +adapterOut: + - type: mongodlab + host: MONGO_HOST + port: 27017 + username: admin + password: MONGO_PASSWORD + database: dlabdb +# bufferSize: 10000 + upsert: true + +# Filter for source and converted data. +filter: + - type: aws + currencyCode: USD + columnDlabTag: CONF_BILLING_TAG + serviceBaseName: SERVICE_BASE_NAME + + +# Parser of source data to common format. +parser: + - type: csv + headerLineNo: 1 + skipLines: 1 + columnMapping: >- + dlab_id=<DLAB_ID>;usage_date=<USAGE_DATE>;product=<PRODUCT>; + usage_type=<USAGE_TYPE>;usage=<USAGE>;cost=<COST>; + resource_id=<RESOURCE_ID>;tags=<TAGS> + aggregate: day + + +# Logging configuration. +logging: + # Default logging level + level: INFO + # Logging levels for appenders. + loggers: + com.epam: DEBUG + org.apache.http: WARN + org.mongodb.driver: WARN + org.hibernate: WARN + # Logging appenders + appenders: + #- type: console + - type: file + currentLogFilename: /var/opt/dlab/log/ssn/billing.log + archive: true + archivedLogFilenamePattern: /var/opt/dlab/log/ssn/billing-%d{yyyy-MM-dd}.log.gz + archivedFileCount: 10 diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py index ceece00..20a36ac 100644 --- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py +++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py @@ -341,8 +341,10 @@ def configure_supervisor_endpoint(endpoint_keystore_password): 'subnet-id'.format(interface)).stdout args.vpc2_id = args.vpc_id args.subnet2_id = args.subnet_id + conn.sudo('sed -i "s|CONF_PARAMETER|--spring.config.location={0}billing_app.yml --conf {0}|g" {1}/tmp/supervisor_svc.conf' + .format(args.dlab_conf_dir, args.dlab_path)) if args.cloud_provider == 'gcp': - conn.sudo('sed -i "s|CONF_PARAMETER_NAME|--spring.config.location=|g" {}/tmp/supervisor_svc.conf' + conn.sudo('sed -i "s|CONF_PARAMETER|--spring.config.location=|g" {}/tmp/supervisor_svc.conf' .format(args.dlab_path)) conn.sudo('sed -i "s|OS_USR|{}|g" {}/tmp/supervisor_svc.conf' .format(args.os_user, args.dlab_path)) @@ -696,12 +698,93 @@ def configure_billing_endpoint(endpoint_keystore_password): .format(args.dlab_path)) billing_yml_path = "{}/conf/billing.yml".format(args.dlab_path) if args.cloud_provider == 'aws': + + conn.put('./billing_app_{}.yml'.format(args.cloud_provider), '{}/conf/billing_app.yml' + .format(args.dlab_path)) + billing_app_yml_path = "{}/conf/billing_app.yml".format(args.dlab_path) + billing_app_properties = [ + { + 'key': "MONGO_HOST", + 'value': args.mongo_host + }, + { + 'key': "MONGO_PASSWORD", + 'value': args.mongo_password + }, + { + 'key': "MONGO_PORT", + 'value': args.mongo_port + }, + { + 'key': "OS_USER", + 'value': args.os_user + }, + { + 'key': "KEY_STORE_PASSWORD", + 'value': endpoint_keystore_password + }, + { + 'key': "KEYCLOAK_CLIENT_ID", + 'value': args.keycloak_client_id + }, + { + 'key': "CLIENT_SECRET", + 'value': args.keycloak_client_secret + }, + { + 'key': "KEYCLOAK_AUTH_SERVER_URL", + 'value': args.keycloak_auth_server_url + } + ] + for param in billing_app_properties: + conn.sudo('sed -i "s|{0}|{1}|g" {2}' + .format(param['key'], param['value'], billing_app_yml_path)) if args.aws_job_enabled == 'true': args.tag_resource_id = 'resourceTags' + ':' + args.tag_resource_id billing_properties = [ { + 'key': "MONGO_HOST", + 'value': args.mongo_host + }, + { + 'key': "MONGO_PASSWORD", + 'value': args.mongo_password + }, + { + 'key': "MONGO_PORT", + 'value': args.mongo_port + }, + { 'key': "BILLING_BUCKET_NAME", 'value': args.billing_bucket + }, + { + 'key': "REPORT_PATH", + 'value': args.report_path + }, + { + 'key': "AWS_JOB_ENABLED", + 'value': args.aws_job_enabled + }, + { + 'key': "ACCOUNT_ID", + 'value': args.billing_aws_account_id + }, + { + 'key': "ACCESS_KEY_ID", + 'value': args.access_key_id + }, + { + 'key': "SECRET_ACCESS_KEY", + 'value': args.secret_access_key + }, + { + 'key': "CONF_BILLING_TAG", + 'value': args.billing_tag + }, + { + 'key': "SERVICE_BASE_NAME", + 'value': args.service_base_name } ] elif args.cloud_provider == 'gcp': diff --git a/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf b/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf index 9c06695..b170043 100644 --- a/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf +++ b/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf @@ -35,7 +35,7 @@ redirect_stderr=true environment=DLAB_CONF_DIR="WEB_CONF" [program:billing] -command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 WEB_APP_DIR/billing.jar CONF_PARAMETER_NAMEWEB_CONFbilling.yml +command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 WEB_APP_DIR/billing.jar CONF_PARAMETERWEB_CONFbilling.yml directory=WEB_APP_DIR autorestart=true priority=20 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
