This is an automated email from the ASF dual-hosted git repository.

mykolabodnar pushed a commit to branch DLAB-1565-aws
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git

commit b270d012d09e813f077b703b25b41212a2712bba
Author: bodnarmykola <[email protected]>
AuthorDate: Wed Mar 18 13:43:14 2020 +0200

    [DLAB-1565]: [AWS] - billing on remote endpoint implementation
---
 .../terraform/bin/deploy/billing_app_aws.yml       | 34 ++++++++
 .../terraform/bin/deploy/billing_aws.yml           | 98 ++++++++++++++++++++++
 .../terraform/bin/deploy/endpoint_fab.py           | 85 ++++++++++++++++++-
 .../terraform/bin/deploy/supervisor_svc.conf       |  2 +-
 4 files changed, 217 insertions(+), 2 deletions(-)

diff --git 
a/infrastructure-provisioning/terraform/bin/deploy/billing_app_aws.yml 
b/infrastructure-provisioning/terraform/bin/deploy/billing_app_aws.yml
new file mode 100644
index 0000000..be04937
--- /dev/null
+++ b/infrastructure-provisioning/terraform/bin/deploy/billing_app_aws.yml
@@ -0,0 +1,34 @@
+spring:
+  main:
+    allow-bean-definition-overriding: true
+  data:
+    mongodb:
+      username: admin
+      password: MONGO_PASSWORD
+      database: dlabdb
+      port: 27017
+      host: MONGO_HOST
+
+server:
+  port: 8088
+  servlet:
+    contextPath: /api/billing
+
+server.ssl.key-store-type: JKS
+server.ssl.key-store: /home/OS_USER/keys/endpoint.keystore.jks
+server.ssl.key-store-password: KEY_STORE_PASSWORD
+server.ssl.key-alias: billing
+
+logging:
+  file: /var/opt/dlab/log/ssn/billing.log
+  level:
+    com:
+      epam: trace
+
+keycloak:
+  bearer-only: true
+  realm: dlab
+  resource: KEYCLOAK_CLIENT_ID
+  credentials.secret: CLIENT_SECRET
+  ssl-required: none
+  auth-server-url: KEYCLOAK_AUTH_SERVER_URL
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml 
b/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml
new file mode 100644
index 0000000..3531846
--- /dev/null
+++ b/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml
@@ -0,0 +1,98 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# 
******************************************************************************
+
+# Specify the parameters enclosed in angle brackets.
+# Run the follows command to get help for details of configuration:
+# java -jar billing-1.0.jar --help conf
+# java -jar billing-1.0.jar --help {adapter | parser | filter | logappender} 
[name]
+
+billingEnabled: true
+
+host: MONGO_HOST
+port: 27017
+username: admin
+password: MONGO_PASSWORD
+database: dlabdb
+
+scheduler:
+# Schedule is comma separated values of time in format hh[:mm[:ss]]. hh - in 
the 24-hour clock, at 8:15PM is 20:15.
+  schedule: 0:00, 1:00, 2:00, 3:00, 4:00, 5:00, 6:00, 7:00, 8:00, 9:00, 10:00, 
11:00, 12:00, 13:00, 14:00, 15:00, 16:00,
+    17:00, 18:00, 19:00, 20:00, 21:00, 22:00, 23:00
+
+# Adapter for reading source data. Known types: file, s3file
+adapterIn:
+  - type: s3file
+    bucket: BILLING_BUCKET_NAME
+    path: REPORT_PATH
+    awsJobEnabled: AWS_JOB_ENABLED
+    accountId: ACCOUNT_ID
+    accessKeyId: ACCESS_KEY_ID
+    secretAccessKey: SECRET_ACCESS_KEY
+
+# Adapter for writing converted data. Known types: console, file, s3file, 
mongodb
+adapterOut:
+  - type: mongodlab
+    host: MONGO_HOST
+    port: 27017
+    username: admin
+    password: MONGO_PASSWORD
+    database: dlabdb
+#    bufferSize: 10000
+    upsert: true
+
+# Filter for source and converted data.
+filter:
+  - type: aws
+    currencyCode: USD
+    columnDlabTag: CONF_BILLING_TAG
+    serviceBaseName: SERVICE_BASE_NAME
+
+
+# Parser of source data to common format.
+parser:
+  - type: csv
+    headerLineNo: 1
+    skipLines: 1
+    columnMapping: >-
+      dlab_id=<DLAB_ID>;usage_date=<USAGE_DATE>;product=<PRODUCT>;
+      usage_type=<USAGE_TYPE>;usage=<USAGE>;cost=<COST>;
+      resource_id=<RESOURCE_ID>;tags=<TAGS>
+    aggregate: day
+
+
+# Logging configuration.
+logging:
+ # Default logging level
+  level: INFO
+  # Logging levels for appenders.
+  loggers:
+    com.epam: DEBUG
+    org.apache.http: WARN
+    org.mongodb.driver: WARN
+    org.hibernate: WARN
+ # Logging appenders
+  appenders:
+    #- type: console
+    - type: file
+      currentLogFilename: /var/opt/dlab/log/ssn/billing.log
+      archive: true
+      archivedLogFilenamePattern: 
/var/opt/dlab/log/ssn/billing-%d{yyyy-MM-dd}.log.gz
+      archivedFileCount: 10
diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py 
b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
index ceece00..20a36ac 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
@@ -341,8 +341,10 @@ def 
configure_supervisor_endpoint(endpoint_keystore_password):
                                            
'subnet-id'.format(interface)).stdout
                 args.vpc2_id = args.vpc_id
                 args.subnet2_id = args.subnet_id
+                conn.sudo('sed -i 
"s|CONF_PARAMETER|--spring.config.location={0}billing_app.yml --conf {0}|g" 
{1}/tmp/supervisor_svc.conf'
+                          .format(args.dlab_conf_dir, args.dlab_path))
             if args.cloud_provider == 'gcp':
-                conn.sudo('sed -i 
"s|CONF_PARAMETER_NAME|--spring.config.location=|g" {}/tmp/supervisor_svc.conf'
+                conn.sudo('sed -i 
"s|CONF_PARAMETER|--spring.config.location=|g" {}/tmp/supervisor_svc.conf'
                           .format(args.dlab_path))
             conn.sudo('sed -i "s|OS_USR|{}|g" {}/tmp/supervisor_svc.conf'
                       .format(args.os_user, args.dlab_path))
@@ -696,12 +698,93 @@ def 
configure_billing_endpoint(endpoint_keystore_password):
                      .format(args.dlab_path))
             billing_yml_path = "{}/conf/billing.yml".format(args.dlab_path)
             if args.cloud_provider == 'aws':
+
+                conn.put('./billing_app_{}.yml'.format(args.cloud_provider), 
'{}/conf/billing_app.yml'
+                         .format(args.dlab_path))
+                billing_app_yml_path = 
"{}/conf/billing_app.yml".format(args.dlab_path)
+                billing_app_properties = [
+                    {
+                        'key': "MONGO_HOST",
+                        'value': args.mongo_host
+                    },
+                    {
+                        'key': "MONGO_PASSWORD",
+                        'value': args.mongo_password
+                    },
+                    {
+                        'key': "MONGO_PORT",
+                        'value': args.mongo_port
+                    },
+                    {
+                        'key': "OS_USER",
+                        'value': args.os_user
+                    },
+                    {
+                        'key': "KEY_STORE_PASSWORD",
+                        'value': endpoint_keystore_password
+                    },
+                    {
+                        'key': "KEYCLOAK_CLIENT_ID",
+                        'value': args.keycloak_client_id
+                    },
+                    {
+                        'key': "CLIENT_SECRET",
+                        'value': args.keycloak_client_secret
+                    },
+                    {
+                        'key': "KEYCLOAK_AUTH_SERVER_URL",
+                        'value': args.keycloak_auth_server_url
+                    }
+                ]
+                for param in billing_app_properties:
+                    conn.sudo('sed -i "s|{0}|{1}|g" {2}'
+                              .format(param['key'], param['value'], 
billing_app_yml_path))
                 if args.aws_job_enabled == 'true':
                     args.tag_resource_id = 'resourceTags' + ':' + 
args.tag_resource_id
                 billing_properties = [
                     {
+                        'key': "MONGO_HOST",
+                        'value': args.mongo_host
+                    },
+                    {
+                        'key': "MONGO_PASSWORD",
+                        'value': args.mongo_password
+                    },
+                    {
+                        'key': "MONGO_PORT",
+                        'value': args.mongo_port
+                    },
+                    {
                         'key': "BILLING_BUCKET_NAME",
                         'value': args.billing_bucket
+                    },
+                    {
+                        'key': "REPORT_PATH",
+                        'value': args.report_path
+                    },
+                    {
+                        'key': "AWS_JOB_ENABLED",
+                        'value': args.aws_job_enabled
+                    },
+                    {
+                        'key': "ACCOUNT_ID",
+                        'value': args.billing_aws_account_id
+                    },
+                    {
+                        'key': "ACCESS_KEY_ID",
+                        'value': args.access_key_id
+                    },
+                    {
+                        'key': "SECRET_ACCESS_KEY",
+                        'value': args.secret_access_key
+                    },
+                    {
+                        'key': "CONF_BILLING_TAG",
+                        'value': args.billing_tag
+                    },
+                    {
+                        'key': "SERVICE_BASE_NAME",
+                        'value': args.service_base_name
                     }
                 ]
             elif args.cloud_provider == 'gcp':
diff --git 
a/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf 
b/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf
index 9c06695..b170043 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf
+++ b/infrastructure-provisioning/terraform/bin/deploy/supervisor_svc.conf
@@ -35,7 +35,7 @@ redirect_stderr=true
 environment=DLAB_CONF_DIR="WEB_CONF"
 
 [program:billing]
-command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 
WEB_APP_DIR/billing.jar CONF_PARAMETER_NAMEWEB_CONFbilling.yml
+command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 
WEB_APP_DIR/billing.jar CONF_PARAMETERWEB_CONFbilling.yml
 directory=WEB_APP_DIR
 autorestart=true
 priority=20


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to