This is an automated email from the ASF dual-hosted git repository.

mykolabodnar pushed a commit to branch DLAB-1565-aws
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git


The following commit(s) were added to refs/heads/DLAB-1565-aws by this push:
     new 2c2b713  [DLAB-1565]: [GCP] - billing on local endpoint updated
2c2b713 is described below

commit 2c2b713c4478e6692345e08234da81316f60538c
Author: bodnarmykola <[email protected]>
AuthorDate: Tue Mar 31 15:48:23 2020 +0300

    [DLAB-1565]: [GCP] - billing on local endpoint updated
---
 .../src/general/lib/os/debian/ssn_lib.py           |  24 +++-
 .../src/general/scripts/gcp/ssn_configure.py       |   5 +-
 .../src/ssn/scripts/configure_billing.py           | 125 ++++++++++++++-------
 .../src/ssn/scripts/configure_ui.py                |   8 +-
 services/billing-gcp/billing.yml                   |  34 ++++--
 5 files changed, 144 insertions(+), 52 deletions(-)

diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py 
b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index bb073c3..488d8d7 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -180,7 +180,9 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
              locale, region_info, ldap_login, tenant_id,
              application_id, hostname, data_lake_name, subscription_id,
              validate_permission_scope, dlab_id, usage_date, product,
-             usage_type, usage, cost, resource_id, tags, billing_dataset_name, 
report_path=''):
+             usage_type, usage, cost, resource_id, tags, billing_dataset_name, 
report_path='', mongo_host,
+             keycloak_client_id, keycloak_client_secret,
+             keycloak_auth_server_url):
     try:
         if not exists(os.environ['ssn_dlab_path'] + 'tmp/ss_started'):
             java_path = sudo("update-alternatives --query java | grep 'Value: 
' | grep -o '/.*/jre'")
@@ -285,7 +287,15 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
                          '--cost {} ' \
                          '--resource_id {} ' \
                          '--tags {} ' \
-                         '--billing_dataset_name "{}" '.\
+                         '--billing_dataset_name "{}" '\
+                         '--mongo_host {} ' \
+                         '--mongo_port {} ' \
+                         '--service_base_name {} ' \
+                         '--os_user {} ' \
+                         '--keystore_password {} ' \
+                         '--keycloak_client_id {} ' \
+                         '--keycloak_client_secret {} ' \
+                         '--keycloak_auth_server_url {} '.\
                             format(cloud_provider,
                                    service_base_name,
                                    tag_resource_id,
@@ -309,7 +319,15 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
                                    cost,
                                    resource_id,
                                    tags,
-                                   billing_dataset_name)
+                                   billing_dataset_name,
+                                   mongo_host,
+                                   mongo_port,
+                                   service_base_name,
+                                   os_user,
+                                   keystore_passwd,
+                                   keycloak_client_id,
+                                   keycloak_client_secret,
+                                   keycloak_auth_server_url)
                 sudo('python /tmp/configure_billing.py {}'.format(params))
             try:
                 if os.environ['conf_stepcerts_enabled'] == 'true':
diff --git 
a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py 
b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index 1fcbd03..54b9781 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -457,12 +457,13 @@ if __name__ == "__main__":
         params = "--hostname {} --keyfile {} --dlab_path {} --os_user {} 
--os_family {} --billing_enabled {} " \
                  "--request_id {} --billing_dataset_name {} \
                  --resource {} --service_base_name {} --cloud_provider {} 
--default_endpoint_name {} " \
-                 "--cloud_params '{}'". \
+                 "--cloud_params '{}' --keycloak_client_id {} 
--keycloak_client_secret {} --keycloak_auth_server_url {}". \
             format(instance_hostname, ssn_conf['ssh_key_path'], 
os.environ['ssn_dlab_path'], ssn_conf['dlab_ssh_user'],
                    os.environ['conf_os_family'], billing_enabled, 
os.environ['request_id'],
                    os.environ['billing_dataset_name'], 
os.environ['conf_resource'],
                    ssn_conf['service_base_name'], 
os.environ['conf_cloud_provider'], ssn_conf['default_endpoint_name'],
-                   json.dumps(cloud_params))
+                   json.dumps(cloud_params), 
os.environ['keycloak_client_name'], os.environ['keycloak_client_secret'],
+                   os.environ['keycloak_auth_server_url'])
         try:
             local("~/scripts/{}.py {}".format('configure_ui', params))
         except:
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py 
b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
index 9f40e72..c87af74 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
@@ -49,15 +49,24 @@ parser.add_argument('--locale', type=str, default='', 
help='Azure locale')
 parser.add_argument('--region_info', type=str, default='', help='Azure region 
info')
 parser.add_argument('--mongo_password', type=str, help='The password for Mongo 
DB')
 parser.add_argument('--dlab_dir', type=str, help='The path to dlab dir')
-parser.add_argument('--dlab_id', type=str, default='', help='Column name in 
report file that contains dlab id tag')
-parser.add_argument('--usage_date', type=str, default='', help='Column name in 
report file that contains usage date tag')
-parser.add_argument('--product', type=str, default='', help='Column name in 
report file that contains product name tag')
-parser.add_argument('--usage_type', type=str, default='', help='Column name in 
report file that contains usage type tag')
-parser.add_argument('--usage', type=str, default='', help='Column name in 
report file that contains usage tag')
-parser.add_argument('--cost', type=str, default='', help='Column name in 
report file that contains cost tag')
-parser.add_argument('--resource_id', type=str, default='', help='Column name 
in report file that contains dlab resource id tag')
-parser.add_argument('--tags', type=str, default='', help='Column name in 
report file that contains tags')
+parser.add_argument('--dlab_id', type=str, 
default='resource_tags_user_user_tag', help='Column name in report file that 
contains dlab id tag')
+parser.add_argument('--usage_date', type=str, 
default='line_item_usage_start_date', help='Column name in report file that 
contains usage date tag')
+parser.add_argument('--product', type=str, default='product_product_name', 
help='Column name in report file that contains product name tag')
+parser.add_argument('--usage_type', type=str, default='line_item_usage_type', 
help='Column name in report file that contains usage type tag')
+parser.add_argument('--usage', type=str, default='line_item_usage_amount', 
help='Column name in report file that contains usage tag')
+parser.add_argument('--cost', type=str, default='line_item_blended_cost', 
help='Column name in report file that contains cost tag')
+parser.add_argument('--resource_id', type=str, 
default='line_item_resource_id', help='Column name in report file that contains 
dlab resource id tag')
+parser.add_argument('--tags', type=str, 
default='line_item_operation,line_item_line_item_description', help='Column 
name in report file that contains tags')
 parser.add_argument('--billing_dataset_name', type=str, default='', help='Name 
of gcp billing dataset (in big query service')
+
+parser.add_argument('--mongo_host', type=str, default='localhost', help='Mongo 
DB host')
+parser.add_argument('--mongo_port', type=str, default='27017', help='Mongo DB 
port')
+parser.add_argument('--service_base_name', type=str, help='Service Base Name')
+parser.add_argument('--os_user', type=str, help='Dlab user')
+parser.add_argument('--keystore_password', type=str, help='Keystore password')
+parser.add_argument('--keycloak_client_id', type=str, help='Keycloak client 
id')
+parser.add_argument('--keycloak_client_secret', type=str, help='Keycloak 
client secret')
+parser.add_argument('--keycloak_auth_server_url', type=str, help='Keycloak 
auth server url')
 args = parser.parse_args()
 
 
@@ -70,38 +79,76 @@ def yml_billing(path):
         if args.cloud_provider == 'aws':
             if args.aws_job_enabled == 'true':
                 args.tag_resource_id =  'resourceTags' + ':' + 
args.tag_resource_id
-            config_orig = config_orig.replace('<BILLING_BUCKET_NAME>', 
args.billing_bucket)
-            config_orig = config_orig.replace('<AWS_JOB_ENABLED>', 
args.aws_job_enabled)
-            config_orig = config_orig.replace('<REPORT_PATH>', 
args.report_path)
-            config_orig = config_orig.replace('<ACCOUNT_ID>', args.account_id)
-            config_orig = config_orig.replace('<ACCESS_KEY_ID>', 
args.access_key_id)
-            config_orig = config_orig.replace('<SECRET_ACCESS_KEY>', 
args.secret_access_key)
-            config_orig = config_orig.replace('<CONF_BILLING_TAG>', 
args.billing_tag)
-            config_orig = config_orig.replace('<CONF_SERVICE_BASE_NAME>', 
args.infrastructure_tag)
-            config_orig = config_orig.replace('<MONGODB_PASSWORD>', 
args.mongo_password)
-            config_orig = config_orig.replace('<DLAB_ID>', args.dlab_id)
-            config_orig = config_orig.replace('<USAGE_DATE>', args.usage_date)
-            config_orig = config_orig.replace('<PRODUCT>', args.product)
-            config_orig = config_orig.replace('<USAGE_TYPE>', args.usage_type)
-            config_orig = config_orig.replace('<USAGE>', args.usage)
-            config_orig = config_orig.replace('<COST>', args.cost)
-            config_orig = config_orig.replace('<RESOURCE_ID>', 
args.resource_id)
-            config_orig = config_orig.replace('<TAGS>', args.tags)
+            config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+            config_orig = config_orig.replace('MONGO_PASSWORD', 
args.mongo_password)
+            config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+            config_orig = config_orig.replace('BILLING_BUCKET_NAME', 
args.billing_bucket)
+            config_orig = config_orig.replace('REPORT_PATH', args.report_path)
+            config_orig = config_orig.replace('AWS_JOB_ENABLED', 
args.aws_job_enabled)
+            config_orig = config_orig.replace('ACCOUNT_ID', args.account_id)
+            config_orig = config_orig.replace('ACCESS_KEY_ID', 
args.access_key_id)
+            config_orig = config_orig.replace('SECRET_ACCESS_KEY', 
args.secret_access_key)
+            config_orig = config_orig.replace('CONF_BILLING_TAG', 
args.billing_tag)
+            config_orig = config_orig.replace('SERVICE_BASE_NAME', 
args.service_base_name)
+            config_orig = config_orig.replace('DLAB_ID', args.dlab_id)
+            config_orig = config_orig.replace('USAGE_DATE', args.usage_date)
+            config_orig = config_orig.replace('PRODUCT', args.product)
+            config_orig = config_orig.replace('USAGE_TYPE', args.usage_type)
+            config_orig = config_orig.replace('USAGE', args.usage)
+            config_orig = config_orig.replace('COST', args.cost)
+            config_orig = config_orig.replace('RESOURCE_ID', args.resource_id)
+            config_orig = config_orig.replace('TAGS', args.tags)
         elif args.cloud_provider == 'azure':
-            config_orig = config_orig.replace('<CLIENT_ID>', args.client_id)
-            config_orig = config_orig.replace('<CLIENT_SECRET>', 
args.client_secret)
-            config_orig = config_orig.replace('<TENANT_ID>', args.tenant_id)
-            config_orig = config_orig.replace('<SUBSCRIPTION_ID>', 
args.subscription_id)
-            config_orig = config_orig.replace('<AUTHENTICATION_FILE>', 
args.authentication_file)
-            config_orig = config_orig.replace('<OFFER_NUMBER>', 
args.offer_number)
-            config_orig = config_orig.replace('<CURRENCY>', args.currency)
-            config_orig = config_orig.replace('<LOCALE>', args.locale)
-            config_orig = config_orig.replace('<REGION_INFO>', 
args.region_info)
-            config_orig = config_orig.replace('<MONGODB_PASSWORD>', 
args.mongo_password)
+            config_orig = config_orig.replace('SERVICE_BASE_NAME', 
args.service_base_name)
+            config_orig = config_orig.replace('OS_USER', args.os_user)
+            config_orig = config_orig.replace('MONGO_PASSWORD', 
args.mongo_password)
+            config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+            config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+            config_orig = config_orig.replace('KEY_STORE_PASSWORD', 
args.keystore_password)
+            config_orig = config_orig.replace('KEYCLOAK_CLIENT_ID', 
args.keycloak_client_id)
+            config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET', 
args.keycloak_client_secret)
+            config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL', 
args.keycloak_auth_server_url)
+            config_orig = config_orig.replace('CLIENT_ID', args.client_id)
+            config_orig = config_orig.replace('CLIENT_SECRET', 
args.client_secret)
+            config_orig = config_orig.replace('TENANT_ID', args.tenant_id)
+            config_orig = config_orig.replace('SUBSCRIPTION_ID', 
args.subscription_id)
+            config_orig = config_orig.replace('AUTHENTICATION_FILE', 
args.authentication_file)
+            config_orig = config_orig.replace('OFFER_NUMBER', 
args.offer_number)
+            config_orig = config_orig.replace('CURRENCY', args.currency)
+            config_orig = config_orig.replace('LOCALE', args.locale)
+            config_orig = config_orig.replace('REGION_INFO', args.region_info)
         elif args.cloud_provider == 'gcp':
-            config_orig = config_orig.replace('<CONF_SERVICE_BASE_NAME>', 
args.infrastructure_tag)
-            config_orig = config_orig.replace('<MONGO_PASSWORD>', 
args.mongo_password)
-            config_orig = config_orig.replace('<BILLING_DATASET_NAME>', 
args.billing_dataset_name)
+            config_orig = config_orig.replace('SERVICE_BASE_NAME', 
args.service_base_name)
+            config_orig = config_orig.replace('OS_USER', args.os_user)
+            config_orig = config_orig.replace('MONGO_PASSWORD', 
args.mongo_password)
+            config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+            config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+            config_orig = config_orig.replace('KEY_STORE_PASSWORD', 
args.keystore_password)
+            config_orig = config_orig.replace('DATASET_NAME', 
args.billing_dataset_name)
+            config_orig = config_orig.replace('KEYCLOAK_CLIENT_ID', 
args.keycloak_client_id)
+            config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET', 
args.keycloak_client_secret)
+            config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL', 
args.keycloak_auth_server_url)
+        f = open(path, 'w')
+        f.write(config_orig)
+        f.close()
+    except:
+        print("Could not write the target file {}".format(path))
+        sys.exit(1)
+
+def yml_billing_app(path):
+    try:
+        with open(path, 'r') as config_yml_r:
+            config_orig = config_yml_r.read()
+
+        config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+        config_orig = config_orig.replace('MONGO_PASSWORDD', 
args.mongo_password)
+        config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+        config_orig = config_orig.replace('OS_USER', args.os_user)
+        config_orig = config_orig.replace('KEY_STORE_PASSWORD', 
args.keystore_password)
+        config_orig = config_orig.replace('KEYCLOAK_CLIENT_ID', 
args.keycloak_client_id)
+        config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET', 
args.keycloak_client_secret)
+        config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL', 
args.keycloak_auth_server_url)
+
         f = open(path, 'w')
         f.write(config_orig)
         f.close()
@@ -134,6 +181,8 @@ if __name__ == "__main__":
     # Access to the bucket without credentials?
     try:
         yml_billing(args.dlab_dir + 'conf/billing.yml')
+        if args.cloud_provider == 'aws':
+            yml_billing_app(args.dlab_dir + 'conf/application.yml')
         yml_self_service(args.dlab_dir + 'conf/self-service.yml')
     except:
         print('Error configure billing')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py 
b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index fdca046..9bb436f 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -72,6 +72,9 @@ parser.add_argument('--usage', type=str, default=None)
 parser.add_argument('--cost', type=str, default=None)
 parser.add_argument('--resource_id', type=str, default=None)
 parser.add_argument('--tags', type=str, default=None)
+parser.add_argument('--keycloak_client_id', type=str, default=None)
+parser.add_argument('--keycloak_client_secret', type=str, default=None)
+parser.add_argument('--keycloak_auth_server_url', type=str, default=None)
 args = parser.parse_args()
 
 dlab_conf_dir = args.dlab_path + 'conf/'
@@ -175,6 +178,8 @@ def build_ui():
                 args.dlab_path))
         elif args.cloud_provider == 'aws':
             sudo('cp {0}/sources/services/billing-aws/billing.yml 
{0}/webapp/billing/conf/'.format(args.dlab_path))
+            sudo('cp 
{0}/sources/services/billing-aws/src/main/resources/application.yml '
+                 '{0}/webapp/billing/conf/'.format(args.dlab_path))
             sudo(
                 'cp {0}/sources/services/billing-aws/target/billing-aws*.jar 
{0}/webapp/billing/lib/'.format(
                     args.dlab_path))
@@ -232,4 +237,5 @@ if __name__ == "__main__":
              args.region_info, args.ldap_login, args.tenant_id, 
args.application_id,
              args.hostname, args.datalake_store_name, args.subscription_id, 
args.validate_permission_scope,
              args.dlab_id, args.usage_date, args.product, args.usage_type,
-             args.usage, args.cost, args.resource_id, args.tags, 
args.billing_dataset_name)
+             args.usage, args.cost, args.resource_id, args.tags, 
args.billing_dataset_name, args.keycloak_client_id,
+             args.keycloak_client_secret, args.keycloak_auth_server_url)
diff --git a/services/billing-gcp/billing.yml b/services/billing-gcp/billing.yml
index 522c7ad..86f9d70 100644
--- a/services/billing-gcp/billing.yml
+++ b/services/billing-gcp/billing.yml
@@ -1,20 +1,38 @@
-server:
-  port: 8088
 spring:
+  main:
+    allow-bean-definition-overriding: true
   data:
     mongodb:
       username: admin
-      password: <MONGO_PASSWORD>
+      password: MONGO_PASSWORD
       database: dlabdb
       port: 27017
-      host: localhost
+      host: MONGO_HOST
 dlab:
-  sbn: <CONF_SERVICE_BASE_NAME>
-  bigQueryDataset: <BILLING_DATASET_NAME>
-  cron: 0 0 */1 * * *
+  sbn: SERVICE_BASE_NAME
+  bigQueryDataset: DATASET_NAME
+  cron: 0 0 * * * *
+
+server:
+  port: 8088
+  servlet:
+    contextPath: /api/billing
+
+server.ssl.key-store-type: JKS
+server.ssl.key-store: /home/OS_USER/keys/endpoint.keystore.jks
+server.ssl.key-store-password: KEY_STORE_PASSWORD
+server.ssl.key-alias: endpoint
 
 logging:
   file: /var/opt/dlab/log/ssn/billing.log
   level:
     com:
-      epam: trace
\ No newline at end of file
+      epam: trace
+
+keycloak:
+  bearer-only: true
+  realm: dlab
+  resource: KEYCLOAK_CLIENT_ID
+  credentials.secret: KEYCLOAK_CLIENT_SECRET
+  ssl-required: none
+  auth-server-url: KEYCLOAK_AUTH_SERVER_URL
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to