This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1565-aws
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1565-aws by this push:
new 2c2b713 [DLAB-1565]: [GCP] - billing on local endpoint updated
2c2b713 is described below
commit 2c2b713c4478e6692345e08234da81316f60538c
Author: bodnarmykola <[email protected]>
AuthorDate: Tue Mar 31 15:48:23 2020 +0300
[DLAB-1565]: [GCP] - billing on local endpoint updated
---
.../src/general/lib/os/debian/ssn_lib.py | 24 +++-
.../src/general/scripts/gcp/ssn_configure.py | 5 +-
.../src/ssn/scripts/configure_billing.py | 125 ++++++++++++++-------
.../src/ssn/scripts/configure_ui.py | 8 +-
services/billing-gcp/billing.yml | 34 ++++--
5 files changed, 144 insertions(+), 52 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index bb073c3..488d8d7 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -180,7 +180,9 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
locale, region_info, ldap_login, tenant_id,
application_id, hostname, data_lake_name, subscription_id,
validate_permission_scope, dlab_id, usage_date, product,
- usage_type, usage, cost, resource_id, tags, billing_dataset_name,
report_path=''):
+ usage_type, usage, cost, resource_id, tags, billing_dataset_name,
report_path='', mongo_host,
+ keycloak_client_id, keycloak_client_secret,
+ keycloak_auth_server_url):
try:
if not exists(os.environ['ssn_dlab_path'] + 'tmp/ss_started'):
java_path = sudo("update-alternatives --query java | grep 'Value:
' | grep -o '/.*/jre'")
@@ -285,7 +287,15 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
'--cost {} ' \
'--resource_id {} ' \
'--tags {} ' \
- '--billing_dataset_name "{}" '.\
+ '--billing_dataset_name "{}" '\
+ '--mongo_host {} ' \
+ '--mongo_port {} ' \
+ '--service_base_name {} ' \
+ '--os_user {} ' \
+ '--keystore_password {} ' \
+ '--keycloak_client_id {} ' \
+ '--keycloak_client_secret {} ' \
+ '--keycloak_auth_server_url {} '.\
format(cloud_provider,
service_base_name,
tag_resource_id,
@@ -309,7 +319,15 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
cost,
resource_id,
tags,
- billing_dataset_name)
+ billing_dataset_name,
+ mongo_host,
+ mongo_port,
+ service_base_name,
+ os_user,
+ keystore_passwd,
+ keycloak_client_id,
+ keycloak_client_secret,
+ keycloak_auth_server_url)
sudo('python /tmp/configure_billing.py {}'.format(params))
try:
if os.environ['conf_stepcerts_enabled'] == 'true':
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index 1fcbd03..54b9781 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -457,12 +457,13 @@ if __name__ == "__main__":
params = "--hostname {} --keyfile {} --dlab_path {} --os_user {}
--os_family {} --billing_enabled {} " \
"--request_id {} --billing_dataset_name {} \
--resource {} --service_base_name {} --cloud_provider {}
--default_endpoint_name {} " \
- "--cloud_params '{}'". \
+ "--cloud_params '{}' --keycloak_client_id {}
--keycloak_client_secret {} --keycloak_auth_server_url {}". \
format(instance_hostname, ssn_conf['ssh_key_path'],
os.environ['ssn_dlab_path'], ssn_conf['dlab_ssh_user'],
os.environ['conf_os_family'], billing_enabled,
os.environ['request_id'],
os.environ['billing_dataset_name'],
os.environ['conf_resource'],
ssn_conf['service_base_name'],
os.environ['conf_cloud_provider'], ssn_conf['default_endpoint_name'],
- json.dumps(cloud_params))
+ json.dumps(cloud_params),
os.environ['keycloak_client_name'], os.environ['keycloak_client_secret'],
+ os.environ['keycloak_auth_server_url'])
try:
local("~/scripts/{}.py {}".format('configure_ui', params))
except:
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
index 9f40e72..c87af74 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
@@ -49,15 +49,24 @@ parser.add_argument('--locale', type=str, default='',
help='Azure locale')
parser.add_argument('--region_info', type=str, default='', help='Azure region
info')
parser.add_argument('--mongo_password', type=str, help='The password for Mongo
DB')
parser.add_argument('--dlab_dir', type=str, help='The path to dlab dir')
-parser.add_argument('--dlab_id', type=str, default='', help='Column name in
report file that contains dlab id tag')
-parser.add_argument('--usage_date', type=str, default='', help='Column name in
report file that contains usage date tag')
-parser.add_argument('--product', type=str, default='', help='Column name in
report file that contains product name tag')
-parser.add_argument('--usage_type', type=str, default='', help='Column name in
report file that contains usage type tag')
-parser.add_argument('--usage', type=str, default='', help='Column name in
report file that contains usage tag')
-parser.add_argument('--cost', type=str, default='', help='Column name in
report file that contains cost tag')
-parser.add_argument('--resource_id', type=str, default='', help='Column name
in report file that contains dlab resource id tag')
-parser.add_argument('--tags', type=str, default='', help='Column name in
report file that contains tags')
+parser.add_argument('--dlab_id', type=str,
default='resource_tags_user_user_tag', help='Column name in report file that
contains dlab id tag')
+parser.add_argument('--usage_date', type=str,
default='line_item_usage_start_date', help='Column name in report file that
contains usage date tag')
+parser.add_argument('--product', type=str, default='product_product_name',
help='Column name in report file that contains product name tag')
+parser.add_argument('--usage_type', type=str, default='line_item_usage_type',
help='Column name in report file that contains usage type tag')
+parser.add_argument('--usage', type=str, default='line_item_usage_amount',
help='Column name in report file that contains usage tag')
+parser.add_argument('--cost', type=str, default='line_item_blended_cost',
help='Column name in report file that contains cost tag')
+parser.add_argument('--resource_id', type=str,
default='line_item_resource_id', help='Column name in report file that contains
dlab resource id tag')
+parser.add_argument('--tags', type=str,
default='line_item_operation,line_item_line_item_description', help='Column
name in report file that contains tags')
parser.add_argument('--billing_dataset_name', type=str, default='', help='Name
of gcp billing dataset (in big query service')
+
+parser.add_argument('--mongo_host', type=str, default='localhost', help='Mongo
DB host')
+parser.add_argument('--mongo_port', type=str, default='27017', help='Mongo DB
port')
+parser.add_argument('--service_base_name', type=str, help='Service Base Name')
+parser.add_argument('--os_user', type=str, help='Dlab user')
+parser.add_argument('--keystore_password', type=str, help='Keystore password')
+parser.add_argument('--keycloak_client_id', type=str, help='Keycloak client
id')
+parser.add_argument('--keycloak_client_secret', type=str, help='Keycloak
client secret')
+parser.add_argument('--keycloak_auth_server_url', type=str, help='Keycloak
auth server url')
args = parser.parse_args()
@@ -70,38 +79,76 @@ def yml_billing(path):
if args.cloud_provider == 'aws':
if args.aws_job_enabled == 'true':
args.tag_resource_id = 'resourceTags' + ':' +
args.tag_resource_id
- config_orig = config_orig.replace('<BILLING_BUCKET_NAME>',
args.billing_bucket)
- config_orig = config_orig.replace('<AWS_JOB_ENABLED>',
args.aws_job_enabled)
- config_orig = config_orig.replace('<REPORT_PATH>',
args.report_path)
- config_orig = config_orig.replace('<ACCOUNT_ID>', args.account_id)
- config_orig = config_orig.replace('<ACCESS_KEY_ID>',
args.access_key_id)
- config_orig = config_orig.replace('<SECRET_ACCESS_KEY>',
args.secret_access_key)
- config_orig = config_orig.replace('<CONF_BILLING_TAG>',
args.billing_tag)
- config_orig = config_orig.replace('<CONF_SERVICE_BASE_NAME>',
args.infrastructure_tag)
- config_orig = config_orig.replace('<MONGODB_PASSWORD>',
args.mongo_password)
- config_orig = config_orig.replace('<DLAB_ID>', args.dlab_id)
- config_orig = config_orig.replace('<USAGE_DATE>', args.usage_date)
- config_orig = config_orig.replace('<PRODUCT>', args.product)
- config_orig = config_orig.replace('<USAGE_TYPE>', args.usage_type)
- config_orig = config_orig.replace('<USAGE>', args.usage)
- config_orig = config_orig.replace('<COST>', args.cost)
- config_orig = config_orig.replace('<RESOURCE_ID>',
args.resource_id)
- config_orig = config_orig.replace('<TAGS>', args.tags)
+ config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+ config_orig = config_orig.replace('MONGO_PASSWORD',
args.mongo_password)
+ config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+ config_orig = config_orig.replace('BILLING_BUCKET_NAME',
args.billing_bucket)
+ config_orig = config_orig.replace('REPORT_PATH', args.report_path)
+ config_orig = config_orig.replace('AWS_JOB_ENABLED',
args.aws_job_enabled)
+ config_orig = config_orig.replace('ACCOUNT_ID', args.account_id)
+ config_orig = config_orig.replace('ACCESS_KEY_ID',
args.access_key_id)
+ config_orig = config_orig.replace('SECRET_ACCESS_KEY',
args.secret_access_key)
+ config_orig = config_orig.replace('CONF_BILLING_TAG',
args.billing_tag)
+ config_orig = config_orig.replace('SERVICE_BASE_NAME',
args.service_base_name)
+ config_orig = config_orig.replace('DLAB_ID', args.dlab_id)
+ config_orig = config_orig.replace('USAGE_DATE', args.usage_date)
+ config_orig = config_orig.replace('PRODUCT', args.product)
+ config_orig = config_orig.replace('USAGE_TYPE', args.usage_type)
+ config_orig = config_orig.replace('USAGE', args.usage)
+ config_orig = config_orig.replace('COST', args.cost)
+ config_orig = config_orig.replace('RESOURCE_ID', args.resource_id)
+ config_orig = config_orig.replace('TAGS', args.tags)
elif args.cloud_provider == 'azure':
- config_orig = config_orig.replace('<CLIENT_ID>', args.client_id)
- config_orig = config_orig.replace('<CLIENT_SECRET>',
args.client_secret)
- config_orig = config_orig.replace('<TENANT_ID>', args.tenant_id)
- config_orig = config_orig.replace('<SUBSCRIPTION_ID>',
args.subscription_id)
- config_orig = config_orig.replace('<AUTHENTICATION_FILE>',
args.authentication_file)
- config_orig = config_orig.replace('<OFFER_NUMBER>',
args.offer_number)
- config_orig = config_orig.replace('<CURRENCY>', args.currency)
- config_orig = config_orig.replace('<LOCALE>', args.locale)
- config_orig = config_orig.replace('<REGION_INFO>',
args.region_info)
- config_orig = config_orig.replace('<MONGODB_PASSWORD>',
args.mongo_password)
+ config_orig = config_orig.replace('SERVICE_BASE_NAME',
args.service_base_name)
+ config_orig = config_orig.replace('OS_USER', args.os_user)
+ config_orig = config_orig.replace('MONGO_PASSWORD',
args.mongo_password)
+ config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+ config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+ config_orig = config_orig.replace('KEY_STORE_PASSWORD',
args.keystore_password)
+ config_orig = config_orig.replace('KEYCLOAK_CLIENT_ID',
args.keycloak_client_id)
+ config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET',
args.keycloak_client_secret)
+ config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL',
args.keycloak_auth_server_url)
+ config_orig = config_orig.replace('CLIENT_ID', args.client_id)
+ config_orig = config_orig.replace('CLIENT_SECRET',
args.client_secret)
+ config_orig = config_orig.replace('TENANT_ID', args.tenant_id)
+ config_orig = config_orig.replace('SUBSCRIPTION_ID',
args.subscription_id)
+ config_orig = config_orig.replace('AUTHENTICATION_FILE',
args.authentication_file)
+ config_orig = config_orig.replace('OFFER_NUMBER',
args.offer_number)
+ config_orig = config_orig.replace('CURRENCY', args.currency)
+ config_orig = config_orig.replace('LOCALE', args.locale)
+ config_orig = config_orig.replace('REGION_INFO', args.region_info)
elif args.cloud_provider == 'gcp':
- config_orig = config_orig.replace('<CONF_SERVICE_BASE_NAME>',
args.infrastructure_tag)
- config_orig = config_orig.replace('<MONGO_PASSWORD>',
args.mongo_password)
- config_orig = config_orig.replace('<BILLING_DATASET_NAME>',
args.billing_dataset_name)
+ config_orig = config_orig.replace('SERVICE_BASE_NAME',
args.service_base_name)
+ config_orig = config_orig.replace('OS_USER', args.os_user)
+ config_orig = config_orig.replace('MONGO_PASSWORD',
args.mongo_password)
+ config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+ config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+ config_orig = config_orig.replace('KEY_STORE_PASSWORD',
args.keystore_password)
+ config_orig = config_orig.replace('DATASET_NAME',
args.billing_dataset_name)
+ config_orig = config_orig.replace('KEYCLOAK_CLIENT_ID',
args.keycloak_client_id)
+ config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET',
args.keycloak_client_secret)
+ config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL',
args.keycloak_auth_server_url)
+ f = open(path, 'w')
+ f.write(config_orig)
+ f.close()
+ except:
+ print("Could not write the target file {}".format(path))
+ sys.exit(1)
+
+def yml_billing_app(path):
+ try:
+ with open(path, 'r') as config_yml_r:
+ config_orig = config_yml_r.read()
+
+ config_orig = config_orig.replace('MONGO_HOST', args.mongo_host)
+ config_orig = config_orig.replace('MONGO_PASSWORDD',
args.mongo_password)
+ config_orig = config_orig.replace('MONGO_PORT', args.mongo_port)
+ config_orig = config_orig.replace('OS_USER', args.os_user)
+ config_orig = config_orig.replace('KEY_STORE_PASSWORD',
args.keystore_password)
+ config_orig = config_orig.replace('KEYCLOAK_CLIENT_ID',
args.keycloak_client_id)
+ config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET',
args.keycloak_client_secret)
+ config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL',
args.keycloak_auth_server_url)
+
f = open(path, 'w')
f.write(config_orig)
f.close()
@@ -134,6 +181,8 @@ if __name__ == "__main__":
# Access to the bucket without credentials?
try:
yml_billing(args.dlab_dir + 'conf/billing.yml')
+ if args.cloud_provider == 'aws':
+ yml_billing_app(args.dlab_dir + 'conf/application.yml')
yml_self_service(args.dlab_dir + 'conf/self-service.yml')
except:
print('Error configure billing')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index fdca046..9bb436f 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -72,6 +72,9 @@ parser.add_argument('--usage', type=str, default=None)
parser.add_argument('--cost', type=str, default=None)
parser.add_argument('--resource_id', type=str, default=None)
parser.add_argument('--tags', type=str, default=None)
+parser.add_argument('--keycloak_client_id', type=str, default=None)
+parser.add_argument('--keycloak_client_secret', type=str, default=None)
+parser.add_argument('--keycloak_auth_server_url', type=str, default=None)
args = parser.parse_args()
dlab_conf_dir = args.dlab_path + 'conf/'
@@ -175,6 +178,8 @@ def build_ui():
args.dlab_path))
elif args.cloud_provider == 'aws':
sudo('cp {0}/sources/services/billing-aws/billing.yml
{0}/webapp/billing/conf/'.format(args.dlab_path))
+ sudo('cp
{0}/sources/services/billing-aws/src/main/resources/application.yml '
+ '{0}/webapp/billing/conf/'.format(args.dlab_path))
sudo(
'cp {0}/sources/services/billing-aws/target/billing-aws*.jar
{0}/webapp/billing/lib/'.format(
args.dlab_path))
@@ -232,4 +237,5 @@ if __name__ == "__main__":
args.region_info, args.ldap_login, args.tenant_id,
args.application_id,
args.hostname, args.datalake_store_name, args.subscription_id,
args.validate_permission_scope,
args.dlab_id, args.usage_date, args.product, args.usage_type,
- args.usage, args.cost, args.resource_id, args.tags,
args.billing_dataset_name)
+ args.usage, args.cost, args.resource_id, args.tags,
args.billing_dataset_name, args.keycloak_client_id,
+ args.keycloak_client_secret, args.keycloak_auth_server_url)
diff --git a/services/billing-gcp/billing.yml b/services/billing-gcp/billing.yml
index 522c7ad..86f9d70 100644
--- a/services/billing-gcp/billing.yml
+++ b/services/billing-gcp/billing.yml
@@ -1,20 +1,38 @@
-server:
- port: 8088
spring:
+ main:
+ allow-bean-definition-overriding: true
data:
mongodb:
username: admin
- password: <MONGO_PASSWORD>
+ password: MONGO_PASSWORD
database: dlabdb
port: 27017
- host: localhost
+ host: MONGO_HOST
dlab:
- sbn: <CONF_SERVICE_BASE_NAME>
- bigQueryDataset: <BILLING_DATASET_NAME>
- cron: 0 0 */1 * * *
+ sbn: SERVICE_BASE_NAME
+ bigQueryDataset: DATASET_NAME
+ cron: 0 0 * * * *
+
+server:
+ port: 8088
+ servlet:
+ contextPath: /api/billing
+
+server.ssl.key-store-type: JKS
+server.ssl.key-store: /home/OS_USER/keys/endpoint.keystore.jks
+server.ssl.key-store-password: KEY_STORE_PASSWORD
+server.ssl.key-alias: endpoint
logging:
file: /var/opt/dlab/log/ssn/billing.log
level:
com:
- epam: trace
\ No newline at end of file
+ epam: trace
+
+keycloak:
+ bearer-only: true
+ realm: dlab
+ resource: KEYCLOAK_CLIENT_ID
+ credentials.secret: KEYCLOAK_CLIENT_SECRET
+ ssl-required: none
+ auth-server-url: KEYCLOAK_AUTH_SERVER_URL
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]