This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1565
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1565 by this push:
new 2490ac7 [DLAB-1565]: Convey billing for remote endpoints - [GCP]
billing.yml and variables fixed
2490ac7 is described below
commit 2490ac7e5b8a147f8a92b0d13ad930c25fe14922
Author: Mykola_Bodnar1 <[email protected]>
AuthorDate: Wed Mar 11 17:24:02 2020 +0200
[DLAB-1565]: Convey billing for remote endpoints - [GCP] billing.yml and
variables fixed
---
.../terraform/bin/deploy/endpoint_fab.py | 35 +++
.../bin/deploy/mongo_files/configure_mongo.py | 103 ++++++++
.../bin/deploy/mongo_files/gcp/mongo_roles.json | 268 +++++++++++++++++++++
.../bin/deploy/mongo_files/mongod.service_template | 34 +++
4 files changed, 440 insertions(+)
diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
index 5718906..bac28cf 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
@@ -1,5 +1,6 @@
from fabric import Connection
from patchwork.files import exists
+from pymongo import MongoClient
import logging
import argparse
import sys
@@ -248,6 +249,34 @@ def ensure_docker_endpoint():
traceback.print_exc()
sys.exit(1)
+def ensure_mongo_endpoint():
+ try:
+ print('[INSTALLING MONGO DATABASE]')
+ if not exists(conn,
'/home/{}/.ensure_dir/mongo_ensured'.format(args.os_user)):
+ conn.sudo('apt-key adv --keyserver hkp://keyserver.ubuntu.com:80
--recv EA312927')
+ conn.sudo('ver=`lsb_release -cs`; echo "deb
http://repo.mongodb.org/apt/ubuntu $ver/mongodb-org/3.2 multiverse" | sudo tee
/etc/apt/sources.list.d/mongodb-org-3.2.list; apt-get update')
+ conn.sudo('apt-get install mongodb-org -y --allow-unauthenticated
install')
+ conn.sudo('systemctl enable mongod.service')
+ conn.sudo('touch /home/{}/.ensure_dir/mongo_ensured'
+ .format(args.os_user))
+ print('[CONFIGURING MONGO DATABASE]')
+ if not exists(conn, '/lib/systemd/system/mongod.service'):
+ conn.put('./mongo_files/mongod.service_template',
'/tmp/mongod.service_template')
+ conn.sudo('sed -i "s/MONGO_USR/mongodb/g"
/tmp/mongod.service_template'.format(args.os_user))
+ conn.sudo('cp -i /tmp/mongod.service_template
/lib/systemd/system/mongod.service'))
+ conn.sudo('systemctl daemon-reload')
+ conn.sudo('systemctl enable mongod.service')
+ if not exist(conn, '/tmp/configure_mongo.py'):
+ conn.put('./mongo_files/configure_mongo.py',
'/tmp/configure_mongo.py')
+ conn.sudo('sed -i "s|PASSWORD|{}|g"
/tmp/configure_mongo.py'.format(args.mongo_password))
+ if not exist(conn, '/tmp/mongo_roles.json'):
+ conn.put('./mongo_files/gcp/mongo_roles.json',
'/tmp/mongo_roles.json')
+ conn.sudo('python /tmp/configure_mongo.py')
+ except Exception as err:
+ logging.error('Failed to install Mongo: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
def create_key_dir_endpoint():
try:
@@ -309,6 +338,9 @@ def
configure_supervisor_endpoint(endpoint_keystore_password):
'subnet-id'.format(interface)).stdout
args.vpc2_id = args.vpc_id
args.subnet2_id = args.subnet_id
+ if args.cloud_provider == 'gcp':
+ conn.sudo('sed -i
"s|CONF_PARAMETER_NAME|--spring.config.location=|g" {}/tmp/supervisor_svc.conf'
+ .format(args.dlab_path))
conn.sudo('sed -i "s|OS_USR|{}|g" {}/tmp/supervisor_svc.conf'
.format(args.os_user, args.dlab_path))
conn.sudo('sed -i "s|WEB_CONF|{}|g" {}/tmp/supervisor_svc.conf'
@@ -895,6 +927,9 @@ def start_deploy():
logging.info("Installing Docker")
ensure_docker_endpoint()
+ logging.info("Installing Mongo Database")
+ ensure_mongo_endpoint()
+
logging.info("Configuring Supervisor")
configure_supervisor_endpoint(endpoint_keystore_password)
diff --git
a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
new file mode 100644
index 0000000..e4b97a2
--- /dev/null
+++
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
@@ -0,0 +1,103 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
******************************************************************************
+
+from pymongo import MongoClient
+import yaml, json, sys
+import subprocess
+import time
+import argparse
+from dlab.fab import *
+
+path = "/etc/mongod.conf"
+outfile = "/etc/mongo_params.yml"
+
+parser = argparse.ArgumentParser()
+
+#parser.add_argument('--mongo_parameters', type=str, default='')
+args = parser.parse_args()
+
+
+def add_2_yml_config(path, section, param, value):
+ try:
+ try:
+ with open(path, 'r') as config_yml_r:
+ config_orig = yaml.load(config_yml_r)
+ except:
+ config_orig = {}
+ sections = []
+ for i in config_orig:
+ sections.append(i)
+ if section in sections:
+ config_orig[section].update({param:value})
+ else:
+ config_orig.update({section:{param:value}})
+ with open(path, 'w') as outfile_yml_w:
+ yaml.dump(config_orig, outfile_yml_w, default_flow_style=False)
+ return True
+ except:
+ print("Could not write the target file")
+ return False
+
+
+def read_yml_conf(path, section, param):
+ try:
+ with open(path, 'r') as config_yml:
+ config = yaml.load(config_yml)
+ result = config[section][param]
+ return result
+ except:
+ print("File does not exist")
+ return ''
+
+
+if __name__ == "__main__":
+ mongo_passwd = "PASSWORD"
+ mongo_ip = read_yml_conf(path,'net','bindIp')
+ mongo_port = read_yml_conf(path,'net','port')
+ #mongo_parameters = json.loads(args.mongo_parameters)
+ # Setting up admin's password and enabling security
+ client = MongoClient(mongo_ip + ':' + str(mongo_port))
+ pass_upd = True
+ try:
+ command = ['service', 'mongod', 'start']
+ subprocess.call(command, shell=False)
+ time.sleep(5)
+ client.dlabdb.add_user('admin', mongo_passwd,
roles=[{'role':'userAdminAnyDatabase','db':'admin'}])
+ client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"])
+ # set_mongo_parameters(client, mongo_parameters)
+
+ # client.dlabdb.security.create_index("expireAt",
expireAfterSeconds=7200)
+ if add_2_yml_config(path,'security','authorization','enabled'):
+ command = ['service', 'mongod', 'restart']
+ subprocess.call(command, shell=False)
+ except:
+ print("Looks like MongoDB have already been secured")
+ pass_upd = False
+
+ # Generating output config
+ add_2_yml_config(outfile, 'network', 'ip', mongo_ip)
+ add_2_yml_config(outfile, 'network', 'port', mongo_port)
+ add_2_yml_config(outfile, 'account', 'user', 'admin')
+ if pass_upd:
+ add_2_yml_config(outfile, 'account', 'pass', mongo_passwd)
+
diff --git
a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/gcp/mongo_roles.json
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/gcp/mongo_roles.json
new file mode 100644
index 0000000..43d12e3
--- /dev/null
+++
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/gcp/mongo_roles.json
@@ -0,0 +1,268 @@
+[
+ {
+ "_id": "nbShapes_n1-highcpu-2_fetching",
+ "description": "Use n1-highcpu-2 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-highcpu-2"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbShapes_n1-highcpu-8_fetching",
+ "description": "Use n1-highcpu-8 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-highcpu-8"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbShapes_n1-highcpu-32_fetching",
+ "description": "Use n1-highcpu-32 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-highcpu-32"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbShapes_n1-highmem-4_fetching",
+ "description": "Use n1-highmem-4 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-highmem-4"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbShapes_n1-highmem-16_fetching",
+ "description": "Use n1-highmem-16 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-highmem-16"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbShapes_n1-highmem-32_fetching",
+ "description": "Use n1-highmem-32 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-highmem-32"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbShapes_n1-standard-2_fetching",
+ "description": "Use n1-standard-2 instance shape for notebook",
+ "exploratory_shapes": [
+ "n1-standard-2"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateDeeplearning",
+ "description": "Create Notebook Deep Learning",
+ "exploratories": [
+ "docker.dlab-deeplearning"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateJupyter",
+ "description": "Create Notebook Jupyter",
+ "exploratories": [
+ "docker.dlab-jupyter"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateJupyterLab",
+ "description": "Create Notebook JupyterLab",
+ "exploratories": [
+ "docker.dlab-jupyterlab"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateSuperset",
+ "description": "Create Notebook Superset",
+ "exploratories": [
+ "docker.dlab-superset"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateRstudio",
+ "description": "Create Notebook RStudio",
+ "exploratories": [
+ "docker.dlab-rstudio"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateTensor",
+ "description": "Create Notebook Jupyter with TensorFlow",
+ "exploratories": [
+ "docker.dlab-tensor"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateTensorRstudio",
+ "description": "Create Notebook RStudio with TensorFlow",
+ "exploratories": [
+ "docker.dlab-tensor-rstudio"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateZeppelin",
+ "description": "Create Notebook Apache Zeppelin",
+ "exploratories": [
+ "docker.dlab-zeppelin"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateDataEngine",
+ "description": "Create Data Engine",
+ "computationals": [
+ "docker.dlab-dataengine"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbCreateDataEngineService",
+ "description": "Create Data Engine Service",
+ "computationals": [
+ "docker.dlab-dataengine-service"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-standard-2_fetching",
+ "description": "Use n1-standard-2 instance shape for cluster",
+ "computational_shapes": [
+ "n1-standard-2"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-highmem-4_fetching",
+ "description": "Use n1-highmem-4 instance shape for cluster",
+ "computational_shapes": [
+ "n1-highmem-4"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-highmem-16_fetching",
+ "description": "Use n1-highmem-16 instance shape for cluster",
+ "computational_shapes": [
+ "n1-highmem-16"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-highmem-32_fetching",
+ "description": "Use n1-highmem-32 instance shape for cluster",
+ "computational_shapes": [
+ "n1-highmem-32"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-highcpu-8_fetching",
+ "description": "Use n1-highcpu-8 instance shape for cluster",
+ "computational_shapes": [
+ "n1-highcpu-8"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-highcpu-2_fetching",
+ "description": "Use n1-highcpu-2 instance shape for cluster",
+ "computational_shapes": [
+ "n1-highcpu-2"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "compShapes_n1-highcpu-32_fetching",
+ "description": "Use n1-highcpu-32 instance shape for cluster",
+ "computational_shapes": [
+ "n1-highcpu-32"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "nbBillingReportFull",
+ "description": "View full billing report for all users",
+ "pages": [
+ "/api/infrastructure_provision/billing"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
+ "_id": "admin",
+ "description": "Allow to execute administration operation",
+ "pages": [
+ "environment/*",
+ "/api/infrastructure/backup",
+ "/roleManagement",
+ "/api/settings",
+ "/user/settings",
+ "/api/project",
+ "/api/endpoint"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ }
+]
diff --git
a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/mongod.service_template
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/mongod.service_template
new file mode 100644
index 0000000..cddbf66
--- /dev/null
+++
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/mongod.service_template
@@ -0,0 +1,34 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
******************************************************************************
+
+[Unit]
+Description=High-performance, schema-free document-oriented database
+After=network.target
+Documentation=https://docs.mongodb.org/manual
+
+[Service]
+Type=forking
+User=MONGO_USR
+Group=MONGO_USR
+ExecStart=/usr/bin/mongod --quiet --config /etc/mongod.conf
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]