This is an automated email from the ASF dual-hosted git repository.

mykolabodnar pushed a commit to branch DLAB-1565
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git


The following commit(s) were added to refs/heads/DLAB-1565 by this push:
     new 2490ac7  [DLAB-1565]: Convey billing for remote endpoints - [GCP] 
billing.yml and variables fixed
2490ac7 is described below

commit 2490ac7e5b8a147f8a92b0d13ad930c25fe14922
Author: Mykola_Bodnar1 <[email protected]>
AuthorDate: Wed Mar 11 17:24:02 2020 +0200

    [DLAB-1565]: Convey billing for remote endpoints - [GCP] billing.yml and 
variables fixed
---
 .../terraform/bin/deploy/endpoint_fab.py           |  35 +++
 .../bin/deploy/mongo_files/configure_mongo.py      | 103 ++++++++
 .../bin/deploy/mongo_files/gcp/mongo_roles.json    | 268 +++++++++++++++++++++
 .../bin/deploy/mongo_files/mongod.service_template |  34 +++
 4 files changed, 440 insertions(+)

diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py 
b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
index 5718906..bac28cf 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
@@ -1,5 +1,6 @@
 from fabric import Connection
 from patchwork.files import exists
+from pymongo import MongoClient
 import logging
 import argparse
 import sys
@@ -248,6 +249,34 @@ def ensure_docker_endpoint():
         traceback.print_exc()
         sys.exit(1)
 
+def ensure_mongo_endpoint():
+    try:
+        print('[INSTALLING MONGO DATABASE]')
+        if not exists(conn, 
'/home/{}/.ensure_dir/mongo_ensured'.format(args.os_user)):
+            conn.sudo('apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 
--recv EA312927')
+            conn.sudo('ver=`lsb_release -cs`; echo "deb 
http://repo.mongodb.org/apt/ubuntu $ver/mongodb-org/3.2 multiverse" | sudo tee 
/etc/apt/sources.list.d/mongodb-org-3.2.list; apt-get update')
+            conn.sudo('apt-get install mongodb-org -y --allow-unauthenticated 
install')
+            conn.sudo('systemctl enable mongod.service')
+            conn.sudo('touch /home/{}/.ensure_dir/mongo_ensured'
+                      .format(args.os_user))
+        print('[CONFIGURING MONGO DATABASE]')
+        if not exists(conn, '/lib/systemd/system/mongod.service'):
+            conn.put('./mongo_files/mongod.service_template', 
'/tmp/mongod.service_template')
+            conn.sudo('sed -i "s/MONGO_USR/mongodb/g" 
/tmp/mongod.service_template'.format(args.os_user))
+            conn.sudo('cp -i /tmp/mongod.service_template 
/lib/systemd/system/mongod.service'))
+            conn.sudo('systemctl daemon-reload')
+            conn.sudo('systemctl enable mongod.service')
+        if not exist(conn, '/tmp/configure_mongo.py'):
+            conn.put('./mongo_files/configure_mongo.py', 
'/tmp/configure_mongo.py')
+            conn.sudo('sed -i "s|PASSWORD|{}|g" 
/tmp/configure_mongo.py'.format(args.mongo_password))
+        if not exist(conn, '/tmp/mongo_roles.json'):
+            conn.put('./mongo_files/gcp/mongo_roles.json', 
'/tmp/mongo_roles.json')
+        conn.sudo('python /tmp/configure_mongo.py')
+    except Exception as err:
+        logging.error('Failed to install Mongo: ', str(err))
+        traceback.print_exc()
+        sys.exit(1)
+
 
 def create_key_dir_endpoint():
     try:
@@ -309,6 +338,9 @@ def 
configure_supervisor_endpoint(endpoint_keystore_password):
                                            
'subnet-id'.format(interface)).stdout
                 args.vpc2_id = args.vpc_id
                 args.subnet2_id = args.subnet_id
+            if args.cloud_provider == 'gcp':
+                conn.sudo('sed -i 
"s|CONF_PARAMETER_NAME|--spring.config.location=|g" {}/tmp/supervisor_svc.conf'
+                          .format(args.dlab_path))
             conn.sudo('sed -i "s|OS_USR|{}|g" {}/tmp/supervisor_svc.conf'
                       .format(args.os_user, args.dlab_path))
             conn.sudo('sed -i "s|WEB_CONF|{}|g" {}/tmp/supervisor_svc.conf'
@@ -895,6 +927,9 @@ def start_deploy():
     logging.info("Installing Docker")
     ensure_docker_endpoint()
 
+    logging.info("Installing Mongo Database")
+    ensure_mongo_endpoint()
+
     logging.info("Configuring Supervisor")
     configure_supervisor_endpoint(endpoint_keystore_password)
 
diff --git 
a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
 
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
new file mode 100644
index 0000000..e4b97a2
--- /dev/null
+++ 
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
@@ -0,0 +1,103 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# 
******************************************************************************
+
+from pymongo import MongoClient
+import yaml, json, sys
+import subprocess
+import time
+import argparse
+from dlab.fab import *
+
+path = "/etc/mongod.conf"
+outfile = "/etc/mongo_params.yml"
+
+parser = argparse.ArgumentParser()
+
+#parser.add_argument('--mongo_parameters', type=str, default='')
+args = parser.parse_args()
+
+
+def add_2_yml_config(path, section, param, value):
+    try:
+        try:
+            with open(path, 'r') as config_yml_r:
+                config_orig = yaml.load(config_yml_r)
+        except:
+            config_orig = {}
+        sections = []
+        for i in config_orig:
+            sections.append(i)
+        if section in sections:
+            config_orig[section].update({param:value})
+        else:
+            config_orig.update({section:{param:value}})
+        with open(path, 'w') as outfile_yml_w:
+            yaml.dump(config_orig, outfile_yml_w, default_flow_style=False)
+        return True
+    except:
+        print("Could not write the target file")
+        return False
+
+
+def read_yml_conf(path, section, param):
+    try:
+        with open(path, 'r') as config_yml:
+            config = yaml.load(config_yml)
+        result = config[section][param]
+        return result
+    except:
+        print("File does not exist")
+        return ''
+
+
+if __name__ == "__main__":
+    mongo_passwd = "PASSWORD"
+    mongo_ip = read_yml_conf(path,'net','bindIp')
+    mongo_port = read_yml_conf(path,'net','port')
+    #mongo_parameters = json.loads(args.mongo_parameters)
+    # Setting up admin's password and enabling security
+    client = MongoClient(mongo_ip + ':' + str(mongo_port))
+    pass_upd = True
+    try:
+        command = ['service', 'mongod', 'start']
+        subprocess.call(command, shell=False)
+        time.sleep(5)
+        client.dlabdb.add_user('admin', mongo_passwd, 
roles=[{'role':'userAdminAnyDatabase','db':'admin'}])
+        client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"])
+        # set_mongo_parameters(client, mongo_parameters)
+
+        # client.dlabdb.security.create_index("expireAt", 
expireAfterSeconds=7200)
+        if add_2_yml_config(path,'security','authorization','enabled'):
+            command = ['service', 'mongod', 'restart']
+            subprocess.call(command, shell=False)
+    except:
+        print("Looks like MongoDB have already been secured")
+        pass_upd = False
+
+    # Generating output config
+    add_2_yml_config(outfile, 'network', 'ip', mongo_ip)
+    add_2_yml_config(outfile, 'network', 'port', mongo_port)
+    add_2_yml_config(outfile, 'account', 'user', 'admin')
+    if pass_upd:
+        add_2_yml_config(outfile, 'account', 'pass', mongo_passwd)
+
diff --git 
a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/gcp/mongo_roles.json
 
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/gcp/mongo_roles.json
new file mode 100644
index 0000000..43d12e3
--- /dev/null
+++ 
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/gcp/mongo_roles.json
@@ -0,0 +1,268 @@
+[
+  {
+    "_id": "nbShapes_n1-highcpu-2_fetching",
+    "description": "Use n1-highcpu-2 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-highcpu-2"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbShapes_n1-highcpu-8_fetching",
+    "description": "Use n1-highcpu-8 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-highcpu-8"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbShapes_n1-highcpu-32_fetching",
+    "description": "Use n1-highcpu-32 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-highcpu-32"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbShapes_n1-highmem-4_fetching",
+    "description": "Use n1-highmem-4 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-highmem-4"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbShapes_n1-highmem-16_fetching",
+    "description": "Use n1-highmem-16 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-highmem-16"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbShapes_n1-highmem-32_fetching",
+    "description": "Use n1-highmem-32 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-highmem-32"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbShapes_n1-standard-2_fetching",
+    "description": "Use n1-standard-2 instance shape for notebook",
+    "exploratory_shapes": [
+      "n1-standard-2"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateDeeplearning",
+    "description": "Create Notebook Deep Learning",
+    "exploratories": [
+      "docker.dlab-deeplearning"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateJupyter",
+    "description": "Create Notebook Jupyter",
+    "exploratories": [
+      "docker.dlab-jupyter"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateJupyterLab",
+    "description": "Create Notebook JupyterLab",
+    "exploratories": [
+      "docker.dlab-jupyterlab"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateSuperset",
+    "description": "Create Notebook Superset",
+    "exploratories": [
+      "docker.dlab-superset"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateRstudio",
+    "description": "Create Notebook RStudio",
+    "exploratories": [
+      "docker.dlab-rstudio"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateTensor",
+    "description": "Create Notebook Jupyter with TensorFlow",
+    "exploratories": [
+      "docker.dlab-tensor"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateTensorRstudio",
+    "description": "Create Notebook RStudio with TensorFlow",
+    "exploratories": [
+      "docker.dlab-tensor-rstudio"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateZeppelin",
+    "description": "Create Notebook Apache Zeppelin",
+    "exploratories": [
+      "docker.dlab-zeppelin"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateDataEngine",
+    "description": "Create Data Engine",
+    "computationals": [
+      "docker.dlab-dataengine"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbCreateDataEngineService",
+    "description": "Create Data Engine Service",
+    "computationals": [
+      "docker.dlab-dataengine-service"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-standard-2_fetching",
+    "description": "Use n1-standard-2 instance shape for cluster",
+    "computational_shapes": [
+      "n1-standard-2"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-highmem-4_fetching",
+    "description": "Use n1-highmem-4 instance shape for cluster",
+    "computational_shapes": [
+      "n1-highmem-4"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-highmem-16_fetching",
+    "description": "Use n1-highmem-16 instance shape for cluster",
+    "computational_shapes": [
+      "n1-highmem-16"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-highmem-32_fetching",
+    "description": "Use n1-highmem-32 instance shape for cluster",
+    "computational_shapes": [
+      "n1-highmem-32"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-highcpu-8_fetching",
+    "description": "Use n1-highcpu-8 instance shape for cluster",
+    "computational_shapes": [
+      "n1-highcpu-8"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-highcpu-2_fetching",
+    "description": "Use n1-highcpu-2 instance shape for cluster",
+    "computational_shapes": [
+      "n1-highcpu-2"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "compShapes_n1-highcpu-32_fetching",
+    "description": "Use n1-highcpu-32 instance shape for cluster",
+    "computational_shapes": [
+      "n1-highcpu-32"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "nbBillingReportFull",
+    "description": "View full billing report for all users",
+    "pages": [
+      "/api/infrastructure_provision/billing"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  },
+  {
+    "_id": "admin",
+    "description": "Allow to execute administration operation",
+    "pages": [
+      "environment/*",
+      "/api/infrastructure/backup",
+      "/roleManagement",
+      "/api/settings",
+      "/user/settings",
+      "/api/project",
+      "/api/endpoint"
+    ],
+    "groups": [
+      "$anyuser"
+    ]
+  }
+]
diff --git 
a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/mongod.service_template
 
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/mongod.service_template
new file mode 100644
index 0000000..cddbf66
--- /dev/null
+++ 
b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/mongod.service_template
@@ -0,0 +1,34 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# 
******************************************************************************
+
+[Unit]
+Description=High-performance, schema-free document-oriented database
+After=network.target
+Documentation=https://docs.mongodb.org/manual
+
+[Service]
+Type=forking
+User=MONGO_USR
+Group=MONGO_USR
+ExecStart=/usr/bin/mongod --quiet --config /etc/mongod.conf
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to