This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-253
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-253 by this push:
new 31dd48f added reverse proxy to dataengine-service
31dd48f is described below
commit 31dd48fc54458e3e1e3a20b75c88a5d57e8cb0a1
Author: Oleh Martushevskyi <[email protected]>
AuthorDate: Wed Mar 13 17:59:56 2019 +0200
added reverse proxy to dataengine-service
---
.../{emr.conf => dataengine-service.conf} | 0
.../scripts/aws/dataengine-service_configure.py | 2 +-
.../scripts/gcp/dataengine-service_configure.py | 61 +++++++++++++++++++++-
.../scripts/os/common_configure_reverse_proxy.py | 6 +--
4 files changed, 63 insertions(+), 6 deletions(-)
diff --git a/infrastructure-provisioning/src/edge/templates/locations/emr.conf
b/infrastructure-provisioning/src/edge/templates/locations/dataengine-service.conf
similarity index 100%
rename from infrastructure-provisioning/src/edge/templates/locations/emr.conf
rename to
infrastructure-provisioning/src/edge/templates/locations/dataengine-service.conf
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
index f5594b2..cce4074 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
@@ -123,7 +123,7 @@ def configure_dataengine_service(instance, emr_conf):
.format(emr_conf['edge_instance_hostname'],
emr_conf['key_path'],
emr_conf['os_user'],
- 'emr',
+ 'dataengine-service',
emr_conf['exploratory_name'],
json.dumps(additional_info))
try:
diff --git
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
index 02964cf..ed8f0ee 100644
---
a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
+++
b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
@@ -36,7 +36,6 @@ import multiprocessing
def configure_dataengine_service(instance, dataproc_conf):
dataproc_conf['instance_ip'] =
meta_lib.GCPMeta().get_private_ip_address(instance)
- # edge_instance_hostname =
GCPMeta().get_private_ip_address(dataproc_conf['edge_instance_name'])
# configuring proxy on Data Engine service
try:
logging.info('[CONFIGURE PROXY ON DATAENGINE SERVICE]')
@@ -75,6 +74,48 @@ def configure_dataengine_service(instance, dataproc_conf):
actions_lib.GCPActions().delete_dataproc_cluster(dataproc_conf['cluster_name'],
os.environ['gcp_region'])
sys.exit(1)
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ slaves = []
+ for idx, instance in
enumerate(dataproc_conf['cluster_core_instances']):
+ slave_ip = meta_lib.GCPMeta().get_private_ip_address(instance)
+ slave = {
+ 'name': 'datanode{}'.format(idx + 1),
+ 'ip': slave_ip,
+ 'dns': instance
+ }
+ slaves.append(slave)
+ additional_info = {
+ "computational_name": dataproc_conf['computational_name'],
+ "master_ip": dataproc_conf['master_ip'],
+ "master_dns": dataproc_conf['master_name'],
+ "slaves": slaves,
+ "tensor": False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(dataproc_conf['edge_instance_hostname'],
+ dataproc_conf['key_path'],
+ dataproc_conf['dlab_ssh_user'],
+ 'dataengine-service',
+ dataproc_conf['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py
{}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure reverse proxy.", str(err))
+
actions_lib.GCPActions().delete_dataproc_cluster(dataproc_conf['cluster_name'],
os.environ['gcp_region'])
+ sys.exit(1)
+
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
@@ -118,7 +159,11 @@ if __name__ == "__main__":
dataproc_conf['edge_instance_name'] =
'{0}-{1}-edge'.format(dataproc_conf['service_base_name'],
dataproc_conf['edge_user_name'])
+ dataproc_conf['edge_instance_hostname'] =
GCPMeta().get_instance_public_ip_by_name(
+ dataproc_conf['edge_instance_name'])
dataproc_conf['dlab_ssh_user'] = os.environ['conf_os_user']
+ dataproc_conf['master_name'] = dataproc_conf['cluster_name'] + '-m'
+ dataproc_conf['master_ip'] =
meta_lib.GCPMeta().get_private_ip_address(dataproc_conf['master_name'])
try:
res = meta_lib.GCPMeta().get_list_instances(os.environ['gcp_zone'],
dataproc_conf['cluster_name'])
@@ -127,6 +172,11 @@ if __name__ == "__main__":
traceback.print_exc()
raise Exception
+ dataproc_conf['cluster_core_instances'] = list()
+ for instance in dataproc_conf['cluster_instances']:
+ if "{}-w-".format(dataproc_conf['cluster_name']) in instance:
+ dataproc_conf['cluster_core_instances'].append(instance)
+
try:
jobs = []
for instance in dataproc_conf['cluster_instances']:
@@ -143,6 +193,8 @@ if __name__ == "__main__":
raise Exception
try:
+ dataproc_master_acces_url = "http://"; +
dataproc_conf['edge_instance_hostname'] + "/{}/".format(
+ dataproc_conf['exploratory_name'] + '_' +
dataproc_conf['computational_name'])
logging.info('[SUMMARY]')
print('[SUMMARY]')
print("Service base name:
{}".format(dataproc_conf['service_base_name']))
@@ -164,7 +216,12 @@ if __name__ == "__main__":
"key_name": dataproc_conf['key_name'],
"instance_id": dataproc_conf['cluster_name'],
"user_own_bucket_name": dataproc_conf['bucket_name'],
- "Action": "Create new Dataproc cluster"}
+ "Action": "Create new Dataproc cluster",
+ "computational_url": [
+ {"description": "EMR Master",
+ "url": dataproc_master_acces_url}
+ ]
+ }
print(json.dumps(res))
result.write(json.dumps(res))
except:
diff --git
a/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
b/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
index 4e09b59..41454d2 100644
---
a/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
+++
b/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
@@ -47,7 +47,7 @@ def make_template():
ungit_template = environment.get_template('{}.conf'.format('ungit'))
tf_template = environment.get_template('{}.conf'.format('tensor'))
config = {}
- if args.type != 'emr' and args.type != 'spark':
+ if args.type != 'dataengine-service' and args.type != 'spark':
config['NAME'] = args.exploratory_name
config['IP'] = additional_info['instance_hostname']
elif args.type == 'spark':
@@ -67,7 +67,7 @@ def make_template():
slaves.append(slave)
config['slaves'] = slaves
conf_file_name = config['CLUSTER_NAME']
- elif args.type == 'emr':
+ elif args.type == 'dataengine-service':
config['CLUSTER_NAME'] = '{}_{}'.format(
args.exploratory_name, additional_info['computational_name'])
config['MASTER_IP'] = additional_info['master_ip']
@@ -79,7 +79,7 @@ def make_template():
f = open('/tmp/{}.conf'.format(conf_file_name), 'w')
f.write(template.render(config))
f.close()
- if args.type != 'emr' and args.type != 'spark':
+ if args.type != 'dataengine-service' and args.type != 'spark':
f = open('/tmp/{}.conf'.format(conf_file_name), 'a')
f.write(ungit_template.render(config))
f.close()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
