This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-515
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-515 by this push:
new 1428f9a [DLAB-515]: EDGE as NAT configuration implemented
1428f9a is described below
commit 1428f9a63707632e6f7a01a50ae4203068a498f2
Author: bodnarmykola <[email protected]>
AuthorDate: Fri May 8 18:02:54 2020 +0300
[DLAB-515]: EDGE as NAT configuration implemented
---
.../src/general/conf/dlab.ini | 4 +-
.../src/general/lib/aws/actions_lib.py | 27 +++++++++++
.../src/general/lib/os/debian/edge_lib.py | 15 +++++++
.../general/scripts/aws/common_create_subnet.py | 27 ++++++-----
.../src/general/scripts/aws/edge_configure.py | 19 ++++++++
.../scripts/aws/edge_configure_route_table.py | 43 ++++++++++++++++++
.../src/general/scripts/aws/project_prepare.py | 38 ++++++++++++++++
.../templates/os/py3spark_local_template.json | 6 +--
.../src/project/scripts/configure_nftables.py | 52 ++++++++++++++++++++++
.../src/project/templates/nftables.conf | 22 +++++++++
10 files changed, 237 insertions(+), 16 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini
b/infrastructure-provisioning/src/general/conf/dlab.ini
index 805b33e..8f93151 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -217,6 +217,8 @@ docker_version = 18.06.3
# user_name =
### Elastic IP which will be associated with Edge node
# elastic_ip =
+### Edge node is NAT
+is_nat = True
#--- [notebook] section contains all parameters that are using for all
notebooks provisioning ---#
[notebook]
@@ -356,4 +358,4 @@ expl_instance_memory = 8000
#--- [reverse_proxy] reverse proxy settings ---#
[reverse_proxy]
### Nginx version
-nginx_version = 1.15.1
+nginx_version = 1.15.1
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 9519053..ba21ffd 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -224,6 +224,24 @@ def create_rt(vpc_id, infra_tag_name, infra_tag_value,
secondary):
"error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+def create_nat_rt(vpc_id, infra_tag_value, edge_instance_id,
private_subnet_id):
+ try:
+ ec2 = boto3.client('ec2')
+ nat_rt = ec2.create_route_table(VpcId=vpc_id)
+ nat_rt_id = nat_rt.get('RouteTable').get('RouteTableId')
+ tag = {"Key": 'Name', "Value": infra_tag_value}
+ create_tag(nat_rt_id, json.dumps(tag))
+ ec2 = boto3.resource('ec2')
+ route_table = ec2.RouteTable(nat_rt_id)
+ route_table.create_route(DestinationCidrBlock='0.0.0.0/0',
InstanceId=edge_instance_id)
+ route_table.associate_with_subnet(SubnetId=private_subnet_id)
+ except Exception as err:
+ logging.info(
+ "Unable to create Route Table: " + str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout))
+ append_result(str({"error": "Unable to create Route Table",
+ "error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+
def create_subnet(vpc_id, subnet, tag, zone):
try:
@@ -474,6 +492,15 @@ def create_instance(definitions, instance_tag,
primary_disk_size=12):
"error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+def modify_instance_sourcedescheck(instance_id):
+ try:
+ ec2 = boto3.client('ec2')
+ ec2.modify_instance_attribute(InstanceId=instance_id,
SourceDestCheck={'Value': False})
+ except Exception as err:
+ logging.info("Unable to modify EC2: " + str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout))
+ append_result(str({"error": "Unable to modify EC2",
+ "error_message": str(err) + "\n Traceback: " +
traceback.print_exc(file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
def tag_intance_volume(instance_id, node_name, instance_tag):
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 3d5773e..9bc02f2 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -151,4 +151,19 @@ def install_nginx_lua(edge_ip, nginx_version,
keycloak_auth_server_url, keycloak
sudo('touch /tmp/nginx_installed')
except Exception as err:
print("Failed install nginx with ldap: " + str(err))
+ sys.exit(1)
+
+def configure_nftables(config):
+ try:
+ if not exists('/tmp/nftables_ensured'):
+ manage_pkg('-y install', 'remote', 'nftables')
+ sudo('systemctl enable nftables.service')
+ sudo('sysctl net.ipv4.ip_forward=1')
+ sudo('sed -i \'s/EDGE_IP/{}/g\'
/opt/dlab/templates/nftables.conf'.format(config['edge_ip']))
+ sudo('sed -i "s|SUBNET_CIDR|{}|g"
/opt/dlab/templates/nftables.conf'.format(config['exploratory_subnet']))
+ sudo('cp /opt/dlab/templates/nftables.conf /etc/')
+ sudo('systemctl restart nftables')
+ sudo('touch /tmp/nftables_ensured')
+ except Exception as err:
+ print("Failed to configure nftables: " + str(err))
sys.exit(1)
\ No newline at end of file
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
index b4dc3c6..83dd48a 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
@@ -129,20 +129,23 @@ if __name__ == "__main__":
subnet_id = get_subnet_by_cidr(subnet_check)
print("SUBNET_ID: {}".format(subnet_id))
if not args.ssn:
- print("Associating route_table with the subnet")
- ec2 = boto3.resource('ec2')
- if os.environ['conf_duo_vpc_enable'] == 'true':
- rt = get_route_table_by_tag(args.infra_tag_value +
'-secondary-tag', args.infra_tag_value)
+ if os.environ['edge_is_nat'] == 'true':
+ print('Subnet will be associted with route table for NAT')
else:
- rt = get_route_table_by_tag(args.infra_tag_name,
args.infra_tag_value)
- route_table = ec2.RouteTable(rt)
- try:
- route_table.associate_with_subnet(SubnetId=subnet_id)
+ print("Associating route_table with the subnet")
+ ec2 = boto3.resource('ec2')
if os.environ['conf_duo_vpc_enable'] == 'true':
- create_peer_routes(os.environ['aws_peering_id'],
args.infra_tag_value)
- except exceptions.ClientError as err:
- if 'Resource.AlreadyAssociated' in str(err):
- print('Other route table is already associted with this
subnet. Skipping...')
+ rt = get_route_table_by_tag(args.infra_tag_value +
'-secondary-tag', args.infra_tag_value)
+ else:
+ rt = get_route_table_by_tag(args.infra_tag_name,
args.infra_tag_value)
+ route_table = ec2.RouteTable(rt)
+ try:
+ route_table.associate_with_subnet(SubnetId=subnet_id)
+ if os.environ['conf_duo_vpc_enable'] == 'true':
+ create_peer_routes(os.environ['aws_peering_id'],
args.infra_tag_value)
+ except exceptions.ClientError as err:
+ if 'Resource.AlreadyAssociated' in str(err):
+ print('Other route table is already associted with
this subnet. Skipping...')
else:
print("Associating route_table with the subnet")
ec2 = boto3.resource('ec2')
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
index d96ef49..5239386 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
@@ -245,6 +245,25 @@ if __name__ == "__main__":
sys.exit(1)
try:
+ print('[CONFIGRING EDGE AS NAT]')
+ if os.environ['edge_is_nat'] == 'true':
+ print('Installing nftables')
+ additional_config = {"exploratory_subnet":
edge_conf['private_subnet_cidr'],
+ "edge_ip": edge_conf['edge_private_ip']}
+ params = "--hostname {} --keyfile {} --additional_config '{}'
--user {}".format(
+ edge_conf['instance_hostname'], edge_conf['keyfile_name'],
json.dumps(additional_config),
+ edge_conf['dlab_ssh_user'])
+ try:
+ local("~/scripts/{}.py {}".format('configure_nftables',
params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ dlab.fab.append_result("Failed to configure NAT." + str(err))
+ clear_resources()
+ sys.exit(1)
+
+ try:
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(edge_conf['instance_name']))
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
new file mode 100644
index 0000000..75acd7a
--- /dev/null
+++
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
@@ -0,0 +1,43 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
******************************************************************************
+
+import argparse
+from dlab.actions_lib import *
+from dlab.meta_lib import *
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--vpc_id', type=str, default='')
+parser.add_argument('--infra_tag_value', type=str, default='')
+parser.add_argument('--edge_instance_id', type=str, default='')
+parser.add_argument('--private_subnet_id', type=str, default='')
+args = parser.parse_args()
+
+sec_str = ''
+if args.secondary:
+ sec_str = 'SECONDARY '
+
+
+if __name__ == "__main__":
+ rt_id = create_nat_rt(args.vpc_id, args.infra_tag_value,
args.edge_instance_id, args.private_subnet_id)
+
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 9d44ba5..b2ed5e9 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -60,6 +60,7 @@ if __name__ == "__main__":
os.environ['conf_os_family'])])
project_conf['instance_size'] = os.environ['aws_edge_instance_size']
project_conf['sg_ids'] = os.environ['aws_security_groups_ids']
+ project_conf['instance_class'] = 'edge'
project_conf['edge_instance_name'] =
'{}-{}-{}-edge'.format(project_conf['service_base_name'],
project_conf['project_name'],
project_conf['endpoint_name'])
@@ -622,6 +623,12 @@ if __name__ == "__main__":
local("~/scripts/{}.py {}".format('common_create_instance',
params))
edge_instance =
dlab.meta_lib.get_instance_by_name(project_conf['tag_name'],
project_conf['edge_instance_name'])
+ if os.environ['edge_is_nat']:
+ try:
+
dlab.actions_lib.modify_instance_sourcedescheck(edge_instance)
+ except:
+ traceback.print_exc()
+ raise Exception
except:
traceback.print_exc()
raise Exception
@@ -670,3 +677,34 @@ if __name__ == "__main__":
dlab.actions_lib.remove_sgroups(project_conf['edge_instance_name'])
dlab.actions_lib.remove_s3('edge', project_conf['project_name'])
sys.exit(1)
+
+ if os.environ['edge_is_nat'] == 'true':
+ try:
+ print('[CONFIGURING ROUTE TABLE FOR NAT]')
+ project_conf['nat_rt_name'] =
'{0}-{1}-{2}-nat-rt'.format(project_conf['service_base_name'],
+
project_conf['project_name'],
+
project_conf['endpoint_name'])
+ params = "--vpc_id {} --infra_tag_value {} --edge_instance_id {}
--private_subnet_id {}".format(
+ project_conf['vpc2_id'], project_conf['nat_rt_name'],
edge_instance, subnet_id)
+ try:
+ local("~/scripts/{}.py
{}".format('edge_configure_route_table', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ dlab.fab.append_result("Failed to configure route table.",
str(err))
+ try:
+ project_conf['edge_public_ip'] =
dlab.meta_lib.get_instance_ip_address(
+ project_conf['tag_name'],
project_conf['edge_instance_name']).get('Public')
+ project_conf['allocation_id'] =
dlab.meta_lib.get_allocation_id_by_elastic_ip(
+ project_conf['edge_public_ip'])
+ except:
+ print("No Elastic IPs to release!")
+ dlab.actions_lib.remove_ec2(project_conf['tag_name'],
project_conf['edge_instance_name'])
+ dlab.actions_lib.remove_all_iam_resources('notebook',
project_conf['project_name'])
+ dlab.actions_lib.remove_all_iam_resources('edge',
project_conf['project_name'])
+
dlab.actions_lib.remove_sgroups(project_conf['dataengine_instances_name'])
+
dlab.actions_lib.remove_sgroups(project_conf['notebook_instance_name'])
+ dlab.actions_lib.remove_sgroups(project_conf['edge_instance_name'])
+ dlab.actions_lib.remove_s3('edge', project_conf['project_name'])
+ sys.exit(1)
\ No newline at end of file
diff --git
a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
index 969228e..e03c629 100644
---
a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
+++
b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
@@ -1,15 +1,15 @@
{
"argv": [
- "/usr/bin/python3.5",
+ "/usr/bin/python3",
"-m",
"ipykernel",
"-f",
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-3.5 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-3.6 / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python3.5",
+ "PYSPARK_PYTHON": "python3.6",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
diff --git
a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
new file mode 100644
index 0000000..c98a80b
--- /dev/null
+++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
@@ -0,0 +1,52 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
******************************************************************************
+
+from fabric.api import *
+from fabric.contrib.files import exists
+from dlab.edge_lib import configure_http_proxy_server
+import argparse
+import json
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--hostname', type=str, default='')
+parser.add_argument('--keyfile', type=str, default='')
+parser.add_argument('--user', type=str, default='')
+parser.add_argument('--additional_config', type=str,
default='{"empty":"string"}')
+args = parser.parse_args()
+
+##############
+# Run script #
+##############
+if __name__ == "__main__":
+ print("Configure connections")
+ try:
+ env['connection_attempts'] = 100
+ env.key_filename = [args.keyfile]
+ env.host_string = '{}@{}'.format(args.user, args.hostname)
+ deeper_config = json.loads(args.additional_config)
+ except:
+ sys.exit(2)
+
+ print("Configuring nftables on edge node.")
+ configure_nftables(deeper_config)
diff --git a/infrastructure-provisioning/src/project/templates/nftables.conf
b/infrastructure-provisioning/src/project/templates/nftables.conf
new file mode 100644
index 0000000..103bfc2
--- /dev/null
+++ b/infrastructure-provisioning/src/project/templates/nftables.conf
@@ -0,0 +1,22 @@
+#!/usr/sbin/nft -f
+
+flush ruleset
+
+table ip nat {
+ chain postrouting {
+ type nat hook postrouting priority 100; policy accept;
+ ip saddr SUBNET_CIDR oif "eth0" snat to EDGE_IP
+ }
+}
+
+table inet filter {
+ chain input {
+ type filter hook input priority 0;
+ }
+ chain forward {
+ type filter hook forward priority 0;
+ }
+ chain output {
+ type filter hook output priority 0;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]