This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-515
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-515 by this push:
new ec13ff1 [DLAB-515]: EDGE as NAT configuration fixed / EMR version
updated
new b2e61b8 Merge remote-tracking branch 'origin/DLAB-515' into DLAB-515
ec13ff1 is described below
commit ec13ff16dfbfa9c800a6d9d6725598a5ef57c854
Author: bodnarmykola <[email protected]>
AuthorDate: Wed May 20 09:06:08 2020 +0300
[DLAB-515]: EDGE as NAT configuration fixed / EMR version updated
---
infrastructure-provisioning/src/general/conf/dlab.ini | 2 +-
.../general/files/aws/dataengine-service_description.json | 4 ++--
.../src/general/lib/os/debian/edge_lib.py | 2 ++
.../src/general/scripts/aws/edge_configure_route_table.py | 5 -----
.../src/general/scripts/aws/project_prepare.py | 6 ++++++
.../src/project/scripts/configure_nftables.py | 2 +-
.../src/project/templates/nftables.conf | 13 ++++++-------
7 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini
b/infrastructure-provisioning/src/general/conf/dlab.ini
index 8f93151..458a4e5 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -218,7 +218,7 @@ docker_version = 18.06.3
### Elastic IP which will be associated with Edge node
# elastic_ip =
### Edge node is NAT
-is_nat = True
+is_nat = true
#--- [notebook] section contains all parameters that are using for all
notebooks provisioning ---#
[notebook]
diff --git
a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
index b4e5ba2..c95503d 100644
---
a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
+++
b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
@@ -24,7 +24,7 @@
},
"templates":
[
- {"version":"emr-5.19.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.5"}, {"Name":"Spark", "Version": "2.3.2"}, {"Name":"Hive", "Version":
"2.3.3"}]},
- {"version":"emr-5.28.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.5"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version":
"2.3.6"}]}
+ {"version":"emr-5.30.0", "applications": [{"Name":"Hadoop", "Version":
"2.8.5"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version":
"2.3.6"}]},
+ {"version":"emr-6.0.0", "applications": [{"Name":"Hadoop", "Version":
"3.2.1"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version":
"3.1.2"}]}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 9bc02f2..1130054 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -158,7 +158,9 @@ def configure_nftables(config):
if not exists('/tmp/nftables_ensured'):
manage_pkg('-y install', 'remote', 'nftables')
sudo('systemctl enable nftables.service')
+ sudo('systemctl start nftables')
sudo('sysctl net.ipv4.ip_forward=1')
+ sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\'
/etc/sysctl.conf')
sudo('sed -i \'s/EDGE_IP/{}/g\'
/opt/dlab/templates/nftables.conf'.format(config['edge_ip']))
sudo('sed -i "s|SUBNET_CIDR|{}|g"
/opt/dlab/templates/nftables.conf'.format(config['exploratory_subnet']))
sudo('cp /opt/dlab/templates/nftables.conf /etc/')
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
index 75acd7a..c3ac2e4 100644
---
a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
+++
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
@@ -33,11 +33,6 @@ parser.add_argument('--edge_instance_id', type=str,
default='')
parser.add_argument('--private_subnet_id', type=str, default='')
args = parser.parse_args()
-sec_str = ''
-if args.secondary:
- sec_str = 'SECONDARY '
-
-
if __name__ == "__main__":
rt_id = create_nat_rt(args.vpc_id, args.infra_tag_value,
args.edge_instance_id, args.private_subnet_id)
diff --git
a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index b2ed5e9..9986ab2 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -401,6 +401,12 @@ if __name__ == "__main__":
"FromPort": 389,
"IpRanges": [{"CidrIp": project_conf['all_ip_cidr']}],
"ToPort": 389, "IpProtocol": "tcp", "UserIdGroupPairs": []
+ },
+ {
+ "PrefixListIds": [],
+ "FromPort": -1,
+ "IpRanges": [{"CidrIp": project_conf['all_ip_cidr']}],
+ "ToPort": -1, "IpProtocol": "icmp", "UserIdGroupPairs": []
}
])
params = "--name {} --vpc_id {} --security_group_rules '{}'
--infra_tag_name {} --infra_tag_value {} \
diff --git
a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
index c98a80b..de4dc12 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
@@ -23,7 +23,7 @@
from fabric.api import *
from fabric.contrib.files import exists
-from dlab.edge_lib import configure_http_proxy_server
+from dlab.edge_lib import configure_nftables
import argparse
import json
import sys
diff --git a/infrastructure-provisioning/src/project/templates/nftables.conf
b/infrastructure-provisioning/src/project/templates/nftables.conf
index 103bfc2..8bd5e3f 100644
--- a/infrastructure-provisioning/src/project/templates/nftables.conf
+++ b/infrastructure-provisioning/src/project/templates/nftables.conf
@@ -2,13 +2,6 @@
flush ruleset
-table ip nat {
- chain postrouting {
- type nat hook postrouting priority 100; policy accept;
- ip saddr SUBNET_CIDR oif "eth0" snat to EDGE_IP
- }
-}
-
table inet filter {
chain input {
type filter hook input priority 0;
@@ -19,4 +12,10 @@ table inet filter {
chain output {
type filter hook output priority 0;
}
+}
+table ip nat {
+ chain postrouting {
+ type nat hook postrouting priority 100; policy accept;
+ ip saddr SUBNET_CIDR oif "eth0" snat to EDGE_IP
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]