This is an automated email from the ASF dual-hosted git repository.

mykolabodnar pushed a commit to branch DLAB-515
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git


The following commit(s) were added to refs/heads/DLAB-515 by this push:
     new ec13ff1  [DLAB-515]: EDGE as NAT configuration fixed / EMR version 
updated
     new b2e61b8  Merge remote-tracking branch 'origin/DLAB-515' into DLAB-515
ec13ff1 is described below

commit ec13ff16dfbfa9c800a6d9d6725598a5ef57c854
Author: bodnarmykola <[email protected]>
AuthorDate: Wed May 20 09:06:08 2020 +0300

    [DLAB-515]: EDGE as NAT configuration fixed / EMR version updated
---
 infrastructure-provisioning/src/general/conf/dlab.ini       |  2 +-
 .../general/files/aws/dataengine-service_description.json   |  4 ++--
 .../src/general/lib/os/debian/edge_lib.py                   |  2 ++
 .../src/general/scripts/aws/edge_configure_route_table.py   |  5 -----
 .../src/general/scripts/aws/project_prepare.py              |  6 ++++++
 .../src/project/scripts/configure_nftables.py               |  2 +-
 .../src/project/templates/nftables.conf                     | 13 ++++++-------
 7 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini 
b/infrastructure-provisioning/src/general/conf/dlab.ini
index 8f93151..458a4e5 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -218,7 +218,7 @@ docker_version = 18.06.3
 ### Elastic IP which will be associated with Edge node
 # elastic_ip =
 ### Edge node is NAT
-is_nat = True
+is_nat = true
 
 #--- [notebook] section contains all parameters that are using for all 
notebooks provisioning ---#
 [notebook]
diff --git 
a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
 
b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
index b4e5ba2..c95503d 100644
--- 
a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
+++ 
b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
@@ -24,7 +24,7 @@
     },
   "templates":
   [
-    {"version":"emr-5.19.0", "applications": [{"Name":"Hadoop", "Version": 
"2.8.5"}, {"Name":"Spark", "Version": "2.3.2"}, {"Name":"Hive", "Version": 
"2.3.3"}]},
-    {"version":"emr-5.28.0", "applications": [{"Name":"Hadoop", "Version": 
"2.8.5"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version": 
"2.3.6"}]}
+    {"version":"emr-5.30.0", "applications": [{"Name":"Hadoop", "Version": 
"2.8.5"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version": 
"2.3.6"}]},
+    {"version":"emr-6.0.0", "applications": [{"Name":"Hadoop", "Version": 
"3.2.1"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version": 
"3.1.2"}]}
   ]
 }
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py 
b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 9bc02f2..1130054 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -158,7 +158,9 @@ def configure_nftables(config):
         if not exists('/tmp/nftables_ensured'):
             manage_pkg('-y install', 'remote', 'nftables')
             sudo('systemctl enable nftables.service')
+            sudo('systemctl start nftables')
             sudo('sysctl net.ipv4.ip_forward=1')
+            sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' 
/etc/sysctl.conf')
             sudo('sed -i \'s/EDGE_IP/{}/g\' 
/opt/dlab/templates/nftables.conf'.format(config['edge_ip']))
             sudo('sed -i "s|SUBNET_CIDR|{}|g" 
/opt/dlab/templates/nftables.conf'.format(config['exploratory_subnet']))
             sudo('cp /opt/dlab/templates/nftables.conf /etc/')
diff --git 
a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
 
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
index 75acd7a..c3ac2e4 100644
--- 
a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
+++ 
b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
@@ -33,11 +33,6 @@ parser.add_argument('--edge_instance_id', type=str, 
default='')
 parser.add_argument('--private_subnet_id', type=str, default='')
 args = parser.parse_args()
 
-sec_str = ''
-if args.secondary:
-    sec_str = 'SECONDARY '
-
-
 if __name__ == "__main__":
     rt_id = create_nat_rt(args.vpc_id, args.infra_tag_value, 
args.edge_instance_id, args.private_subnet_id)
 
diff --git 
a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py 
b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index b2ed5e9..9986ab2 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -401,6 +401,12 @@ if __name__ == "__main__":
                 "FromPort": 389,
                 "IpRanges": [{"CidrIp": project_conf['all_ip_cidr']}],
                 "ToPort": 389, "IpProtocol": "tcp", "UserIdGroupPairs": []
+            },
+            {
+                "PrefixListIds": [],
+                "FromPort": -1,
+                "IpRanges": [{"CidrIp": project_conf['all_ip_cidr']}],
+                "ToPort": -1, "IpProtocol": "icmp", "UserIdGroupPairs": []
             }
         ])
         params = "--name {} --vpc_id {} --security_group_rules '{}' 
--infra_tag_name {} --infra_tag_value {} \
diff --git 
a/infrastructure-provisioning/src/project/scripts/configure_nftables.py 
b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
index c98a80b..de4dc12 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
@@ -23,7 +23,7 @@
 
 from fabric.api import *
 from fabric.contrib.files import exists
-from dlab.edge_lib import configure_http_proxy_server
+from dlab.edge_lib import configure_nftables
 import argparse
 import json
 import sys
diff --git a/infrastructure-provisioning/src/project/templates/nftables.conf 
b/infrastructure-provisioning/src/project/templates/nftables.conf
index 103bfc2..8bd5e3f 100644
--- a/infrastructure-provisioning/src/project/templates/nftables.conf
+++ b/infrastructure-provisioning/src/project/templates/nftables.conf
@@ -2,13 +2,6 @@
 
 flush ruleset
 
-table ip nat {
-        chain postrouting {
-                type nat hook postrouting priority 100; policy accept;
-                ip saddr SUBNET_CIDR oif "eth0" snat to EDGE_IP
-        }
-}
-
 table inet filter {
         chain input {
                 type filter hook input priority 0;
@@ -19,4 +12,10 @@ table inet filter {
         chain output {
                 type filter hook output priority 0;
         }
+}
+table ip nat {
+        chain postrouting {
+                type nat hook postrouting priority 100; policy accept;
+                ip saddr SUBNET_CIDR oif "eth0" snat to EDGE_IP
+        }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to