This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DATALAB-2409 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 645e7b63f67406d9e81c78cc71638c0f0384eb6e Author: bodnarmykola <[email protected]> AuthorDate: Wed Aug 18 12:07:50 2021 +0300 [DATALAB-2409] - edge lib refactored --- .../src/edge/scripts/configure_http_proxy.py | 3 +- .../src/general/lib/os/debian/edge_lib.py | 56 +--------------------- .../src/general/lib/os/fab.py | 51 ++++++++++++++++++++ .../src/general/lib/os/redhat/edge_lib.py | 31 ------------ .../src/project/scripts/configure_http_proxy.py | 3 +- .../src/project/scripts/configure_nftables.py | 3 +- .../src/project/templates/squid.conf | 6 +-- 7 files changed, 57 insertions(+), 96 deletions(-) diff --git a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py index 3580b43..0e9034e 100644 --- a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py +++ b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py @@ -24,7 +24,6 @@ import argparse import json import sys -from datalab.edge_lib import configure_http_proxy_server from fabric import * from datalab.fab import * @@ -48,5 +47,5 @@ if __name__ == "__main__": sys.exit(2) print("Installing proxy for notebooks.") - configure_http_proxy_server(deeper_config) + datalab.fab.configure_http_proxy_server(deeper_config) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py index 7a91691..34d1273 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py @@ -23,41 +23,12 @@ import os import sys -from datalab.common_lib import manage_pkg from fabric import * from patchwork.files import exists from patchwork import files import datalab.fab - -def configure_http_proxy_server(config): - try: - if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'): - manage_pkg('-y install', 'remote', 'squid') - template_file = config['template_file'] - proxy_subnet = config['exploratory_subnet'] - datalab.fab.conn.put(template_file, '/tmp/squid.conf') - datalab.fab.conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf') -# datalab.fab.conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet)) -# datalab.fab.conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name'])) -# datalab.fab.conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host'])) -# datalab.fab.conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn'])) -# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user'])) -# datalab.fab.conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password'])) -# datalab.fab.conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth')) - replace_string = '' - for cidr in config['vpc_cidrs']: - replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr) - datalab.fab.conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) - replace_string = '' - for cidr in config['allowed_ip_cidr']: - replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr) - datalab.fab.conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) - datalab.fab.conn.sudo('systemctl restart squid') - datalab.fab.conn.sudo('touch /tmp/http_proxy_ensured') - except Exception as err: - print("Failed to install and configure squid: " + str(err)) - sys.exit(1) - +from datalab.common_lib import manage_pkg +from datalab.logger import logging def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id, keycloak_client_secret, user, hostname, step_cert_sans): @@ -189,27 +160,4 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak datalab.fab.configure_nginx_LE(os.environ['conf_letsencrypt_domain_name'], os.environ['project_name'].lower()) except Exception as err: print("Failed install nginx with ldap: " + str(err)) - sys.exit(1) - -def configure_nftables(config): - try: - if not exists(datalab.fab.conn,'/tmp/nftables_ensured'): - manage_pkg('-y install', 'remote', 'nftables') - datalab.fab.conn.sudo('systemctl enable nftables.service') - datalab.fab.conn.sudo('systemctl start nftables') - datalab.fab.conn.sudo('sysctl net.ipv4.ip_forward=1') - if os.environ['conf_cloud_provider'] == 'aws': - interface = 'eth0' - elif os.environ['conf_cloud_provider'] == 'gcp': - interface = 'ens4' - datalab.fab.conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf') - datalab.fab.conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip'])) - datalab.fab.conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface)) - datalab.fab.conn.sudo( - 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet'])) - datalab.fab.conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/') - datalab.fab.conn.sudo('systemctl restart nftables') - datalab.fab.conn.sudo('touch /tmp/nftables_ensured') - except Exception as err: - print("Failed to configure nftables: " + str(err)) sys.exit(1) \ No newline at end of file diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py index 40c6c92..707bc60 100644 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@ -255,6 +255,57 @@ def configure_nginx_LE(domain_name, node): sys.exit(1) +#function for edge node only +def configure_http_proxy_server(config): + try: + if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'): + manage_pkg('-y install', 'remote', 'squid') + template_file = config['template_file'] + proxy_subnet = config['exploratory_subnet'] + conn.put(template_file, '/tmp/squid.conf') + conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf') + conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet)) + replace_string = '' + for cidr in config['vpc_cidrs']: + replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr) + conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) + replace_string = '' + for cidr in config['allowed_ip_cidr']: + replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr) + conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) + conn.sudo('systemctl restart squid') + fab.conn.sudo('touch /tmp/http_proxy_ensured') + except Exception as err: + logging.error('Fai to install and configure squid:', str(err)) + traceback.print_exc() + sys.exit(1) + + +def configure_nftables(config): + try: + if not exists(datalab.fab.conn,'/tmp/nftables_ensured'): + manage_pkg('-y install', 'remote', 'nftables') + conn.sudo('systemctl enable nftables.service') + conn.sudo('systemctl start nftables') + conn.sudo('sysctl net.ipv4.ip_forward=1') + if os.environ['conf_cloud_provider'] == 'aws': + interface = 'eth0' + elif os.environ['conf_cloud_provider'] == 'gcp': + interface = 'ens4' + conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf') + conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip'])) + conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface)) + conn.sudo( + 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet'])) + conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/') + conn.sudo('systemctl restart nftables') + conn.sudo('touch /tmp/nftables_ensured') + except Exception as err: + logging.error('Failed to configure nftables:', (err)) + traceback.print_exc() + sys.exit(1) + + # functions for all computation resources def ensure_python_venv(python_venv_version): try: diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py index ae81a2b..7617419 100644 --- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py @@ -29,37 +29,6 @@ from patchwork.files import exists from patchwork import files -def configure_http_proxy_server(config): - try: - if not exists(conn,'/tmp/http_proxy_ensured'): - manage_pkg('-y install', 'remote', 'squid') - template_file = config['template_file'] - proxy_subnet = config['exploratory_subnet'] - conn.put(template_file, '/tmp/squid.conf') - conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf') - conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet)) - conn.sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name'])) - conn.sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host'])) - conn.sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn'])) - conn.sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user'])) - conn.sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password'])) - conn.sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib64/squid/basic_ldap_auth')) - replace_string = '' - for cidr in config['vpc_cidrs']: - replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr) - conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) - replace_string = '' - for cidr in config['allowed_ip_cidr']: - replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr) - conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) - conn.sudo('systemctl restart squid') - conn.sudo('chkconfig squid on') - conn.sudo('touch /tmp/http_proxy_ensured') - except Exception as err: - print("Failed to install and configure squid: " + str(err)) - sys.exit(1) - - def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id, keycloak_client_secret, user, hostname, step_cert_sans): try: diff --git a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py index a692145..4af93ff 100644 --- a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py +++ b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py @@ -24,7 +24,6 @@ import argparse import json import sys -from datalab.edge_lib import configure_http_proxy_server from fabric import * from datalab.fab import * @@ -48,6 +47,6 @@ if __name__ == "__main__": sys.exit(2) print("Installing proxy for notebooks.") - configure_http_proxy_server(deeper_config) + datalab.fab.configure_http_proxy_server(deeper_config) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/project/scripts/configure_nftables.py b/infrastructure-provisioning/src/project/scripts/configure_nftables.py index b3c24a9..8fe14cd 100644 --- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py +++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py @@ -24,7 +24,6 @@ import argparse import json import sys -from datalab.edge_lib import configure_nftables from fabric import * from datalab.fab import * @@ -48,5 +47,5 @@ if __name__ == "__main__": sys.exit(2) print("Configuring nftables on edge node.") - configure_nftables(deeper_config) + datalab.fab.configure_nftables(deeper_config) conn.close() \ No newline at end of file diff --git a/infrastructure-provisioning/src/project/templates/squid.conf b/infrastructure-provisioning/src/project/templates/squid.conf index 0948b46..39a6cbf 100644 --- a/infrastructure-provisioning/src/project/templates/squid.conf +++ b/infrastructure-provisioning/src/project/templates/squid.conf @@ -19,8 +19,6 @@ # # ****************************************************************************** -#auth_param basic program LDAP_AUTH_PATH -b "LDAP_DN" -D "LDAP_SERVICE_USERNAME,LDAP_DN" -w LDAP_SERVICE_PASSWORD -f uid=%s LDAP_HOST - acl DataLab_user_src_subnet src PROXY_SUBNET VPC_CIDRS ALLOWED_CIDRS @@ -40,13 +38,11 @@ acl Safe_ports port 591 # filemaker acl Safe_ports port 777 # multiling http acl CONNECT method CONNECT -#acl ldap-auth proxy_auth EDGE_USER_NAME - http_access deny !Safe_ports http_access allow localhost manager http_access deny manager http_access allow DataLab_user_src_subnet -http_access allow AllowedCIDRS ldap-auth +http_access allow AllowedCIDRS http_access allow localhost http_access deny all --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
