This is an automated email from the ASF dual-hosted git repository. lfrolov pushed a commit to branch DATALAB-2567 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 574662f5270c30b0277fb5c4c8b5897558bedffd Author: leonidfrolov <[email protected]> AuthorDate: Tue Dec 7 11:33:18 2021 +0200 [DATALAB-2567]: replaced scp with rsync and disabled scp --- .../src/base/scripts/install_prerequisites.py | 4 ++-- .../src/general/lib/os/debian/ssn_lib.py | 2 +- .../src/general/lib/os/redhat/ssn_lib.py | 2 +- .../aws/jupyter_install_dataengine-service_kernels.py | 2 +- .../aws/rstudio_install_dataengine-service_kernels.py | 2 +- .../aws/zeppelin_install_dataengine-service_kernels.py | 2 +- .../gcp/jupyter_install_dataengine-service_kernels.py | 2 +- .../gcp/rstudio_install_dataengine-service_kernels.py | 2 +- .../gcp/zeppelin_install_dataengine-service_kernels.py | 2 +- .../src/ssn/scripts/configure_docker.py | 8 ++++---- .../src/ssn/scripts/configure_ssn_node.py | 4 ++-- infrastructure-provisioning/src/ssn/scripts/configure_ui.py | 12 ++++++------ 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py index 430805f..007f0e1 100644 --- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py +++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py @@ -60,8 +60,8 @@ if __name__ == "__main__": traceback.print_exc() sys.exit(1) - #logging.info("Disable scp on nodes") - #disable_edge_scp_binary(args.user) + logging.info("Disable scp on nodes") + disable_edge_scp_binary(args.user) #logging.info("Updating openssh to version") #ensure_openssh_version(args.user) diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py index 5439abc..ae4f6bb 100644 --- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py @@ -270,7 +270,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path, append_result("Unable to upload webapp jars") sys.exit(1) if billing_enabled: - datalab.fab.conn.local('scp -i {} /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile, + datalab.fab.conn.local('rsync -e "ssh -i {}" /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile, host_string)) params = '--cloud_provider {} ' \ '--infrastructure_tag {} ' \ diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py index 39ad178..4ebed0f 100644 --- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py +++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py @@ -289,7 +289,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir, web_path, sys.exit(1) if billing_enabled: - conn.local('scp -i {} /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile, + conn.local('rsync -e "ssh -i {}" /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile, host_string)) params = '--cloud_provider {} ' \ '--infrastructure_tag {} ' \ diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py index 0efb00f..6e76b31 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py @@ -67,7 +67,7 @@ def configure_notebook(args): conn.sudo('chmod 755 /usr/local/bin/jupyter_dataengine-service_create_configs.py') conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /tmp/datalab_libs/') - subprocess.run('scp -i {} /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs'.format(args.keyfile, args.os_user, args.notebook_ip), shell=True, check=True) + subprocess.run('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs'.format(args.keyfile, args.os_user, args.notebook_ip), shell=True, check=True) conn.run('chmod a+x /tmp/datalab_libs/*') conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/') if exists(conn, '/usr/lib64'): diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py index 66eccb5..75d4db6 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py @@ -53,7 +53,7 @@ def configure_notebook(args): conn.sudo('chmod 755 /usr/local/bin/rstudio_dataengine-service_create_configs.py') conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /tmp/datalab_libs/') - conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) + conn.local('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) conn.run('chmod a+x /tmp/datalab_libs/*') conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/') if exists(conn, '/usr/lib64'): diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py index ae2616c..311cb04 100644 --- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py @@ -58,7 +58,7 @@ def configure_notebook(args): conn.sudo('chmod 755 /usr/local/bin/zeppelin_dataengine-service_create_configs.py') conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /tmp/datalab_libs/') - conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) + conn.local('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}@{}:/tmp/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) conn.run('chmod a+x /tmp/datalab_libs/*') conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/') if exists(conn, '/usr/lib64'): diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py index 7fc3ae9..a209d90 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py @@ -63,7 +63,7 @@ def configure_notebook(args): datalab.fab.conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') datalab.fab.conn.run('mkdir -p /tmp/datalab_libs/') host_string = args.os_user + "@" + args.notebook_ip - datalab.fab.conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string)) + datalab.fab.conn.local('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string)) datalab.fab.conn.run('chmod a+x /tmp/datalab_libs/*') datalab.fab.conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/') if exists(datalab.fab.conn, '/usr/lib64'): diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py index be225cd..f098a95 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py @@ -55,7 +55,7 @@ def configure_notebook(args): conn.sudo('chmod 755 /usr/local/bin/create_configs.py') conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /home/{}/datalab_libs/'.format(args.os_user)) - conn.local('scp -i {0} /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) + conn.local('rsync -e "ssh -i {0}" /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) conn.run('chmod a+x /home/{}/datalab_libs/*'.format(args.os_user)) conn.sudo('mv /home/{}/datalab_libs/* /usr/lib/python3.8/datalab/'.format(args.os_user)) conn.sudo('rm -rf /home/{}/datalab_libs/'.format(args.os_user)) diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py index 8269b93..e17520a 100644 --- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py +++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py @@ -58,7 +58,7 @@ def configure_notebook(args): conn.sudo('chmod 755 /usr/local/bin/create_configs.py') conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /home/{}/datalab_libs/'.format(args.os_user)) - conn.local('scp -i {0} /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) + conn.local('rsync -e "ssh -i {0}" /usr/lib/python3.8/datalab/*.py {1}@{2}:/home/{1}/datalab_libs/'.format(args.keyfile, args.os_user, args.notebook_ip)) conn.run('chmod a+x /home/{}/datalab_libs/*'.format(args.os_user)) conn.sudo('mv /home/{}/datalab_libs/* /usr/lib/python3.8/datalab/'.format(args.os_user)) conn.sudo('rm -rf /home/{}/datalab_libs/'.format(args.os_user)) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py index 1e6e31d..e379440 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py @@ -52,8 +52,8 @@ def modify_conf_file(args): for os_var in os.environ: if "'" not in os.environ[os_var] and os_var != 'aws_access_key' and os_var != 'aws_secret_access_key': variables_list[os_var] = os.environ[os_var] - conn.local('scp -r -i {} /project_tree/* {}:{}sources/'.format(args.keyfile, host_string, args.datalab_path)) - conn.local('scp -i {} /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile, + conn.local('rsync -r -e "ssh -i {}" /project_tree/* {}:{}sources/'.format(args.keyfile, host_string, args.datalab_path)) + conn.local('rsync -e "ssh -i {}" /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile, host_string)) conn.sudo("python3 /tmp/configure_conf_file.py --datalab_dir {} --variables_list '{}'".format( args.datalab_path, json.dumps(variables_list))) @@ -92,7 +92,7 @@ def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider): host_string = '{}@{}'.format(args.os_user, args.hostname) with open('/tmp/config', 'w') as f: f.write(base64.b64decode(gcr_creds)) - conn.local('scp -i {} /tmp/config {}:/tmp/config'.format(args.keyfile, host_string, os_user)) + conn.local('rsync -e "ssh -i {}" /tmp/config {}:/tmp/config'.format(args.keyfile, host_string, os_user)) conn.sudo('mkdir /home/{}/.docker'.format(os_user)) conn.sudo('cp /tmp/config /home/{}/.docker/config.json'.format(os_user)) conn.sudo('sed -i "s|ODAHU_IMAGE|{}|" {}sources/infrastructure-provisioning/src/general/files/{}/odahu_Dockerfile' @@ -110,7 +110,7 @@ def build_docker_images(image_list): try: host_string = '{}@{}'.format(args.os_user, args.hostname) if os.environ['conf_cloud_provider'] == 'azure': - conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/' + conn.local('rsync -e "ssh -i {}" /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/' 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path)) conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json ' '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user)) diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py index 4b0cde6..f411fbb 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py @@ -63,7 +63,7 @@ def cp_key(keyfile, host_string, os_user): key_name=keyfile.split("/") conn.sudo('mkdir -p /home/' + os_user + '/keys') conn.sudo('chown -R ' + os_user + ':' + os_user + ' /home/' + os_user + '/keys') - conn.local('scp -r -q -i {0} {0} {1}:/home/{3}/keys/{2}'.format(keyfile, host_string, key_name[-1], os_user)) + conn.local('rsync -r -q -e "ssh -i {0}" {0} {1}:/home/{3}/keys/{2}'.format(keyfile, host_string, key_name[-1], os_user)) conn.sudo('chmod 600 /home/' + os_user + '/keys/*.pem') except Exception as err: traceback.print_exc() @@ -210,7 +210,7 @@ def copy_ssn_libraries(): conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /tmp/datalab_libs/') subprocess.run( - 'scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string), + 'rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string), shell=True, check=True) conn.run('chmod a+x /tmp/datalab_libs/*') conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/') diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py index 4a4476e..0c3a871 100644 --- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py +++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py @@ -95,7 +95,7 @@ def copy_ssn_libraries(): try: conn.sudo('mkdir -p /usr/lib/python3.8/datalab/') conn.run('mkdir -p /tmp/datalab_libs/') - subprocess.run('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string), shell=True, check=True) + subprocess.run('rsync -e "ssh -i {}" /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string), shell=True, check=True) conn.run('chmod a+x /tmp/datalab_libs/*') conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/') if exists(conn, '/usr/lib64'): @@ -113,23 +113,23 @@ def configure_mongo(mongo_passwd, default_endpoint_name): subprocess.run('sed -i "s/MONGO_USR/mongodb/g" /root/templates/mongod.service_template', shell=True, check=True) elif os.environ['conf_os_family'] == 'redhat': subprocess.run('sed -i "s/MONGO_USR/mongod/g" /root/templates/mongod.service_template', shell=True, check=True) - subprocess.run('scp -i {} /root/templates/mongod.service_template {}:/tmp/mongod.service'.format(args.keyfile, + subprocess.run('rsync -e "ssh -i {}" /root/templates/mongod.service_template {}:/tmp/mongod.service'.format(args.keyfile, host_string), shell=True, check=True) conn.sudo('mv /tmp/mongod.service /lib/systemd/system/mongod.service') conn.sudo('systemctl daemon-reload') conn.sudo('systemctl enable mongod.service') subprocess.run('sed -i "s|PASSWORD|{}|g" /root/scripts/resource_status.py'.format(mongo_passwd), shell=True, check=True) - subprocess.run('scp -i {} /root/scripts/resource_status.py {}:/tmp/resource_status.py'.format(args.keyfile, + subprocess.run('rsync -e "ssh -i {}" /root/scripts/resource_status.py {}:/tmp/resource_status.py'.format(args.keyfile, host_string), shell=True, check=True) conn.sudo('mv /tmp/resource_status.py ' + os.environ['ssn_datalab_path'] + 'tmp/') subprocess.run('sed -i "s|PASSWORD|{}|g" /root/scripts/configure_mongo.py'.format(mongo_passwd), shell=True, check=True) - subprocess.run('scp -i {} /root/scripts/configure_mongo.py {}:/tmp/configure_mongo.py'.format(args.keyfile, + subprocess.run('rsync -e "ssh -i {}" /root/scripts/configure_mongo.py {}:/tmp/configure_mongo.py'.format(args.keyfile, host_string), shell=True, check=True) conn.sudo('mv /tmp/configure_mongo.py ' + args.datalab_path + 'tmp/') - subprocess.run('scp -i {} /root/files/{}/mongo_roles.json {}:/tmp/mongo_roles.json'.format(args.keyfile, + subprocess.run('rsync -e "ssh -i {}" /root/files/{}/mongo_roles.json {}:/tmp/mongo_roles.json'.format(args.keyfile, args.cloud_provider, host_string), shell=True, check=True) - subprocess.run('scp -i {} /root/files/local_endpoint.json {}:/tmp/local_endpoint.json'.format(args.keyfile, + subprocess.run('rsync -e "ssh -i {}" /root/files/local_endpoint.json {}:/tmp/local_endpoint.json'.format(args.keyfile, host_string), shell=True, check=True) conn.sudo('mv /tmp/mongo_roles.json ' + args.datalab_path + 'tmp/') conn.sudo('sed -i "s|DEF_ENDPOINT_NAME|{0}|g" /tmp/local_endpoint.json'.format(default_endpoint_name)) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
