This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091 by this push:
new 82eb50e [DATALAB-2091]: fixed variable assignment from remote command
output
82eb50e is described below
commit 82eb50e610202e281717e84c8a78b12c508fdb59
Author: leonidfrolov <[email protected]>
AuthorDate: Mon Feb 22 15:18:28 2021 +0200
[DATALAB-2091]: fixed variable assignment from remote command output
---
.../scripts/deploy_repository/deploy_repository.py | 34 ++++++++--------
.../src/general/lib/aws/actions_lib.py | 28 ++++++-------
.../src/general/lib/azure/actions_lib.py | 22 +++++------
.../src/general/lib/gcp/actions_lib.py | 22 +++++------
.../src/general/lib/os/fab.py | 46 +++++++++++-----------
.../src/general/lib/os/redhat/common_lib.py | 2 +-
.../src/general/lib/os/redhat/edge_lib.py | 4 +-
.../src/general/lib/os/redhat/notebook_lib.py | 19 ++++-----
.../src/general/lib/os/redhat/ssn_lib.py | 4 +-
.../src/general/scripts/os/check_inactivity.py | 2 +-
.../general/scripts/os/common_configure_spark.py | 6 +--
.../scripts/os/jupyterlab_container_start.py | 2 +-
.../src/jupyter/scripts/configure_jupyter_node.py | 2 +-
.../src/ssn/scripts/configure_gitlab.py | 2 +-
.../src/ssn/scripts/configure_ssn_node.py | 4 +-
.../zeppelin/scripts/configure_zeppelin_node.py | 8 ++--
16 files changed, 104 insertions(+), 103 deletions(-)
diff --git
a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index 629fdb2..97cb864 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -835,7 +835,7 @@ def nexus_service_waiter():
while not nexus_started and checks_count < 200:
print('Waiting nexus to be started...')
time.sleep(5)
- result = conn.sudo('nmap -p 8443 localhost | grep closed >
/dev/null ; echo $?')
+ result = conn.sudo('nmap -p 8443 localhost | grep closed >
/dev/null ; echo $?').stdout
result = result[:1]
if result == '1':
nexus_started = True
@@ -896,9 +896,9 @@ def install_nexus():
args.region))
conn.sudo('wget
http://repo.{}.amazonaws.com/2017.09/updates/mirror.list -O
/tmp/updates_mirror.list'.format(
args.region))
- amazon_main_repo = conn.sudo("cat /tmp/main_mirror.list | grep {}
| sed 's/$basearch//g'".format(args.region))
+ amazon_main_repo = conn.sudo("cat /tmp/main_mirror.list | grep {}
| sed 's/$basearch//g'".format(args.region)).stdout
amazon_updates_repo = conn.sudo("cat /tmp/updates_mirror.list |
grep {} | sed 's/$basearch//g'".format(
- args.region))
+ args.region)).stdout
conn.sudo('sed -i "s|AMAZON_MAIN_URL|{}|g"
/tmp/configureNexus.groovy'.format(amazon_main_repo))
conn.sudo('sed -i "s|AMAZON_UPDATES_URL|{}|g"
/tmp/configureNexus.groovy'.format(amazon_updates_repo))
conn.sudo('rm -f /tmp/main_mirror.list')
@@ -920,15 +920,15 @@ def install_nexus():
with conn.cd('nexus-repository-apt'):
conn.sudo('mvn')
apt_plugin_version = conn.sudo('find nexus-repository-apt/ -name
"nexus-repository-apt-*.jar" '
- '-printf "%f\\n" | grep -v
"sources"').replace('nexus-repository-apt-',
+ '-printf "%f\\n" | grep -v
"sources"').stdout.replace('nexus-repository-apt-',
'').replace('.jar', '')
compress_plugin_version = conn.sudo('find /opt/nexus/ -name
"commons-compress-*.jar" '
- '-printf "%f\\n"
').replace('commons-compress-', '').replace('.jar', '')
+ '-printf "%f\\n"
').stdout.replace('commons-compress-', '').replace('.jar', '')
xz_plugin_version = conn.sudo('find /opt/nexus/ -name "xz-*.jar" '
- '-printf "%f\\n" ').replace('xz-',
'').replace('.jar', '')
+ '-printf "%f\\n" ').stdout.replace('xz-',
'').replace('.jar', '')
conn.sudo('mkdir -p
/opt/nexus/system/net/staticsnow/nexus-repository-apt/{0}/'.format(apt_plugin_version))
apt_plugin_jar_path = conn.sudo('find nexus-repository-apt/ -name
"nexus-repository-apt-{0}.jar"'.format(
- apt_plugin_version))
+ apt_plugin_version)).stdout
conn.sudo('cp -f {0}
/opt/nexus/system/net/staticsnow/nexus-repository-apt/{1}/'.format(
apt_plugin_jar_path, apt_plugin_version
))
@@ -964,12 +964,12 @@ def install_nexus():
with conn.cd('nexus-repository-r'):
conn.sudo('mvn clean install')
r_plugin_version = conn.sudo('find nexus-repository-r/ -name
"nexus-repository-r-*.jar" '
- '-printf "%f\\n" | grep -v
"sources"').replace('nexus-repository-r-', '').replace(
+ '-printf "%f\\n" | grep -v
"sources"').stdout.replace('nexus-repository-r-', '').replace(
'.jar', '')
conn.sudo('mkdir -p
/opt/nexus/system/org/sonatype/nexus/plugins/nexus-repository-r/{}/'.format(
r_plugin_version))
r_plugin_jar_path = conn.sudo('find nexus-repository-r/ -name
"nexus-repository-r-{0}.jar"'.format(
- r_plugin_version))
+ r_plugin_version)).stdout
conn.sudo('cp -f {0}
/opt/nexus/system/org/sonatype/nexus/plugins/nexus-repository-r/{1}/'.format(
r_plugin_jar_path, r_plugin_version
))
@@ -1034,7 +1034,7 @@ def install_nexus():
def install_nginx():
try:
if not
exists(conn,'/home/{}/.ensure_dir/nginx_ensured'.format(configuration['conf_os_user'])):
- hostname = conn.sudo('hostname')
+ hostname = conn.sudo('hostname').stdout
conn.sudo('apt-get install -y nginx')
conn.sudo('rm -f /etc/nginx/conf.d/*
/etc/nginx/sites-enabled/default')
conn.put('templates/nexus.conf', '/tmp/nexus.conf')
@@ -1058,11 +1058,11 @@ def mounting_disks():
try:
if not
exists(conn,'/home/{}/.ensure_dir/additional_disk_mounted'.format(configuration['conf_os_user'])):
conn.sudo('mkdir -p /opt/sonatype-work')
- disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1 | tr '\\n' ',' | sed 's|.$||g'")
+ disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1 | tr '\\n' ',' | sed 's|.$||g'").stdout
conn.sudo('bash -c \'echo -e "o\nn\np\n1\n\n\nw" | fdisk /dev/{}\'
'.format(disk_name))
conn.sudo('sleep 10')
partition_name = conn.sudo("lsblk -r | grep part | grep {} | awk
{} | sort | tail -n 1 | "
- "tr '\\n' ',' | sed
's|.$||g'".format(disk_name, "'{print $1}'"))
+ "tr '\\n' ',' | sed
's|.$||g'".format(disk_name, "'{print $1}'")).stdout
conn.sudo('mkfs.ext4 -F -q /dev/{}'.format(partition_name))
conn.sudo('mount /dev/{0}
/opt/sonatype-work'.format(partition_name))
conn.sudo('bash -c "echo \'/dev/{} /opt/sonatype-work ext4
errors=remount-ro 0 1\' >> /etc/fstab"'.format(
@@ -1106,11 +1106,11 @@ def mount_efs():
def configure_ssl():
try:
if not
exists(conn,'/home/{}/.ensure_dir/ssl_ensured'.format(configuration['conf_os_user'])):
- hostname = conn.sudo('hostname')
- private_ip = conn.sudo('curl
http://169.254.169.254/latest/meta-data/local-ipv4')
+ hostname = conn.sudo('hostname').stdout
+ private_ip = conn.sudo('curl
http://169.254.169.254/latest/meta-data/local-ipv4').stdout
subject_alt_name = 'subjectAltName = IP:{}'.format(private_ip)
if args.network_type == 'public':
- public_ip = conn.sudo('curl
http://169.254.169.254/latest/meta-data/public-ipv4')
+ public_ip = conn.sudo('curl
http://169.254.169.254/latest/meta-data/public-ipv4').stdout
subject_alt_name += ',IP:{}'.format(public_ip)
conn.sudo('cp /etc/ssl/openssl.cnf /tmp/openssl.cnf')
conn.sudo('echo "[ subject_alt_name ]" >> /tmp/openssl.cnf')
@@ -1133,9 +1133,9 @@ def set_hostname():
hostname = '{0}.{1}'.format(args.subdomain,
args.hosted_zone_name)
else:
if args.network_type == 'public':
- hostname = conn.sudo('curl
http://169.254.169.254/latest/meta-data/public-hostname')
+ hostname = conn.sudo('curl
http://169.254.169.254/latest/meta-data/public-hostname').stdout
else:
- hostname = conn.sudo('curl
http://169.254.169.254/latest/meta-data/hostname')
+ hostname = conn.sudo('curl
http://169.254.169.254/latest/meta-data/hostname').stdout
conn.sudo('hostnamectl set-hostname {0}'.format(hostname))
conn.sudo('touch
/home/{}/.ensure_dir/hostname_set'.format(configuration['conf_os_user']))
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 00debfb..5c1dccf 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -591,7 +591,7 @@ def create_iam_role(role_name, role_profile, region,
service='ec2', tag=None, us
if service == 'ec2':
try:
conn.create_instance_profile(InstanceProfileName=role_profile)
- waiter = conn.get_waiter('instance_profile_exists')
+ waiter = conn.get_waiter('instance_profile_exists').stdout
waiter.wait(InstanceProfileName=role_profile)
except botocore.exceptions.ClientError as e_profile:
if e_profile.response['Error']['Code'] == 'EntityAlreadyExists':
@@ -1266,9 +1266,9 @@ def remove_kernels(emr_name, tag_name, nb_tag_value,
ssh_user, key_path, emr_ver
try:
livy_port = conn.sudo("cat /opt/" + emr_version +
"/" + emr_name +
"/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | "
- "awk '{printf $3}'")
+ "awk '{printf $3}'").stdout
process_number = conn.sudo("netstat -natp
2>/dev/null | grep ':" + livy_port +
- "' | awk '{print $7}' | sed
's|/.*||g'")
+ "' | awk '{print $7}' | sed
's|/.*||g'").stdout
conn.sudo('kill -9 ' + process_number)
conn.sudo('systemctl disable livy-server-' +
livy_port)
except:
@@ -1297,7 +1297,7 @@ def remove_kernels(emr_name, tag_name, nb_tag_value,
ssh_user, key_path, emr_ver
zeppelin_restarted = False
while not zeppelin_restarted:
conn.sudo('sleep 5')
- result = conn.sudo('nmap -p 8080 localhost | grep
"closed" > /dev/null; echo $?')
+ result = conn.sudo('nmap -p 8080 localhost | grep
"closed" > /dev/null; echo $?').stdout
result = result[:1]
if result == '1':
zeppelin_restarted = True
@@ -1643,10 +1643,10 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
spark_jars_paths = None
if exists('/opt/spark/conf/spark-defaults.conf'):
try:
- spark_jars_paths = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf | grep -e "^spark.jars " ')
+ spark_jars_paths = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf | grep -e "^spark.jars " ').stdout
except:
spark_jars_paths = None
- region = conn.sudo('curl
http://169.254.169.254/latest/meta-data/placement/availability-zone')[:-1]
+ region = conn.sudo('curl
http://169.254.169.254/latest/meta-data/placement/availability-zone').stdout[:-1]
if region == 'us-east-1':
endpoint_url = 'https://s3.amazonaws.com'
elif region == 'cn-north-1':
@@ -1660,7 +1660,7 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
'/tmp/notebook_spark-defaults_local.conf')
if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template
/opt/spark/conf/spark-env.sh')
- java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >>
/opt/spark/conf/spark-env.sh".format(java_home))
if os.environ['application'] == 'zeppelin':
conn.sudo('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr
\'\\n\' \',\')\" >> '
@@ -1672,10 +1672,10 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
conn.sudo('echo "spark.{0}.memory {1}m" >>
/opt/spark/conf/spark-defaults.conf'.format(memory_type,
spark_memory))
if 'spark_configurations' in os.environ:
- datalab_header = conn.sudo('cat
/tmp/notebook_spark-defaults_local.conf | grep "^#"')
+ datalab_header = conn.sudo('cat
/tmp/notebook_spark-defaults_local.conf | grep "^#"').stdout
spark_configurations =
ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
- spark_defaults = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf')
+ spark_defaults = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf').stdout
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
if param.split(' ')[0] != '#':
@@ -1878,9 +1878,9 @@ def remove_dataengine_kernels(tag_name, notebook_name,
os_user, key_path, cluste
if os.environ['notebook_multiple_clusters'] == 'true':
try:
livy_port = conn.sudo("cat /opt/" + cluster_name +
- "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'")
+ "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'").stdout
process_number = conn.sudo("netstat -natp 2>/dev/null |
grep ':" + livy_port +
- "' | awk '{print $7}' | sed
's|/.*||g'")
+ "' | awk '{print $7}' | sed
's|/.*||g'").stdout
conn.sudo('kill -9 ' + process_number)
conn.sudo('systemctl disable livy-server-' + livy_port)
except:
@@ -1909,7 +1909,7 @@ def remove_dataengine_kernels(tag_name, notebook_name,
os_user, key_path, cluste
zeppelin_restarted = False
while not zeppelin_restarted:
conn.sudo('sleep 5')
- result = conn.sudo('nmap -p 8080 localhost | grep "closed" >
/dev/null; echo $?')
+ result = conn.sudo('nmap -p 8080 localhost | grep "closed" >
/dev/null; echo $?').stdout
result = result[:1]
if result == '1':
zeppelin_restarted = True
@@ -1930,7 +1930,7 @@ def remove_dataengine_kernels(tag_name, notebook_name,
os_user, key_path, cluste
def prepare_disk(os_user):
if not exists(conn,'/home/' + os_user + '/.ensure_dir/disk_ensured'):
try:
- disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1")
+ disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1").stdout
conn.sudo('''bash -c 'echo -e "o\nn\np\n1\n\n\nw" | fdisk /dev/{}'
'''.format(disk_name))
conn.sudo('mkfs.ext4 -F /dev/{}1'.format(disk_name))
conn.sudo('mount /dev/{}1 /opt/'.format(disk_name))
@@ -1974,7 +1974,7 @@ def find_des_jars(all_jars, des_path):
all_jars.remove(j)
additional_jars = ['hadoop-aws', 'aws-java-sdk-s3', 'hadoop-lzo',
'aws-java-sdk-core']
aws_filter = '\|'.join(additional_jars)
- aws_jars = conn.sudo('find {0} -name *.jar | grep
"{1}"'.format(des_path, aws_filter)).split('\r\n')
+ aws_jars = conn.sudo('find {0} -name *.jar | grep
"{1}"'.format(des_path, aws_filter)).stdout.split('\r\n')
all_jars.extend(aws_jars)
return all_jars
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index eb80145..6af3f41 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -979,9 +979,9 @@ class AzureActions:
if os.environ['notebook_multiple_clusters'] == 'true':
try:
livy_port = conn.sudo("cat /opt/" + cluster_name +
- "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'")
+ "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'").stdout
process_number = conn.sudo("netstat -natp 2>/dev/null
| grep ':" + livy_port +
- "' | awk '{print $7}' | sed
's|/.*||g'")
+ "' | awk '{print $7}' | sed
's|/.*||g'").stdout
conn.sudo('kill -9 ' + process_number)
conn.sudo('systemctl disable livy-server-' + livy_port)
except:
@@ -1010,7 +1010,7 @@ class AzureActions:
zeppelin_restarted = False
while not zeppelin_restarted:
conn.sudo('sleep 5')
- result = conn.sudo('nmap -p 8080 localhost | grep "closed"
> /dev/null; echo $?')
+ result = conn.sudo('nmap -p 8080 localhost | grep "closed"
> /dev/null; echo $?').stdout
result = result[:1]
if result == '1':
zeppelin_restarted = True
@@ -1064,7 +1064,7 @@ class AzureActions:
def ensure_local_jars(os_user, jars_dir):
if not
exists(conn,'/home/{}/.ensure_dir/local_jars_ensured'.format(os_user)):
try:
- hadoop_version = conn.sudo("ls /opt/spark/jars/hadoop-common* |
sed -n 's/.*\([0-9]\.[0-9]\.[0-9]\).*/\\1/p'")
+ hadoop_version = conn.sudo("ls /opt/spark/jars/hadoop-common* |
sed -n 's/.*\([0-9]\.[0-9]\.[0-9]\).*/\\1/p'").stdout
print("Downloading local jars for Azure")
conn.sudo('mkdir -p {}'.format(jars_dir))
if os.environ['azure_datalake_enable'] == 'false':
@@ -1100,7 +1100,7 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
spark_jars_paths = None
if exists('/opt/spark/conf/spark-defaults.conf'):
try:
- spark_jars_paths = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf | grep -e "^spark.jars " ')
+ spark_jars_paths = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf | grep -e "^spark.jars " ').stdout
except:
spark_jars_paths = None
user_storage_account_tag =
"{}-{}-{}-bucket".format(os.environ['conf_service_base_name'],
@@ -1147,13 +1147,13 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
spark_memory))
if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template
/opt/spark/conf/spark-env.sh')
- java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >>
/opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
- datalab_header = conn.sudo('cat
/tmp/notebook_spark-defaults_local.conf | grep "^#"')
+ datalab_header = conn.sudo('cat
/tmp/notebook_spark-defaults_local.conf | grep "^#"').stdout
spark_configurations =
ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
- spark_defaults = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf')
+ spark_defaults = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf').stdout
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
if param.split(' ')[0] != '#':
@@ -1246,7 +1246,7 @@ def prepare_disk(os_user):
allow = False
counter = 0
remount_azure_disk()
- disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1")
+ disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1").stdout
with settings(warn_only=True):
conn.sudo('umount -l /dev/{}1'.format(disk_name))
while not allow:
@@ -1256,7 +1256,7 @@ def prepare_disk(os_user):
else:
conn.sudo('''bash -c 'echo -e "o\nn\np\n1\n\n\nw" | fdisk
/dev/{}' 2>&1 | tee /tmp/tee.tmp '''.format(
disk_name), warn_only=True)
- out = conn.sudo('cat /tmp/tee.tmp')
+ out = conn.sudo('cat /tmp/tee.tmp').stdout
if 'Syncing disks' in out:
allow = True
elif 'The kernel still uses the old table.' in out:
@@ -1308,7 +1308,7 @@ def ensure_local_spark(os_user, spark_link,
spark_version, hadoop_version, local
conn.sudo("""echo 'export
HADOOP_CLASSPATH="$HADOOP_HOME/share/hadoop/tools/lib/*"' >>
/opt/hadoop/etc/hadoop/hadoop-env.sh""")
conn.sudo('echo "export HADOOP_HOME=/opt/hadoop/" >>
/opt/spark/conf/spark-env.sh')
conn.sudo('echo "export SPARK_HOME=/opt/spark/" >>
/opt/spark/conf/spark-env.sh')
- spark_dist_classpath = conn.sudo('/opt/hadoop/bin/hadoop
classpath')
+ spark_dist_classpath = conn.sudo('/opt/hadoop/bin/hadoop
classpath').stdout
conn.sudo('echo "export SPARK_DIST_CLASSPATH={}" >>
/opt/spark/conf/spark-env.sh'.format(
spark_dist_classpath))
conn.sudo('touch
/home/{}/.ensure_dir/local_spark_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 2f5b025..65baf7e 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1151,9 +1151,9 @@ class GCPActions:
if os.environ['notebook_multiple_clusters'] == 'true':
try:
livy_port = conn.sudo("cat /opt/" + dataproc_version +
"/" + dataproc_name
- + "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'")
+ + "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'").stdout
process_number = conn.sudo("netstat -natp 2>/dev/null
| grep ':" + livy_port +
- "' | awk '{print $7}' | sed
's|/.*||g'")
+ "' | awk '{print $7}' | sed
's|/.*||g'").stdout
conn.sudo('kill -9 ' + process_number)
conn.sudo('systemctl disable livy-server-' + livy_port)
except:
@@ -1179,7 +1179,7 @@ class GCPActions:
zeppelin_restarted = False
while not zeppelin_restarted:
conn.sudo('sleep 5')
- result = conn.sudo('nmap -p 8080 localhost | grep "closed"
> /dev/null; echo $?')
+ result = conn.sudo('nmap -p 8080 localhost | grep "closed"
> /dev/null; echo $?').stdout
result = result[:1]
if result == '1':
zeppelin_restarted = True
@@ -1371,7 +1371,7 @@ def installing_python(region, bucket, user_name,
cluster_name, application='', p
def prepare_disk(os_user):
if not exists(conn,'/home/' + os_user + '/.ensure_dir/disk_ensured'):
try:
- disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1")
+ disk_name = conn.sudo("lsblk | grep disk | awk '{print $1}' | sort
| tail -n 1").stdout
conn.sudo('''bash -c 'echo -e "o\nn\np\n1\n\n\nw" | fdisk /dev/{}'
'''.format(disk_name))
conn.sudo('mkfs.ext4 -F /dev/{}1'.format(disk_name))
conn.sudo('mount /dev/{}1 /opt/'.format(disk_name))
@@ -1400,7 +1400,7 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
spark_jars_paths = None
if exists('/opt/spark/conf/spark-defaults.conf'):
try:
- spark_jars_paths = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf | grep -e "^spark.jars " ')
+ spark_jars_paths = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf | grep -e "^spark.jars " ').stdout
except:
spark_jars_paths = None
conn.put(templates_dir + 'notebook_spark-defaults_local.conf',
'/tmp/notebook_spark-defaults_local.conf')
@@ -1414,13 +1414,13 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
spark_memory))
if not exists(conn,'/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template
/opt/spark/conf/spark-env.sh')
- java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >>
/opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
- datalab_header = conn.sudo('cat
/tmp/notebook_spark-defaults_local.conf | grep "^#"')
+ datalab_header = conn.sudo('cat
/tmp/notebook_spark-defaults_local.conf | grep "^#"').stdout
spark_configurations =
ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
- spark_defaults = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf')
+ spark_defaults = conn.sudo('cat
/opt/spark/conf/spark-defaults.conf').stdout
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
if param.split(' ')[0] != '#':
@@ -1458,9 +1458,9 @@ def remove_dataengine_kernels(notebook_name, os_user,
key_path, cluster_name):
if os.environ['notebook_multiple_clusters'] == 'true':
try:
livy_port = conn.sudo("cat /opt/" + cluster_name +
- "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'")
+ "/livy/conf/livy.conf | grep
livy.server.port | tail -n 1 | awk '{printf $3}'").stdout
process_number = conn.sudo("netstat -natp 2>/dev/null |
grep ':" + livy_port +
- "' | awk '{print $7}' | sed
's|/.*||g'")
+ "' | awk '{print $7}' | sed
's|/.*||g'").stdout
conn.sudo('kill -9 ' + process_number)
conn.sudo('systemctl disable livy-server-' + livy_port)
except:
@@ -1489,7 +1489,7 @@ def remove_dataengine_kernels(notebook_name, os_user,
key_path, cluster_name):
zeppelin_restarted = False
while not zeppelin_restarted:
conn.sudo('sleep 5')
- result = conn.sudo('nmap -p 8080 localhost | grep "closed" >
/dev/null; echo $?')
+ result = conn.sudo('nmap -p 8080 localhost | grep "closed" >
/dev/null; echo $?').stdout
result = result[:1]
if result == '1':
zeppelin_restarted = True
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py
b/infrastructure-provisioning/src/general/lib/os/fab.py
index 3eb50d5..828391c 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -76,18 +76,18 @@ def install_pip_pkg(requisites, pip_version, lib_group):
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
conn.sudo('{0} install -U {1} --no-cache-dir 2>&1 | tee
/tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > '
' /tmp/{0}install_{3}.log; then echo "" >
/tmp/{0}install_{3}.log;fi'.format(pip_version, pip_pkg, error_parser, name))
- err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version,
pip_pkg.split("==")[0])).replace('"', "'")
+ err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version,
pip_pkg.split("==")[0])).stdout.replace('"', "'")
conn.sudo('{0} freeze --all | if ! grep -w -i {1} >
/tmp/{0}install_{1}.list; then echo "" >
/tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version,
name))
+ res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version,
name)).stdout
conn.sudo('cat /tmp/tee.tmp | if ! grep "Successfully installed" >
/tmp/{0}install_{1}.list; then echo "" >
/tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- installed_out = conn.sudo('cat
/tmp/{0}install_{1}.list'.format(pip_version, name))
+ installed_out = conn.sudo('cat
/tmp/{0}install_{1}.list'.format(pip_version, name)).stdout
changed_pip_pkg = False
if res == '':
changed_pip_pkg = pip_pkg.split("==")[0].replace("_",
"-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo('{0} freeze --all | if ! grep -w -i {1} >
/tmp/{0}install_{1}.list; then echo "" > '
'/tmp/{0}install_{1}.list;fi'.format(pip_version,
changed_pip_pkg))
- res = conn.sudo('cat
/tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg))
+ res = conn.sudo('cat
/tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout
if err and name not in installed_out:
status_msg = 'installation_error'
if 'ERROR: No matching distribution found for {}'.format(name)
in err:
@@ -124,7 +124,7 @@ def install_pip_pkg(requisites, pip_version, lib_group):
else:
conn.sudo('{0} show {1} 2>&1 | if ! grep Version:
/tmp/tee.tmp > '
'/tmp/{0}_install_{1}.log; then echo "" >
/tmp/{0}_install_{1}.log;fi'.format(pip_version, i))
- dep[n] = conn.sudo('cat
/tmp/{0}_install_{1}.log'.format(pip_version, i)).replace('Version: ', '{}
v.'.format(i))
+ dep[n] = conn.sudo('cat
/tmp/{0}_install_{1}.log'.format(pip_version, i)).stdout.replace('Version: ',
'{} v.'.format(i))
dep = [i for i in dep if i]
status.append({"group": lib_group, "name": name, "version":
version, "status": status_msg,
"error_message": err, "available_versions":
versions, "add_pkgs": dep})
@@ -215,13 +215,13 @@ def configure_jupyter(os_user, jupyter_conf_file,
templates_dir, jupyter_version
"/caffe/python:/home/" + os_user +
"/pytorch/build:$PYTHONPATH ; |g' /tmp/jupyter-notebook.service")
conn.sudo("sed -i 's|CONF_PATH|{}|'
/tmp/jupyter-notebook.service".format(jupyter_conf_file))
conn.sudo("sed -i 's|OS_USR|{}|'
/tmp/jupyter-notebook.service".format(os_user))
- http_proxy = conn.run('echo $http_proxy')
- https_proxy = conn.run('echo $https_proxy')
+ http_proxy = conn.run('echo $http_proxy').stdout
+ https_proxy = conn.run('echo $https_proxy').stdout
#sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTP_PROXY={}\"\'
/tmp/jupyter-notebook.service'.format(
# http_proxy))
#sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\'
/tmp/jupyter-notebook.service'.format(
# https_proxy))
- java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o
--color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\'
/tmp/jupyter-notebook.service'.format(
java_home))
conn.sudo('\cp /tmp/jupyter-notebook.service
/etc/systemd/system/jupyter-notebook.service')
@@ -453,8 +453,8 @@ def install_r_pkg(requisites):
else:
conn.sudo('R -e \'devtools::install_version("{0}", version =
{1}, repos = "https://cloud.r-project.org";, dependencies = NA)\' 2>&1 | '
'tee /tmp/tee.tmp; if ! grep -w -E "({2})"
/tmp/tee.tmp > /tmp/install_{0}.log; then echo "" >
/tmp/install_{0}.log;fi'.format(name, vers, error_parser))
- dep = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print
$1}\'').replace('\r\n', ' ')
- dep_ver = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print
$4}\'').replace('\r\n', ' ').replace(')', '').split(' ')
+ dep = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print
$1}\'').stdout.replace('\r\n', ' ')
+ dep_ver = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print
$4}\'').stdout.replace('\r\n', ' ').replace(')', '').split(' ')
if dep == '':
dep = []
else:
@@ -465,9 +465,9 @@ def install_r_pkg(requisites):
else:
dep[n] = '{} v.{}'.format(dep[n], dep_ver[n])
dep = [i for i in dep if i]
- err = conn.sudo('cat
/tmp/install_{0}.log'.format(name)).replace('"', "'")
+ err = conn.sudo('cat
/tmp/install_{0}.log'.format(name)).stdout.replace('"', "'")
conn.sudo('R -e \'installed.packages()[,c(3:4)]\' | if ! grep -w
{0} > /tmp/install_{0}.list; then echo "" >
/tmp/install_{0}.list;fi'.format(name))
- res = conn.sudo('cat /tmp/install_{0}.list'.format(name))
+ res = conn.sudo('cat /tmp/install_{0}.list'.format(name)).stdout
if err:
status_msg = 'installation_error'
if 'couldn\'t find package \'{}\''.format(name) in err:
@@ -479,7 +479,7 @@ def install_r_pkg(requisites):
if 'Error in download_version_url(package, version, repos, type)
:' in err or 'Error in parse_deps(paste(spec,' in err:
conn.sudo('R -e \'install.packages("versions",
repos="https://cloud.r-project.org";, dep=TRUE)\'')
versions = conn.sudo('R -e \'library(versions);
available.versions("' + name + '")\' 2>&1 | grep -A 50 '
- '\'date available\' | awk \'{print
$2}\'').replace('\r\n', ' ')[5:].split(' ')
+ '\'date available\' | awk \'{print
$2}\'').stdout.replace('\r\n', ' ')[5:].split(' ')
if versions != ['']:
status_msg = 'invalid_version'
else:
@@ -499,11 +499,11 @@ def install_r_pkg(requisites):
def update_spark_jars(jars_dir='/opt/jars'):
try:
- configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name
spark-defaults.conf -type f').split('\r\n')
+ configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name
spark-defaults.conf -type f').stdout.split('\r\n')
if exists(jars_dir):
for conf in filter(None, configs):
des_path = ''
- all_jars = conn.sudo('find {0} -name
"*.jar"'.format(jars_dir)).split('\r\n')
+ all_jars = conn.sudo('find {0} -name
"*.jar"'.format(jars_dir)).stdout.split('\r\n')
if ('-des-' in conf):
des_path = '/'.join(conf.split('/')[:3])
all_jars = find_des_jars(all_jars, des_path)
@@ -528,10 +528,10 @@ def install_java_pkg(requisites):
ivy_settings = 'ivysettings.xml'
dest_dir = '/opt/jars/java'
try:
- ivy_jar = conn.sudo('find /opt /usr -name "*ivy-{0}.jar" | head -n
1'.format(os.environ['notebook_ivy_version']))
+ ivy_jar = conn.sudo('find /opt /usr -name "*ivy-{0}.jar" | head -n
1'.format(os.environ['notebook_ivy_version'])).stdout
conn.sudo('mkdir -p {0} {1}'.format(ivy_dir, dest_dir))
conn.put('{0}{1}'.format(templates_dir, ivy_settings),
'{0}/{1}'.format(ivy_dir, ivy_settings), use_sudo=True)
- proxy_string = conn.sudo('cat /etc/profile | grep http_proxy | cut -f2
-d"="')
+ proxy_string = conn.sudo('cat /etc/profile | grep http_proxy | cut -f2
-d"="').stdout
proxy_re = '(?P<proto>http.*)://(?P<host>[^:/ ]+):(?P<port>[0-9]*)'
proxy_find = re.search(proxy_re, proxy_string)
java_proxy = "export _JAVA_OPTIONS='-Dhttp.proxyHost={0}
-Dhttp.proxyPort={1} \
@@ -544,10 +544,10 @@ def install_java_pkg(requisites):
conn.sudo('{8}; java -jar {0} -settings {1}/{2} -cache {3}
-dependency {4} {5} {6} 2>&1 | tee /tmp/tee.tmp; \
if ! grep -w -E "({7})" /tmp/tee.tmp > /tmp/install_{5}.log;
then echo "" > /tmp/install_{5}.log;fi'
.format(ivy_jar, ivy_dir, ivy_settings, ivy_cache_dir, group,
artifact, version, error_parser, java_proxy))
- err = conn.sudo('cat
/tmp/install_{0}.log'.format(artifact)).replace('"', "'").strip()
+ err = conn.sudo('cat
/tmp/install_{0}.log'.format(artifact)).stdout.replace('"', "'").strip()
conn.sudo('find {0} -name "{1}*.jar" | head -n 1 | rev | cut -f1
-d "/" | rev | \
if ! grep -w -i {1} > /tmp/install_{1}.list; then echo "" >
/tmp/install_{1}.list;fi'.format(ivy_cache_dir, artifact))
- res = conn.sudo('cat /tmp/install_{0}.list'.format(artifact))
+ res = conn.sudo('cat
/tmp/install_{0}.list'.format(artifact)).stdout
if res:
conn.sudo('cp -f $(find {0} -name "*.jar" | xargs)
{1}'.format(ivy_cache_dir, dest_dir))
status.append({"group": "java", "name":
"{0}:{1}".format(group, artifact), "version": version, "status": "installed"})
@@ -601,7 +601,7 @@ def install_ungit(os_user, notebook_name, edge_ip):
manage_npm_pkg('-g install
ungit@{}'.format(os.environ['notebook_ungit_version']))
conn.put('/root/templates/ungit.service', '/tmp/ungit.service')
conn.sudo("sed -i 's|OS_USR|{}|'
/tmp/ungit.service".format(os_user))
- http_proxy = conn.run('echo $http_proxy')
+ http_proxy = conn.run('echo $http_proxy').stdout
conn.sudo("sed -i 's|PROXY_HOST|{}|g'
/tmp/ungit.service".format(http_proxy))
conn.sudo("sed -i 's|NOTEBOOK_NAME|{}|' /tmp/ungit.service".format(
notebook_name))
@@ -636,7 +636,7 @@ def install_ungit(os_user, notebook_name, edge_ip):
try:
conn.sudo("sed -i 's|--rootPath=/.*-ungit|--rootPath=/{}-ungit|'
/etc/systemd/system/ungit.service".format(
notebook_name))
- http_proxy = conn.run('echo $http_proxy')
+ http_proxy = conn.run('echo $http_proxy').stdout
conn.sudo("sed -i 's|HTTPS_PROXY=.*3128|HTTPS_PROXY={}|g'
/etc/systemd/system/ungit.service".format(http_proxy))
conn.sudo("sed -i 's|HTTP_PROXY=.*3128|HTTP_PROXY={}|g'
/etc/systemd/system/ungit.service".format(http_proxy))
conn.sudo('systemctl daemon-reload')
@@ -805,10 +805,10 @@ def restart_zeppelin(creds=False, os_user='',
hostname='', keyfile=''):
def get_spark_memory(creds=False, os_user='', hostname='', keyfile=''):
if creds:
with settings(host_string='{}@{}'.format(os_user, hostname)):
- mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d
":"')
+ mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d
":"').stdout
instance_memory = int(mem)
else:
- mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d ":"')
+ mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d
":"').stdout
instance_memory = int(mem)
try:
if instance_memory >
int(os.environ['dataengine_expl_instance_memory']):
diff --git
a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index a9fd236..f303c0c 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -96,7 +96,7 @@ def change_pkg_repos():
def find_java_path_remote():
- java_path = conn.sudo("alternatives --display java | grep 'slave jre: ' |
awk '{print $3}'")
+ java_path = conn.sudo("alternatives --display java | grep 'slave jre: ' |
awk '{print $3}'").stdout
return java_path
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index 9f84133..cee47fa 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -74,7 +74,7 @@ def install_nginx_lua(edge_ip, nginx_version,
keycloak_auth_server_url, keycloak
conn.sudo('mkdir -p /home/{0}/keys'.format(user))
conn.sudo('''bash -c 'echo "{0}" | base64 --decode >
/etc/ssl/certs/root_ca.crt' '''.format(
os.environ['conf_stepcerts_root_ca']))
- fingerprint = conn.sudo('step certificate fingerprint
/etc/ssl/certs/root_ca.crt')
+ fingerprint = conn.sudo('step certificate fingerprint
/etc/ssl/certs/root_ca.crt').stdout
conn.sudo('step ca bootstrap --fingerprint {0} --ca-url
"{1}"'.format(fingerprint,
os.environ['conf_stepcerts_ca_url']))
conn.sudo('echo "{0}" >
/home/{1}/keys/provisioner_password'.format(
@@ -84,7 +84,7 @@ def install_nginx_lua(edge_ip, nginx_version,
keycloak_auth_server_url, keycloak
conn.sudo('step ca token {3} --kid {0} --ca-url "{1}" --root
/etc/ssl/certs/root_ca.crt '
'--password-file /home/{2}/keys/provisioner_password {4}
--output-file /tmp/step_token'.format(
os.environ['conf_stepcerts_kid'],
os.environ['conf_stepcerts_ca_url'], user, cn, sans))
- token = conn.sudo('cat /tmp/step_token')
+ token = conn.sudo('cat /tmp/step_token').stdout
conn.sudo('step ca certificate "{0}"
/etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
'--token "{1}" --kty=RSA --size 2048 --provisioner {2}
'.format(cn, token,
os.environ['conf_stepcerts_kid']))
diff --git
a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index 4ffcd36..71dda89 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -67,7 +67,7 @@ def ensure_r_local_kernel(spark_version, os_user,
templates_dir, kernels_dir):
except:
pass
conn.sudo('cd /usr/local/spark/R/lib/SparkR; R -e
"devtools::install(\'.\')"')
- r_version = conn.sudo("R --version | awk '/version / {print $3}'")
+ r_version = conn.sudo("R --version | awk '/version / {print
$3}'").stdout
conn.put(templates_dir + 'r_template.json', '/tmp/r_template.json')
conn.sudo('sed -i "s|R_VER|' + r_version + '|g"
/tmp/r_template.json')
conn.sudo('sed -i "s|SP_VER|' + spark_version + '|g"
/tmp/r_template.json')
@@ -120,8 +120,8 @@ def install_rstudio(os_user, local_spark_path,
rstudio_pass, rstudio_version):
conn.sudo('touch /home/{}/.Rprofile'.format(os_user))
conn.sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user))
conn.sudo('''echo 'library(SparkR, lib.loc =
c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >>
/home/{}/.Rprofile'''.format(os_user))
- http_proxy = conn.run('echo $http_proxy')
- https_proxy = conn.run('echo $https_proxy')
+ http_proxy = conn.run('echo $http_proxy').stdout
+ https_proxy = conn.run('echo $https_proxy').stdout
conn.sudo('''echo 'Sys.setenv(http_proxy = \"{}\")' >>
/home/{}/.Rprofile'''.format(http_proxy, os_user))
conn.sudo('''echo 'Sys.setenv(https_proxy = \"{}\")' >>
/home/{}/.Rprofile'''.format(https_proxy, os_user))
conn.sudo('rstudio-server start')
@@ -328,11 +328,11 @@ def install_os_pkg(requisites):
os_pkg = name
manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | tee
/tmp/tee.tmp; if ! grep -w -E "({1})" '
'/tmp/tee.tmp >
/tmp/os_install_{2}.log; then echo "" >
/tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
- install_output = conn.sudo('cat /tmp/tee.tmp')
- err = conn.sudo('cat
/tmp/os_install_{}.log'.format(name)).replace('"', "'")
+ install_output = conn.sudo('cat /tmp/tee.tmp').stdout
+ err = conn.sudo('cat
/tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'")
conn.sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})"
/tmp/tee.tmp > '
'/tmp/os_install_{0}.log; then echo "" >
/tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
- dep = conn.sudo('cat /tmp/os_install_{}.log'.format(name))
+ dep = conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout
if dep == '':
dep = []
else:
@@ -345,14 +345,14 @@ def install_os_pkg(requisites):
versions = []
res = conn.sudo(
'python3 -c "import os,sys,yum; yb = yum.YumBase(); pl =
yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name ==
\'{0}\']"'.format(
- name)).split('\r\n')[1]
+ name)).stdout.split('\r\n')[1]
if err:
status_msg = 'installation_error'
elif res != []:
version = res.split("'")[1].split("-")[0]
status_msg = "installed"
if 'No package {} available'.format(os_pkg) in install_output:
- versions = conn.sudo('yum --showduplicates list ' + name + ' |
expand | grep ' + name + ' | awk \'{print $2}\'').replace('\r\n', '')
+ versions = conn.sudo('yum --showduplicates list ' + name + ' |
expand | grep ' + name + ' | awk \'{print $2}\'').stdout.replace('\r\n', '')
if versions and versions != 'Error: No matching Packages to
list':
versions = versions.split(' ')
status_msg = 'invalid_version'
@@ -386,7 +386,8 @@ def get_available_os_pkgs():
try:
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
downgrade_python_version()
- yum_raw = conn.sudo('python3 -c "import os,sys,yum; yb =
yum.YumBase(); pl = yb.doPackageLists(); print {pkg.name:pkg.vr for pkg in
pl.available}"')
+ yum_raw = conn.sudo('python3 -c "import os,sys,yum; yb =
yum.YumBase(); pl = yb.doPackageLists(); '
+ 'print {pkg.name:pkg.vr for pkg in
pl.available}"').stdout
yum_re = re.sub\
(r'\w*\s\w*\D\s\w*.\w*.\s\w*.\w*.\w.\w*.\w*.\w*', '', yum_raw)
yum_list = yum_re.replace("'", "\"")
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index bebeb93..03b5d2d 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -210,7 +210,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir,
web_path,
keycloak_client_secret, keycloak_auth_server_url, report_path=''):
try:
if not
exists(conn,'{}tmp/ss_started'.format(os.environ['ssn_datalab_path'])):
- java_path = conn.sudo("alternatives --display java | grep 'slave
jre: ' | awk '{print $3}'")
+ java_path = conn.sudo("alternatives --display java | grep 'slave
jre: ' | awk '{print $3}'").stdout
supervisor_conf = '/etc/supervisord.d/supervisor_svc.ini'
conn.local('sed -i "s|MONGO_PASSWORD|{}|g"
/root/templates/ssn.yml'.format(mongo_passwd))
conn.local('sed -i "s|KEYSTORE_PASSWORD|{}|g"
/root/templates/ssn.yml'.format(keystore_passwd))
@@ -248,7 +248,7 @@ def start_ss(keyfile, host_string, datalab_conf_dir,
web_path,
conn.sudo('mkdir -p /var/log/application')
conn.run('mkdir -p /tmp/yml_tmp/')
for service in ['self-service', 'provisioning-service',
'billing']:
- jar = conn.sudo('cd {0}{1}/lib/; find {1}*.jar -type
f'.format(web_path, service))
+ jar = conn.sudo('cd {0}{1}/lib/; find {1}*.jar -type
f'.format(web_path, service)).stdout
conn.sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar
'.format(web_path, jar, service))
conn.sudo('cp {0}/webapp/{1}/conf/*.yml
/tmp/yml_tmp/'.format(datalab_path, service))
# Replacing Keycloak and cloud parameters
diff --git
a/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
b/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
index ebef80d..5cfa436 100644
--- a/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
+++ b/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
@@ -49,7 +49,7 @@ if __name__ == "__main__":
inactivity_file = 'local_inactivity'
if exists('{}{}'.format(inactivity_dir, inactivity_file)):
- timestamp = conn.sudo('cat {}{}'.format(inactivity_dir,
inactivity_file))
+ timestamp = conn.sudo('cat {}{}'.format(inactivity_dir,
inactivity_file)).stdout
else:
timestamp = '0000000000'
diff --git
a/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
b/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
index 80f1a65..d9c9951 100644
---
a/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
+++
b/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
@@ -42,7 +42,7 @@ args = parser.parse_args()
def update_spark_defaults_conf(spark_conf):
try:
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S %Z", time.gmtime())
- configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name
spark-defaults.conf -type f').split('\r\n')
+ configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name
spark-defaults.conf -type f').stdout.split('\r\n')
for conf in filter(None, configs):
conn.sudo('''sed -i '/^# Updated/d' {0}'''.format(conf))
conn.sudo('''echo "# Updated by DATALAB at {0} >>
{1}'''.format(timestamp, conf))
@@ -54,10 +54,10 @@ def update_spark_defaults_conf(spark_conf):
def add_custom_spark_properties(cluster_name):
try:
if os.path.exists('/opt/{0}'.format(cluster_name)):
- datalab_header = conn.sudo('cat
/tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name))
+ datalab_header = conn.sudo('cat
/tmp/{0}/notebook_spark-defaults_local.conf | grep
"^#"'.format(cluster_name)).stdout
spark_configurations =
ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
- spark_defaults = conn.sudo('cat
/opt/{0}/spark/conf/spark-defaults.conf'.format(cluster_name))
+ spark_defaults = conn.sudo('cat
/opt/{0}/spark/conf/spark-defaults.conf'.format(cluster_name)).stdout
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
if param.split(' ')[0] != '#':
diff --git
a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
index cbefa01..dd996a0 100644
---
a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++
b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
@@ -39,7 +39,7 @@ def start_jupyterlab_container(jupyterlab_dir):
try:
with conn.cd('{}'.format(jupyterlab_dir)):
conn.run('docker build --network=host --file Dockerfile_jupyterlab
-t jupyter-lab .'.format(args.os_user))
- container_id = conn.run('docker ps | awk \'NR==2{print $1}\'')
+ container_id = conn.run('docker ps | awk \'NR==2{print
$1}\'').stdout
if container_id != '':
conn.run('docker stop ' + container_id)
conn.run('docker run -d --restart unless-stopped -p 8888:8888 \
diff --git
a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index 1bea6f0..4f8cebb 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -103,7 +103,7 @@ if __name__ == "__main__":
# INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
print("Install local Spark")
ensure_local_spark(args.os_user, spark_link, spark_version,
hadoop_version, local_spark_path)
- local_spark_scala_version = conn.sudo('spark-submit --version 2>&1 | grep
-o -P "Scala version \K.{0,7}"')
+ local_spark_scala_version = conn.sudo('spark-submit --version 2>&1 | grep
-o -P "Scala version \K.{0,7}"').stdout
print("Install storage jars")
ensure_local_jars(args.os_user, jars_dir)
print("Configure local Spark")
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
b/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
index aac527b..27662c7 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
@@ -148,7 +148,7 @@ def configure_gitlab():
with settings(hide('everything')):
raw = conn.run('curl -k --request POST
"{0}://localhost/api/v4/session?login=root&password={1}"'
- .format(proto, os.environ['gitlab_root_password']))
+ .format(proto, os.environ['gitlab_root_password'])).stdout
data = json.loads(raw)
if not json.loads(os.environ['gitlab_signup_enabled']):
print('Disabling signup...')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index c504683..c99d912 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -142,7 +142,7 @@ def configure_ssl_certs(hostname, custom_ssl_cert):
conn.sudo('mkdir -p /home/{0}/keys'.format(args.os_user))
conn.sudo('''bash -c 'echo "{0}" | base64 --decode >
/etc/ssl/certs/root_ca.crt' '''.format(
os.environ['conf_stepcerts_root_ca']))
- fingerprint = conn.sudo('step certificate fingerprint
/etc/ssl/certs/root_ca.crt')
+ fingerprint = conn.sudo('step certificate fingerprint
/etc/ssl/certs/root_ca.crt').stdout
conn.sudo('step ca bootstrap --fingerprint {0} --ca-url
"{1}"'.format(fingerprint,
os.environ['conf_stepcerts_ca_url']))
conn.sudo('echo "{0}" >
/home/{1}/keys/provisioner_password'.format(
@@ -153,7 +153,7 @@ def configure_ssl_certs(hostname, custom_ssl_cert):
'--password-file /home/{2}/keys/provisioner_password {4}
--output-file /tmp/step_token'.format(
os.environ['conf_stepcerts_kid'],
os.environ['conf_stepcerts_ca_url'],
args.os_user, cn, sans))
- token = conn.sudo('cat /tmp/step_token')
+ token = conn.sudo('cat /tmp/step_token').stdout
conn.sudo('step ca certificate "{0}"
/etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
'--token "{1}" --kty=RSA --size 2048 --provisioner {2}
'.format(cn, token,
os.environ['conf_stepcerts_kid']))
diff --git
a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
index 6673a25..046adcf 100644
---
a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
+++
b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
@@ -83,7 +83,7 @@ def configure_zeppelin(os_user):
conn.sudo('tar -zxvf /tmp/zeppelin-' + zeppelin_version +
'-bin-netinst.tgz -C /opt/')
conn.sudo('ln -s /opt/zeppelin-' + zeppelin_version +
'-bin-netinst /opt/zeppelin')
conn.sudo('cp /opt/zeppelin/conf/zeppelin-env.sh.template
/opt/zeppelin/conf/zeppelin-env.sh')
- java_home = conn.run("update-alternatives --query java | grep -o
\'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o
\'/.*/java-8.*/jre\'").stdout.splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >>
/opt/zeppelin/conf/zeppelin-env.sh".format(java_home))
conn.sudo('cp /opt/zeppelin/conf/zeppelin-site.xml.template
/opt/zeppelin/conf/zeppelin-site.xml')
conn.sudo('sed -i \"/# export ZEPPELIN_PID_DIR/c\export
ZEPPELIN_PID_DIR=/var/run/zeppelin\" /opt/zeppelin/conf/zeppelin-env.sh')
@@ -107,8 +107,8 @@ def configure_zeppelin(os_user):
try:
conn.put(templates_dir + 'zeppelin-notebook.service',
'/tmp/zeppelin-notebook.service')
conn.sudo("sed -i 's|OS_USR|" + os_user + "|'
/tmp/zeppelin-notebook.service")
- http_proxy = conn.run('echo $http_proxy')
- https_proxy = conn.run('echo $https_proxy')
+ http_proxy = conn.run('echo $http_proxy').stdout
+ https_proxy = conn.run('echo $https_proxy').stdout
conn.sudo('sed -i \'/\[Service\]/
a\Environment=\"HTTP_PROXY={}\"\' /tmp/zeppelin-notebook.service'.format(
http_proxy))
conn.sudo('sed -i \'/\[Service\]/
a\Environment=\"HTTPS_PROXY={}\"\' /tmp/zeppelin-notebook.service'.format(
@@ -137,7 +137,7 @@ def configure_local_livy_kernels(args):
spark_memory = get_spark_memory()
conn.sudo('sed -i "s|DRIVER_MEMORY|{}m|g"
/tmp/interpreter.json'.format(spark_memory))
while not port_number_found:
- port_free = conn.sudo('nmap -p ' + str(default_port) + ' localhost
| grep "closed" > /dev/null; echo $?')
+ port_free = conn.sudo('nmap -p ' + str(default_port) + ' localhost
| grep "closed" > /dev/null; echo $?').stdout
port_free = port_free[:1]
if port_free == '0':
livy_port = default_port
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
