This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DLAB-1980
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-1980 by this push:
new ea956ac [DLAB-1980]: added another check whether /opt/{cluster_dir}
exists
ea956ac is described below
commit ea956acefedf6a9832fbdeb53c89a881771c3ce4
Author: leonidfrolov <[email protected]>
AuthorDate: Wed Sep 2 16:30:47 2020 +0300
[DLAB-1980]: added another check whether /opt/{cluster_dir} exists
---
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 2 +-
infrastructure-provisioning/src/general/lib/azure/actions_lib.py | 4 +---
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 2 +-
3 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index bce8c76..43b1f44 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1830,7 +1830,7 @@ def configure_dataengine_spark(cluster_name, jars_dir,
cluster_dir, datalake_ena
if os.path.exists('{0}'.format(cluster_dir)):
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name,
cluster_dir))
- if spark_configs:
+ if spark_configs and os.path.exists('{0}'.format(cluster_dir)):
dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf |
grep "^#"'.format(cluster_name),
capture=True)
spark_configurations = ast.literal_eval(spark_configs)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index ba00ab8..9c77ecf 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1170,15 +1170,13 @@ def configure_dataengine_spark(cluster_name, jars_dir,
cluster_dir, datalake_ena
for property in additional_spark_properties.split('\n'):
local('echo "{0}" >>
/tmp/{1}/notebook_spark-defaults_local.conf'.format(property, cluster_name))
if os.path.exists('{0}'.format(cluster_dir)):
- print('=========')
- local('ls -la /opt/')
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name,
cluster_dir))
if datalake_enabled == 'false':
local('cp -f /opt/spark/conf/core-site.xml
{}spark/conf/'.format(cluster_dir))
else:
local('cp -f /opt/hadoop/etc/hadoop/core-site.xml
{}hadoop/etc/hadoop/core-site.xml'.format(cluster_dir))
- if spark_configs:
+ if spark_configs and os.path.exists('{0}'.format(cluster_dir)):
dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf |
grep "^#"'.format(cluster_name),
capture=True)
spark_configurations = ast.literal_eval(spark_configs)
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 5ecfdf0..185f890 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1524,7 +1524,7 @@ def configure_dataengine_spark(cluster_name, jars_dir,
cluster_dir, datalake_ena
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name,
cluster_dir))
local('cp -f /opt/spark/conf/core-site.xml
{}spark/conf/'.format(cluster_dir))
- if spark_configs:
+ if spark_configs and os.path.exists('{0}'.format(cluster_dir)):
dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf |
grep "^#"'.format(cluster_name),
capture=True)
spark_configurations = ast.literal_eval(spark_configs)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
