This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-483
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-483 by this push:
new 2e222ee fixed issue with missed spark properties
2e222ee is described below
commit 2e222eea366aa3f8dfb6f5ddbf4789bc541897e4
Author: Oleh Martushevskyi <[email protected]>
AuthorDate: Tue Mar 19 17:38:35 2019 +0200
fixed issue with missed spark properties
---
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 284c744..85d589a 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1746,6 +1746,13 @@ def configure_dataengine_spark(cluster_name, jars_dir,
cluster_dir, datalake_ena
"""" >>
/tmp/{}/notebook_spark-defaults_local.conf'""".format(cluster_name))
local('echo "spark.hadoop.fs.s3a.server-side-encryption-algorithm
AES256" >> '
'/tmp/{}/notebook_spark-defaults_local.conf'.format(cluster_name))
+ if os.path.exists('{0}spark/conf/spark-defaults.conf'.format(cluster_dir)):
+ additional_spark_properties = local('diff --changed-group-format="%>"
--unchanged-group-format="" '
+
'/tmp/{0}/notebook_spark-defaults_local.conf '
+ '{1}spark/conf/spark-defaults.conf
| grep -v "^#"'.format(
+ cluster_name, cluster_dir),
capture=True)
+ for property in additional_spark_properties.split('\n'):
+ local('echo "{0}" >>
/tmp/{1}/notebook_spark-defaults_local.conf'.format(property, cluster_name))
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name,
cluster_dir))
if spark_configs:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
