This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-483
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-483 by this push:
new a34eb23 fixed issue with missed spark properties
a34eb23 is described below
commit a34eb238d87ab2f04a6ce1b4e2b96f85c8c387f5
Author: Oleh Martushevskyi <[email protected]>
AuthorDate: Tue Mar 19 17:40:15 2019 +0200
fixed issue with missed spark properties
---
.../src/general/lib/azure/actions_lib.py | 10 +++++++++-
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 10 +++++++++-
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index c9195ab..11302e3 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1152,7 +1152,15 @@ def configure_local_spark(jars_dir, templates_dir,
memory_type='driver'):
def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir,
datalake_enabled, spark_configs=''):
local("jar_list=`find {0} -name '*.jar' | tr '\\n' ','` ; echo
\"spark.jars $jar_list\" >> \
/tmp/{1}/notebook_spark-defaults_local.conf".format(jars_dir,
cluster_name))
- local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name, cluster_dir))
+ if os.path.exists('{0}spark/conf/spark-defaults.conf'.format(cluster_dir)):
+ additional_spark_properties = local('diff --changed-group-format="%>"
--unchanged-group-format="" '
+
'/tmp/{0}/notebook_spark-defaults_local.conf '
+ '{1}spark/conf/spark-defaults.conf
| grep -v "^#"'.format(
+ cluster_name, cluster_dir),
capture=True)
+ for property in additional_spark_properties.split('\n'):
+ local('echo "{0}" >>
/tmp/{1}/notebook_spark-defaults_local.conf'.format(property, cluster_name))
+ local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name,
+
cluster_dir))
if datalake_enabled == 'false':
local('cp -f /opt/spark/conf/core-site.xml
{}spark/conf/'.format(cluster_dir))
else:
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index ec77515..52c33f0 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1340,7 +1340,15 @@ def install_dataengine_spark(cluster_name, spark_link,
spark_version, hadoop_ver
def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir,
datalake_enabled, spark_configs=''):
local("jar_list=`find {0} -name '*.jar' | tr '\\n' ','` ; echo
\"spark.jars $jar_list\" >> \
/tmp/{1}/notebook_spark-defaults_local.conf".format(jars_dir,
cluster_name))
- local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name, cluster_dir))
+ if os.path.exists('{0}spark/conf/spark-defaults.conf'.format(cluster_dir)):
+ additional_spark_properties = local('diff --changed-group-format="%>"
--unchanged-group-format="" '
+
'/tmp/{0}/notebook_spark-defaults_local.conf '
+ '{1}spark/conf/spark-defaults.conf
| grep -v "^#"'.format(
+ cluster_name, cluster_dir),
capture=True)
+ for property in additional_spark_properties.split('\n'):
+ local('echo "{0}" >>
/tmp/{1}/notebook_spark-defaults_local.conf'.format(property, cluster_name))
+ local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf
{1}spark/conf/spark-defaults.conf'.format(cluster_name,
+
cluster_dir))
local('cp -f /opt/spark/conf/core-site.xml
{}spark/conf/'.format(cluster_dir))
if spark_configs:
spark_configurations = ast.literal_eval(spark_configs)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
