BIGTOP-2821: expose extra config options for spark Closes #238
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/15242bd3 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/15242bd3 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/15242bd3 Branch: refs/heads/branch-1.2 Commit: 15242bd3618ee8ed4deef82cbefbbabeb554f0bd Parents: a312ea6 Author: Kevin W Monroe <[email protected]> Authored: Fri Jun 23 11:41:54 2017 -0500 Committer: Evans Ye <[email protected]> Committed: Sat Jul 1 12:52:59 2017 +0000 ---------------------------------------------------------------------- bigtop-deploy/puppet/modules/spark/manifests/init.pp | 3 +++ .../puppet/modules/spark/templates/spark-defaults.conf | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/15242bd3/bigtop-deploy/puppet/modules/spark/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp index ebb25d0..6fe23dc 100644 --- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp @@ -146,6 +146,9 @@ class spark { $use_yarn_shuffle_service = false, $event_log_dir = "hdfs:///var/log/spark/apps", $history_log_dir = "hdfs:///var/log/spark/apps", + $extra_lib_dirs = "/usr/lib/hadoop/lib/native", + $driver_mem = "1g", + $executor_mem = "1g", ) { ### This is an ungodly hack to deal with the consequence of adding http://git-wip-us.apache.org/repos/asf/bigtop/blob/15242bd3/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf index a99d9bd..2ef4a04 100644 --- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf +++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf @@ -30,4 +30,9 @@ spark.yarn.historyServer.address <%= @master_host %>:<%= @history_ui_port %> <% end -%> spark.history.ui.port <%= @history_ui_port %> spark.shuffle.service.enabled <%= @use_yarn_shuffle_service %> -spark.driver.extraJavaOptions -Djava.library.path=/usr/lib/hadoop/lib/native +<% if @extra_lib_dirs -%> +spark.driver.extraLibraryPath <%= @extra_lib_dirs %> +spark.executor.extraLibraryPath <%= @extra_lib_dirs %> +<% end -%> +spark.driver.memory <%= @driver_mem %> +spark.executor.memory <%= @executor_mem %>
