Repository: bigtop Updated Branches: refs/heads/master 5ea264bee -> e14693842
BIGTOP-2821: expose extra config options for spark Closes #238 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e1469384 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e1469384 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e1469384 Branch: refs/heads/master Commit: e14693842b0ccadcbed37a08f6a1c4077ee8a406 Parents: 5ea264b Author: Kevin W Monroe <[email protected]> Authored: Fri Jun 23 11:41:54 2017 -0500 Committer: Kevin W Monroe <[email protected]> Committed: Wed Jun 28 09:45:18 2017 -0500 ---------------------------------------------------------------------- bigtop-deploy/puppet/modules/spark/manifests/init.pp | 3 +++ .../puppet/modules/spark/templates/spark-defaults.conf | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e1469384/bigtop-deploy/puppet/modules/spark/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp index ebb25d0..6fe23dc 100644 --- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp @@ -146,6 +146,9 @@ class spark { $use_yarn_shuffle_service = false, $event_log_dir = "hdfs:///var/log/spark/apps", $history_log_dir = "hdfs:///var/log/spark/apps", + $extra_lib_dirs = "/usr/lib/hadoop/lib/native", + $driver_mem = "1g", + $executor_mem = "1g", ) { ### This is an ungodly hack to deal with the consequence of adding http://git-wip-us.apache.org/repos/asf/bigtop/blob/e1469384/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf index a99d9bd..2ef4a04 100644 --- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf +++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf @@ -30,4 +30,9 @@ spark.yarn.historyServer.address <%= @master_host %>:<%= @history_ui_port %> <% end -%> spark.history.ui.port <%= @history_ui_port %> spark.shuffle.service.enabled <%= @use_yarn_shuffle_service %> -spark.driver.extraJavaOptions -Djava.library.path=/usr/lib/hadoop/lib/native +<% if @extra_lib_dirs -%> +spark.driver.extraLibraryPath <%= @extra_lib_dirs %> +spark.executor.extraLibraryPath <%= @extra_lib_dirs %> +<% end -%> +spark.driver.memory <%= @driver_mem %> +spark.executor.memory <%= @executor_mem %>
