Repository: bigtop Updated Branches: refs/heads/master f981e3c9f -> fb081bb13
BIGTOP-2766. [Puppet] Spark worker startup failed due to default master_url is yarn Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fb081bb1 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fb081bb1 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fb081bb1 Branch: refs/heads/master Commit: fb081bb13223ccca80d9f4542f5517f40c2b8ffd Parents: f981e3c Author: Evans Ye <[email protected]> Authored: Thu May 11 03:23:19 2017 +0800 Committer: Evans Ye <[email protected]> Committed: Mon May 15 05:58:43 2017 -0700 ---------------------------------------------------------------------- bigtop-deploy/puppet/hieradata/site.yaml | 7 ++++--- bigtop-deploy/puppet/manifests/cluster.pp | 7 +++++++ bigtop-deploy/puppet/modules/spark/manifests/init.pp | 2 +- .../puppet/modules/spark/templates/spark-defaults.conf | 8 ++++++++ bigtop-deploy/puppet/modules/spark/templates/spark-env.sh | 8 ++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb081bb1/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index f18aa9c..02f8e45 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -13,11 +13,13 @@ hadoop::hadoop_storage_dirs: #hadoop_cluster_node::cluster_components: # - alluxio +# - ambari # - apex # - crunch # - flink # - flume # - giraph +# - gpdb # - ignite_hadoop # - hbase # - hcat @@ -31,14 +33,13 @@ hadoop::hadoop_storage_dirs: # - qfs # - solrcloud # - spark +# - spark-standalone # - sqoop # - sqoop2 # - tez # - yarn -# - zookeeper # - ycsb -# - gpdb -# - ambari +# - zookeeper #bigtop::roles_enabled: false http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb081bb1/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index e4e22b2..7579606 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -69,6 +69,11 @@ $roles_map = { worker => ["solr-server"], }, spark => { + worker => ["spark-on-yarn"], + client => ["spark-client"], + library => ["spark-yarn-slave"], + }, + spark-standalone => { master => ["spark-master"], worker => ["spark-worker"], }, @@ -278,4 +283,6 @@ class node_with_components inherits hadoop_cluster_node { class { 'node_with_roles': roles => $roles, } + + notice("Roles to deploy: ${roles}") } http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb081bb1/bigtop-deploy/puppet/modules/spark/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp index 4c72d20..ebb25d0 100644 --- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp @@ -135,7 +135,7 @@ class spark { } class common( - $master_url = 'yarn', + $master_url = undef, $master_host = $fqdn, $zookeeper_connection_string = undef, $master_port = 7077, http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb081bb1/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf index 5978bec..a99d9bd 100644 --- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf +++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf @@ -13,7 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +<% if @master_url -%> spark.master <%= @master_url %> +<% else -%> +<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%> +spark.master spark://<%= @master_host %>:<%= @master_port %> +<% else -%> +spark.master yarn +<% end -%> +<% end -%> spark.eventLog.enabled true spark.eventLog.dir <%= @event_log_dir %> spark.history.fs.logDirectory <%= @history_log_dir %> http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb081bb1/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh index 0aabc65..2f3e406 100755 --- a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh +++ b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh @@ -24,7 +24,15 @@ export HIVE_CONF_DIR=${HIVE_CONF_DIR:-/etc/hive/conf} export STANDALONE_SPARK_MASTER_HOST=<%= @master_host %> export SPARK_MASTER_PORT=<%= @master_port %> export SPARK_MASTER_IP=$STANDALONE_SPARK_MASTER_HOST +<% if @master_url -%> export SPARK_MASTER_URL=<%= @master_url %> +<% else -%> +<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%> +export SPARK_MASTER_URL=spark://<%= @master_host %>:<%= @master_port %> +<% else -%> +export SPARK_MASTER_URL=yarn +<% end -%> +<% end -%> export SPARK_MASTER_WEBUI_PORT=<%= @master_ui_port %> export SPARK_WORKER_DIR=${SPARK_WORKER_DIR:-/var/run/spark/work}
