Repository: bigtop Updated Branches: refs/heads/master 5aab80ca6 -> 6616117e3
BIGTOP-1746: Introduce roles in bigtop puppet deployment Signed-off-by: Evans Ye <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6616117e Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6616117e Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6616117e Branch: refs/heads/master Commit: 6616117e303971419a5059b54e2fed5838a567d6 Parents: 5aab80c Author: Gajendran <[email protected]> Authored: Sun Aug 30 13:33:21 2015 -0700 Committer: Evans Ye <[email protected]> Committed: Sat Sep 12 16:36:03 2015 +0000 ---------------------------------------------------------------------- .../puppet/hieradata/bigtop/cluster.yaml | 28 ++ bigtop-deploy/puppet/hieradata/site.yaml | 2 + bigtop-deploy/puppet/manifests/cluster.pp | 336 ++++++++++--------- bigtop-deploy/puppet/manifests/site.pp | 28 +- .../puppet/modules/bigtop-util/Gemfile | 8 + .../puppet/modules/bigtop-util/Gemfile.lock | 51 +++ .../puppet/modules/bigtop-util/README.md | 23 ++ .../puppet/modules/bigtop-util/Rakefile | 2 + .../lib/puppet/parser/functions/get_roles.rb | 61 ++++ .../spec/functions/get_roles_spec.rb | 60 ++++ .../puppet/modules/crunch/manifests/init.pp | 7 + .../puppet/modules/giraph/manifests/init.pp | 7 + .../modules/hadoop-flume/manifests/init.pp | 7 + .../modules/hadoop-hbase/manifests/init.pp | 18 + .../modules/hadoop-hive/manifests/init.pp | 12 +- .../modules/hadoop-hive/templates/hive-site.xml | 4 +- .../modules/hadoop-oozie/manifests/init.pp | 16 + .../puppet/modules/hadoop-pig/manifests/init.pp | 7 + .../modules/hadoop-zookeeper/manifests/init.pp | 11 + .../puppet/modules/hadoop/manifests/init.pp | 64 +++- .../modules/hadoop/templates/hadoop-env.sh | 4 +- .../modules/hadoop/templates/mapred-site.xml | 15 - .../puppet/modules/hcatalog/manifests/init.pp | 11 + .../puppet/modules/hue/manifests/init.pp | 12 + .../modules/ignite-hadoop/manifests/init.pp | 6 + .../puppet/modules/kerberos/manifests/init.pp | 9 + .../puppet/modules/mahout/manifests/init.pp | 7 + .../puppet/modules/solr/manifests/init.pp | 7 + .../puppet/modules/spark/manifests/init.pp | 11 + .../puppet/modules/sqoop2/manifests/init.pp | 11 + .../puppet/modules/tachyon/manifests/init.pp | 11 + .../puppet/modules/tez/manifests/init.pp | 8 + .../puppet/modules/ycsb/manifests/init.pp | 7 + 33 files changed, 662 insertions(+), 209 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 85583d1..41d477b 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -24,6 +24,27 @@ # - hadoop # - yarn +# A list of roles can be assigned to each node in a cluster via "bigtop::roles" +# hiera conf. This is useful if you want to run specific daemons/processes of a component +# (lets say hadoop) on specific nodes. To enable roles, you need to explicitly set +# "bigtop::roles_enabled" conf to true. By default, it is not enabled. If not enabled, +# bigtop will honour the hadoop_cluster_node::cluster_components parameter and assume +# respective roles to each node based on hadoop_head_node, hadoop_gateway_node +# and other nodes (worker nodes) classification. For full list of roles +# supported by each component, refer to $roles_map datastructure in cluster.pp file +# for example: + +#bigtop::roles +# - namenode +# - resourcemanager + +# By specifying above roles, you instruct a node to run only namenode and resourcemanager. +# When roles are disabled, all worker daemons for each component assume hadoop_head_node as +# their master. With roles enabled, this assumption may no longer be valid and its upto the user +# to specify the FQDN for each master daemons to all nodes so that worker daemons can communicate +# with master. FQDN and port for master daemons of each component can be specified using below +# configurations. + # Storage directories (will be created if doesn't exist) #hadoop::hadoop_storage_dirs: # - /data/1 @@ -141,3 +162,10 @@ hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_conf # tez hadoop::common::tez_conf_dir: "/etc/tez/conf" hadoop::common::tez_jars: "/usr/lib/tez" + +# to enable tez in hadoop, uncomment the lines below +# hadoop::common::use_tez: true +# hadoop::common_mapred_app::mapreduce_framework_name: "yarn-tez" + +# to enable tez in hive, uncomment the lines below +# hadoop-hive::client::hive_execution_engine: "tez" http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index f8c1b7b..70c0529 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -35,5 +35,7 @@ hadoop::hadoop_storage_dirs: # - zookeeper # - ycsb +#bigtop::roles_enabled: false + #bigtop::jdk_package_name: "openjdk-7-jre-headless" #bigtop::bigtop_repo_uri: "http://mirror.example.com/path/to/mirror/"; http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 19cfc65..15a247b 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -13,37 +13,103 @@ # See the License for the specific language governing permissions and # limitations under the License. +$roles_map = { + hdfs-non-ha => { + master => ["namenode"], + worker => ["datanode"], + standby => ["secondarynamenode"], + }, + hdfs-ha => { + master => ["namenode"], + worker => ["datanode"], + standby => ["standby-namenode"], + }, + yarn => { + master => ["resourcemanager"], + worker => ["nodemanager"], + client => ["hadoop-client"], + # mapred is the default app which runs on yarn. + library => ["mapred-app"], + }, + mapred => { + library => ["mapred-app"], + }, + hbase => { + master => ["hbase-master"], + worker => ["hbase-server"], + client => ["hbase-client"], + }, + ignite-hadoop => { + worker => ["ignite-server"], + }, + solrcloud => { + worker => ["solr-server"], + }, + spark => { + master => ["spark-master"], + worker => ["spark-worker"], + }, + tachyon => { + master => ["tachyon-master"], + worker => ["tachyon-worker"], + }, + flume => { + worker => ["flume-agent"], + }, + kerberos => { + master => ["kerberos-server"], + }, + oozie => { + master => ["oozie-server"], + client => ["oozie-client"], + }, + hcat => { + master => ["hcatalog-server"], + gateway_server => ["webhcat-server"], + }, + sqoop => { + gateway_server => ["sqoop-server"], + client => ["sqoop-client"], + }, + httpfs => { + gateway_server => ["httpfs-server"], + }, + hue => { + gateway_server => ["hue-server"], + }, + mahout => { + client => ["mahout-client"], + }, + giraph => { + client => ["giraph-client"], + }, + crunch => { + client => ["crunch-client"], + }, + pig => { + client => ["pig-client"], + }, + hive => { + client => ["hive-client"], + }, + tez => { + client => ["tez-client"], + }, + zookeeper => { + worker => ["zookeeper-server"], + client => ["zookeeper-client"], + }, + ycsb => { + client => ["ycsb-client"], + }, +} + class hadoop_cluster_node ( $hadoop_security_authentication = hiera("hadoop::hadoop_security_authentication", "simple"), - - # Lookup component array or comma separated components (i.e. - # hadoop,spark,hbase ) as a default via facter. - $cluster_components = "$::components" + $bigtop_real_users = [ 'jenkins', 'testuser', 'hudson' ], + $cluster_components = ["all"] ) { - # Ensure (even if a single value) that the type is an array. - if is_array($cluster_components) { - $components = $cluster_components - } else { - $components = any2array($cluster_components, ",") - } - - $all = ($components[0] == undef) - - if ($hadoop_security_authentication == "kerberos") { - include kerberos::client - } - - # Flume agent is the only component that goes on EVERY node in the cluster - if ($all or "flume" in $components) { - include hadoop-flume::agent - } -} - - -class hadoop_worker_node ( - $bigtop_real_users = [ 'jenkins', 'testuser', 'hudson' ] - ) inherits hadoop_cluster_node { user { $bigtop_real_users: ensure => present, system => false, @@ -53,168 +119,106 @@ class hadoop_worker_node ( if ($hadoop_security_authentication == "kerberos") { kerberos::host_keytab { $bigtop_real_users: } User<||> -> Kerberos::Host_keytab<||> + include kerberos::client } - include hadoop::datanode - if ($all or "yarn" in $components) { - include hadoop::nodemanager - } - if ($all or "hbase" in $components) { - include hadoop-hbase::server - } - - if ($all or "ignite-hadoop" in $components) { - ignite-hadoop::server { "ignite-hadoop-node": } - } - - ### If mapred is not installed, yarn can fail. - ### So, when we install yarn, we also need mapred for now. - ### This dependency should be cleaned up eventually. - if ($all or "mapred-app" or "yarn" in $components) { - include hadoop::mapred-app - } - - if ($all or "solrcloud" in $components) { - include solr::server - } + $hadoop_head_node = hiera("bigtop::hadoop_head_node") + $standby_head_node = hiera("bigtop::standby_head_node", "") + $hadoop_gateway_node = hiera("bigtop::hadoop_gateway_node", $hadoop_head_node) - if ($all or "spark" in $components) { - include spark::worker + $ha_enabled = $standby_head_node ? { + "" => false, + default => true, } - if ($all or "tachyon" in $components) { - include tachyon::worker + # look into alternate hiera datasources configured using this path in + # hiera.yaml + $hadoop_hiera_ha_path = $ha_enabled ? { + false => "noha", + true => "ha", } - } -class hadoop_head_node inherits hadoop_worker_node { - exec { "init hdfs": - path => ['/bin','/sbin','/usr/bin','/usr/sbin'], - command => 'bash -x /usr/lib/hadoop/libexec/init-hdfs.sh', - require => Package['hadoop-hdfs'], - timeout => 0 - } - Class['Hadoop::Namenode'] -> Class['Hadoop::Datanode'] -> Exec<| title == "init hdfs" |> - -if ($hadoop_security_authentication == "kerberos") { - include kerberos::server - include kerberos::kdc - include kerberos::kdc::admin_server - } - - include hadoop::namenode - - if ($hadoop::common_hdfs::ha == "disabled") { - include hadoop::secondarynamenode - } - - if ($all or "yarn" in $components) { - include hadoop::resourcemanager - include hadoop::historyserver - include hadoop::proxyserver - Exec<| title == "init hdfs" |> -> Class['Hadoop::Resourcemanager'] -> Class['Hadoop::Nodemanager'] - Exec<| title == "init hdfs" |> -> Class['Hadoop::Historyserver'] - } - - if ($all or "hbase" in $components) { - include hadoop-hbase::master - Exec<| title == "init hdfs" |> -> Class['Hadoop-hbase::Master'] - } - - if ($all or "oozie" in $components) { - include hadoop-oozie::server - if ($all or "mapred-app" in $components) { - Class['Hadoop::Mapred-app'] -> Class['Hadoop-oozie::Server'] +class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_node { + define deploy_module($roles) { + class { "${name}::deploy": + roles => $roles, } - Exec<| title == "init hdfs" |> -> Class['Hadoop-oozie::Server'] } - if ($all or "hcat" in $components) { - include hcatalog::server - include hcatalog::webhcat::server + $modules = [ + "crunch", + "giraph", + "hadoop", + "hadoop-hbase", + "ignite-hadoop", + "hadoop-flume", + "hadoop-hive", + "hadoop-oozie", + "hadoop-pig", + "sqoop2", + "hadoop-zookeeper", + "hcatalog", + "hue", + "mahout", + "solr", + "spark", + "tachyon", + "tez", + "ycsb", + "kerberos" + ] + + deploy_module { $modules: + roles => $roles, } +} - if ($all or "spark" in $components) { - include spark::master - } +class node_with_components inherits hadoop_cluster_node { - if ($all or "tachyon" in $components) { - include tachyon::master + # Ensure (even if a single value) that the type is an array. + if (is_array($cluster_components)) { + $components_array = $cluster_components + } else { + if ($cluster_components == undef) { + $components_array = ["all"] + } else { + $components_array = [$cluster_components] + } } - if ($all or "hbase" in $components) { - include hadoop-zookeeper::server + $given_components = $components_array[0] ? { + "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]), + default => $components_array, } - - if ($all or "tez" in $components) { - include tez::client - Class['tez::client'] -> Exec<| title == "init hdfs" |> + $ha_dependent_components = $ha_enabled ? { + true => ["hdfs-ha"], + default => ["hdfs-non-ha"], } + $components = concat($given_components, $ha_dependent_components) - # class hadoop::rsync_hdfs isn't used anywhere - #Exec<| title == "init hdfs" |> -> Class['Hadoop::Rsync_hdfs'] -} - -class standby_head_node inherits hadoop_cluster_node { - include hadoop::namenode -} - -class hadoop_gateway_node inherits hadoop_cluster_node { - if ($all or "sqoop2" in $components) { - include sqoop2::server - } + $master_role_types = ["master", "worker", "library"] + $standby_role_types = ["standby", "library"] + $worker_role_types = ["worker", "library"] + $gateway_role_types = ["client", "gateway_server"] - if ($all or "httpfs" in $components) { - include hadoop::httpfs - if ($all or "hue" in $components) { - Class['Hadoop::Httpfs'] -> Class['Hue::Server'] - } - } - - if ($all or "hue" in $components) { - include hue::server - if ($all or "hbase" in $components) { - Class['Hadoop-hbase::Client'] -> Class['Hue::Server'] + if ($::fqdn == $hadoop_head_node or $::fqdn == $hadoop_gateway_node) { + if ($hadoop_gateway_node == $hadoop_head_node) { + $role_types = concat($master_role_types, $gateway_role_types) + } elsif ($::fqdn == $hadoop_head_node) { + $role_types = $master_role_types + } else { + $role_types = $gateway_role_types } + } elsif ($::fqdn == $standby_head_node) { + $role_types = $standby_role_types + } else { + $role_types = $worker_role_types } - include hadoop::client + $roles = get_roles($components, $role_types, $roles_map) - if ($all or "mahout" in $components) { - include mahout::client - } - if ($all or "giraph" in $components) { - include giraph::client - } - if ($all or "crunch" in $components) { - include crunch::client - } - if ($all or "pig" in $components) { - include hadoop-pig::client - } - if ($all or "hive" in $components) { - include hadoop-hive::client - } - if ($all or "sqoop2" in $components) { - include sqoop2::client - } - if ($all or "sqoop" in $components) { - include sqoop::client - } - if ($all or "oozie" in $components) { - include hadoop-oozie::client - } - if ($all or "hbase" in $components) { - include hadoop-hbase::client - } - if ($all or "zookeeper" in $components) { - include hadoop-zookeeper::client - } - if ($all or "tez" in $components) { - include tez::client - } - if ($all or "ycsb" in $components) { - include ycsb::client + class { 'node_with_roles': + roles => $roles, } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/manifests/site.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/site.pp b/bigtop-deploy/puppet/manifests/site.pp index 8e8276c..ae71627 100644 --- a/bigtop-deploy/puppet/manifests/site.pp +++ b/bigtop-deploy/puppet/manifests/site.pp @@ -56,31 +56,17 @@ package { $jdk_package_name: import "cluster.pp" node default { - $hadoop_head_node = hiera("bigtop::hadoop_head_node") - $standby_head_node = hiera("bigtop::standby_head_node", "") - $hadoop_gateway_node = hiera("bigtop::hadoop_gateway_node", $hadoop_head_node) - # look into alternate hiera datasources configured using this path in - # hiera.yaml - $hadoop_hiera_ha_path = $standby_head_node ? { - "" => "noha", - default => "ha", - } + $roles_enabled = hiera("bigtop::roles_enabled", false) - case $::fqdn { - $hadoop_head_node: { - include hadoop_head_node - } - $standby_head_node: { - include standby_head_node - } - default: { - include hadoop_worker_node - } + if (!is_bool($roles_enabled)) { + fail("bigtop::roles hiera conf is not of type boolean. It should be set to either true or false") } - if ($hadoop_gateway_node == $::fqdn) { - include hadoop_gateway_node + if ($roles_enabled) { + include node_with_roles + } else { + include node_with_components } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/Gemfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/bigtop-util/Gemfile b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile new file mode 100644 index 0000000..4abd416 --- /dev/null +++ b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile @@ -0,0 +1,8 @@ +source 'https://rubygems.org' + +group :test do + gem 'rake' + gem 'puppet', ENV['PUPPET_VERSION'] || '~> 3.7.0' + gem 'rspec-puppet', git: 'https://github.com/rodjek/rspec-puppet.git' + gem 'puppetlabs_spec_helper' +end http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock new file mode 100644 index 0000000..a24c0b3 --- /dev/null +++ b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock @@ -0,0 +1,51 @@ +GIT + remote: https://github.com/rodjek/rspec-puppet.git + revision: 8459e14807977244c00bdbcf190062c529b63474 + specs: + rspec-puppet (2.0.0) + rspec (~> 2.0) + +GEM + remote: https://rubygems.org/ + specs: + CFPropertyList (2.2.8) + diff-lcs (1.2.5) + facter (2.4.1) + CFPropertyList (~> 2.2.6) + hiera (1.3.4) + json_pure + json_pure (1.8.2) + metaclass (0.0.4) + mocha (1.1.0) + metaclass (~> 0.0.1) + puppet (3.7.4) + facter (> 1.6, < 3) + hiera (~> 1.0) + json_pure + puppet-lint (1.1.0) + puppet-syntax (2.0.0) + rake + puppetlabs_spec_helper (0.9.1) + mocha + puppet-lint + puppet-syntax + rake + rspec-puppet + rake (10.4.2) + rspec (2.99.0) + rspec-core (~> 2.99.0) + rspec-expectations (~> 2.99.0) + rspec-mocks (~> 2.99.0) + rspec-core (2.99.2) + rspec-expectations (2.99.2) + diff-lcs (>= 1.1.3, < 2.0) + rspec-mocks (2.99.3) + +PLATFORMS + ruby + +DEPENDENCIES + puppet (~> 3.7.0) + puppetlabs_spec_helper + rake + rspec-puppet! http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/bigtop-util/README.md b/bigtop-deploy/puppet/modules/bigtop-util/README.md new file mode 100644 index 0000000..287504c --- /dev/null +++ b/bigtop-deploy/puppet/modules/bigtop-util/README.md @@ -0,0 +1,23 @@ +# Bigtop Util Puppet Module + +This module provides common utility functions used by other Bigtop puppet modules. + +## Testing + +Test dependencies are managed by Bundler: + +```bash +gem install bundler +``` + +To install the required gems for testing, `cd` into `bigtop-util` and use: + +```bash +bundle install --path vendor/bundle +``` + +Now unit tests can be run using: + +```bash +bundle exec rake spec +``` http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/Rakefile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/bigtop-util/Rakefile b/bigtop-deploy/puppet/modules/bigtop-util/Rakefile new file mode 100644 index 0000000..14f1c24 --- /dev/null +++ b/bigtop-deploy/puppet/modules/bigtop-util/Rakefile @@ -0,0 +1,2 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/rake_tasks' http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb b/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb new file mode 100644 index 0000000..3f78d31 --- /dev/null +++ b/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Function to return an array of roles given the roles mapping, array of components and role types to lookup. +# Example map: +# $map = { +# component1 => { +# role_type1 => ["xxx"], +# role_type2 => ["yyy"], +# role_type3 => ["zzz"], +# } +# } +# calling get_roles(["component1"], ["role_type1", "role_type2"], $map) will return ["xxx", "yyy"] + +Puppet::Parser::Functions.newfunction(:get_roles, :type => :rvalue) do |arguments| + if arguments.size != 3 + fail Puppet::ParseError, "get_roles() method: Incorrect number of arguments. arguments given #{arguments.size} for 3" + end + + components = arguments[0] + role_types = arguments[1] + roles_map = arguments[2] + + unless components.is_a? Array + fail Puppet::ParseError, "get_roles(): Requires first argument to be array" + end + + unless role_types.is_a? Array + fail Puppet::ParseError, "get_roles(): Requires second argument to be array" + end + + unless roles_map.is_a? Hash + fail Puppet::ParseError, "get_roles(): Requires third argument to be hash" + end + + roles = Array.new + components.each do |component| + role_types.each do |role_type| + if roles_map.key?(component) + component_map = roles_map[component] + if component_map.key?(role_type) + temp_roles = component_map[role_type] + roles.concat(temp_roles) + end + end + end + end + roles +end \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb b/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb new file mode 100644 index 0000000..5a680e5 --- /dev/null +++ b/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'rubygems' +require 'puppetlabs_spec_helper/module_spec_helper' + +describe 'get_roles' do + let(:scope) { PuppetlabsSpec::PuppetInternals.scope } + subject { scope.function_get_roles([components, role_types, roles_map]) } + + context 'simple test' do + let(:components) { ["c1", "c2"] } + + let(:role_types) { ["rt1", "rt3"] } + + let(:roles_map) do + { + "c1" => { + "rt1" => ["r1"], + "rt2" => ["r2"] + }, + "c2" => { + "rt3" => ["r3"], + "rt4" => ["r4"] + }, + } + end + + it { is_expected.to match_array(["r1", "r3"]) } + end + + context 'missing roles_type/component test' do + let(:components) { ["c1", "c2"] } + + let(:role_types) { ["rt1", "rt3"] } + + let(:roles_map) do + { + "c1" => { + "rt1" => ["r10", "r11"], + "rt2" => ["r2"] + }, + } + end + + it { is_expected.to match_array(["r10", "r11"]) } + end +end http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/crunch/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/crunch/manifests/init.pp b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp index b31edf6..fa190eb 100644 --- a/bigtop-deploy/puppet/modules/crunch/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class crunch { + + class deploy ($roles) { + if ("crunch-client" in $roles) { + include crunch::client + } + } + class client { package { ["crunch", "crunch-doc"]: ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/giraph/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/giraph/manifests/init.pp b/bigtop-deploy/puppet/modules/giraph/manifests/init.pp index 1dc0d9b..a52a133 100644 --- a/bigtop-deploy/puppet/modules/giraph/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/giraph/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class giraph { + + class deploy ($roles) { + if ("giraph-client" in $roles) { + include giraph::client + } + } + class client($zookeeper_quorum = 'localhost') { package { "giraph": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp index daf352a..e6012cf 100644 --- a/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class hadoop-flume { + + class deploy ($roles) { + if ("flume-agent" in $roles) { + include hadoop-flume::agent + } + } + class agent($sources = [], $sinks = [], $channels = []) { package { "flume-agent": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp index 454e978..f61a23b 100644 --- a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp @@ -14,6 +14,24 @@ # limitations under the License. class hadoop-hbase { + + class deploy ($roles) { + if ("hbase-server" in $roles) { + include hadoop-hbase::server + } + + if ("hbase-master" in $roles) { + include hadoop::init_hdfs + include hadoop-hbase::master + include hadoop-zookeeper::server + Class['Hadoop::Init_hdfs'] -> Class['Hadoop-hbase::Master'] + } + + if ("hbase-client" in $roles) { + include hadoop-hbase::client + } + } + class client-package { package { "hbase": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp index f9dede4..b0eee0a 100644 --- a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp @@ -14,7 +14,17 @@ # limitations under the License. class hadoop-hive { - class client($hbase_master = "", $hbase_zookeeper_quorum = "") { + + class deploy ($roles) { + if ("hive-client" in $roles) { + include hadoop-hive::client + } + } + + class client($hbase_master = "", + $hbase_zookeeper_quorum = "", + $hive_execution_engine = "mr") { + package { "hive": ensure => latest, } http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml index 4e33907..2aca237 100644 --- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml @@ -42,12 +42,10 @@ </property> <% end %> -<% if @all or @components.include? "tez" -%> <property> <name>hive.execution.engine</name> - <value>tez</value> + <value><%= @hive_execution_engine %></value> </property> -<% end %> <property> <name>javax.jdo.option.ConnectionURL</name> http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp index f1177e9..0829360 100644 --- a/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp @@ -14,6 +14,22 @@ # limitations under the License. class hadoop-oozie { + + class deploy ($roles) { + if ("oozie-client" in $roles) { + include hadoop-oozie::client + } + + if ("oozie-server" in $roles) { + include hadoop::init_hdfs + include hadoop-oozie::server + Class['Hadoop::Init_hdfs'] -> Class['Hadoop-oozie::Server'] + if ("mapred-app" in $roles) { + Class['Hadoop::Mapred-app'] -> Class['Hadoop-oozie::Server'] + } + } + } + class client { package { "oozie-client": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp index 37bfde0..3ffaa70 100644 --- a/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class hadoop-pig { + + class deploy ($roles) { + if ("pig-client" in $roles) { + include hadoop-pig::client + } + } + class client { package { "pig": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp index d8bccfe..a44834c 100644 --- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp @@ -14,6 +14,17 @@ # limitations under the License. class hadoop-zookeeper { + + class deploy ($roles) { + if ("zookeeper-client" in $roles) { + include hadoop-zookeeper::client + } + + if ("zookeeper-server" in $roles) { + include hadoop-zookeeper::server + } + } + class client { package { "zookeeper": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp index 57c739c..777c26e 100644 --- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp @@ -27,6 +27,67 @@ class hadoop ($hadoop_security_authentication = "simple", include stdlib + class deploy ($roles) { + if ("datanode" in $roles) { + include hadoop::datanode + } + + if ("namenode" in $roles) { + include hadoop::init_hdfs + include hadoop::namenode + + if ("datanode" in $roles) { + Class['Hadoop::Namenode'] -> Class['Hadoop::Datanode'] -> Class['Hadoop::Init_hdfs'] + } else { + Class['Hadoop::Namenode'] -> Class['Hadoop::Init_hdfs'] + } + } + + if ($hadoop::common_hdfs::ha != "disabled" and "standby-namenode" in $roles) { + include hadoop::namenode + } + + if ("mapred-app" in $roles) { + include hadoop::mapred-app + } + + if ("nodemanager" in $roles) { + include hadoop::nodemanager + } + + if ("resourcemanager" in $roles) { + include hadoop::resourcemanager + include hadoop::historyserver + include hadoop::proxyserver + + Class['Hadoop::Init_hdfs'] -> Class['Hadoop::Resourcemanager'] + if ("nodemanager" in $roles) { + Class['Hadoop::Resourcemanager'] -> Class['Hadoop::Nodemanager'] + } + Class['Hadoop::Init_hdfs'] -> Class['Hadoop::Historyserver'] + } + + if ($hadoop::common_hdfs::ha == "disabled" and "secondarynamenode" in $roles) { + include hadoop::secondarynamenode + } + + if ("httpfs-server" in $roles) { + include hadoop::httpfs + } + + if ("hadoop-client" in $roles) { + include hadoop::client + } + } + + class init_hdfs { + exec { "init hdfs": + path => ['/bin','/sbin','/usr/bin','/usr/sbin'], + command => 'bash -x /usr/lib/hadoop/libexec/init-hdfs.sh', + require => Package['hadoop-hdfs'] + } + } + class common ($hadoop_java_home = undef, $hadoop_classpath = undef, $hadoop_heapsize = undef, @@ -46,6 +107,7 @@ class hadoop ($hadoop_security_authentication = "simple", $hadoop_pid_dir = undef, $hadoop_ident_string = undef, $hadoop_niceness = undef, + $use_tez = false, $tez_conf_dir = undef, $tez_jars = undef, ) inherits hadoop { @@ -298,7 +360,7 @@ class hadoop ($hadoop_security_authentication = "simple", $mapreduce_jobhistory_host = undef, $mapreduce_jobhistory_port = "10020", $mapreduce_jobhistory_webapp_port = "19888", - $mapreduce_framework_name = undef, + $mapreduce_framework_name = "yarn", $mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred"), $mapreduce_cluster_temp_dir = "/mapred/system", $yarn_app_mapreduce_am_staging_dir = "/user", http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh index f959f5b..a1397c2 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh +++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh @@ -80,14 +80,14 @@ # The scheduling priority for daemon processes. See 'man nice'. <%= shell_config("HADOOP_NICENESS") %> +<% if (@use_tez) -%> # tez environment, needed to enable tez -<% if (@all or @components.include? "tez") -%> <%= shell_config("TEZ_CONF_DIR") %> <%= shell_config("TEZ_JARS") %> # Add tez into HADOOP_CLASSPATH export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/* -<% end -%> +<% end -%> ### WARNING: the following is NOT really optional. It is a shame that stock Hadoop ### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off ### and have a working Hadoop cluster. http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml index 2723e82..ea49926 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml @@ -92,21 +92,6 @@ <value><%= @mapreduce_framework_name %></value> </property> -<!-- If not specified, we automatically choose the best framework for users --> -<% else -%> -<% if @all or @components.include? "tez" -%> - <property> - <name>mapreduce.framework.name</name> - <value>yarn-tez</value> - </property> - -<% else -%> - <property> - <name>mapreduce.framework.name</name> - <value>yarn</value> - </property> - -<% end -%> <% end -%> <property> <name>mapreduce.cluster.local.dir</name> http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp index 6585dd3..d78956d 100644 --- a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp @@ -14,6 +14,17 @@ # limitations under the License. class hcatalog { + + class deploy ($roles) { + if ("hcatalog-server" in $roles) { + include hcatalog::server + } + + if ("webhcat-server" in $roles) { + include hcatalog::webhcat::server + } + } + class server($port = "9083", $kerberos_realm = "") { package { "hcatalog-server": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hue/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp index 687a055..72d08c0 100644 --- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp @@ -14,6 +14,18 @@ # limitations under the License. class hue { + class deploy ($roles) { + if ("hue-server" in $roles) { + include hue::server + if ("httpfs-server" in $roles) { + Class['Hadoop::Httpfs'] -> Class['Hue::Server'] + } + if ("hbase-client" in $roles) { + Class['Hadoop-hbase::Client'] -> Class['Hue::Server'] + } + } + } + class server($sqoop2_url = "http://localhost:12000/sqoop";, $solr_url = "http://localhost:8983/solr/";, $hbase_thrift_url = "", $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_proxy_url, $history_server_url, $hive_host = "", $hive_port = "10000", http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp index 6a967c6..de658a5 100644 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp @@ -14,6 +14,12 @@ # limitations under the License. class ignite-hadoop { + class deploy ($roles) { + if ("ignite-server" in $roles) { + ignite-hadoop::server { "ignite-hadoop-node": } + } + } + define server() { package { "ignite-hadoop": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp b/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp index 4a726a0..946f425 100644 --- a/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp @@ -14,6 +14,15 @@ # limitations under the License. class kerberos { + + class deploy ($roles) { + if ("kerberos-server" in $roles) { + include kerberos::server + include kerberos::kdc + include kerberos::kdc::admin_server + } + } + class site ($domain = inline_template('<%= domain %>'), $realm = inline_template('<%= domain.upcase %>'), $kdc_server = 'localhost', http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/mahout/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/mahout/manifests/init.pp b/bigtop-deploy/puppet/modules/mahout/manifests/init.pp index 0d9bd8c..0c55a9b 100644 --- a/bigtop-deploy/puppet/modules/mahout/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/mahout/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class mahout { + + class deploy ($roles) { + if ("mahout-client" in $roles) { + include mahout::client + } + } + class client { package { "mahout": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/solr/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/solr/manifests/init.pp b/bigtop-deploy/puppet/modules/solr/manifests/init.pp index 119fbd1..c90e0e7 100644 --- a/bigtop-deploy/puppet/modules/solr/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/solr/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class solr { + + class deploy ($roles) { + if ("solr-server" in $roles) { + include solr::server + } + } + class server($port = "1978", $port_admin = "1979", $zk = "localhost:2181", $root_url = "hdfs://localhost:8020/solr", $kerberos_realm = "") { package { "solr-server": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/spark/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp index 2f87ead..9b33bb9 100644 --- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp @@ -14,6 +14,17 @@ # limitations under the License. class spark { + + class deploy ($roles) { + if ("spark-master" in $roles) { + include spark::master + } + + if ("spark-worker" in $roles) { + include spark::worker + } + } + class common ($master_host = $fqdn, $master_port = "7077", $master_ui_port = "18080") { package { "spark-core": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp b/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp index 9648483..60b6939 100644 --- a/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp @@ -14,6 +14,17 @@ # limitations under the License. class sqoop2 { + + class deploy ($roles) { + if ("sqoop-client" in $roles) { + include sqoop2::client + } + + if ("sqoop-server" in $roles) { + include sqoop2::server + } + } + class client { package { "sqoop2-client": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp index 8d53b25..ef7e5df 100644 --- a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp @@ -10,6 +10,17 @@ # See the License for the specific language governing permissions and # limitations under the License. class tachyon { + + class deploy ($roles) { + if ("tachyon-master" in $roles) { + include tachyon::master + } + + if ("tachyon-worker" in $roles) { + include tachyon::worker + } + } + class common ($master_host){ package { "tachyon-tfs": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/tez/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tez/manifests/init.pp b/bigtop-deploy/puppet/modules/tez/manifests/init.pp index 6db2220..4e307aa 100644 --- a/bigtop-deploy/puppet/modules/tez/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/tez/manifests/init.pp @@ -14,6 +14,14 @@ # limitations under the License. class tez { + class deploy ($roles) { + if ("tez-client" in $roles) { + include hadoop::init_hdfs + include tez::client + Class['Tez::Client'] -> Class['Hadoop::Init_hdfs'] + } + } + class client { package { "tez": ensure => latest, http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp b/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp index 8e7e7d5..ed5c86a 100644 --- a/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp @@ -14,6 +14,13 @@ # limitations under the License. class ycsb { + + class deploy ($roles) { + if ("ycsb-client" in $roles) { + include ycsb::client + } + } + class client { package { ["ycsb"]: ensure => latest,
