Repository: bigtop Updated Branches: refs/heads/master 047e4ddd7 -> 31fa70d73
BIGTOP-2708. puppet code always installs hdfs by default Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/31fa70d7 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/31fa70d7 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/31fa70d7 Branch: refs/heads/master Commit: 31fa70d73af117407f00cf2ba473a76074f472ba Parents: 047e4dd Author: Roman Shaposhnik <[email protected]> Authored: Sat Mar 25 02:21:38 2017 +0000 Committer: Roman Shaposhnik <[email protected]> Committed: Fri Mar 24 19:24:26 2017 -0700 ---------------------------------------------------------------------- bigtop-deploy/puppet/manifests/cluster.pp | 31 +++++++++++++++++--- bigtop-deploy/puppet/manifests/site.pp | 5 +++- .../puppet/modules/hadoop/manifests/init.pp | 4 +-- .../pseudo-cluster/config/hieradata/site.yaml | 3 +- .../sandbox/site.yaml.template.centos-6_hadoop | 2 +- .../sandbox/site.yaml.template.debian-8_hadoop | 2 +- provisioner/docker/config_centos6.yaml | 4 +-- provisioner/docker/config_centos7.yaml | 4 +-- provisioner/docker/config_debian8.yaml | 4 +-- provisioner/docker/config_ubuntu_trusty.yaml | 25 ---------------- provisioner/docker/config_ubuntu_xenial.yaml | 25 ++++++++++++++++ provisioner/vagrant/vagrantconfig.yaml | 4 +-- 12 files changed, 70 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index d9f6e01..1313082 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -13,6 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The following is a bit of a tricky map. The idea here is that the keys +# correspond to anything that could be specified in +# hadoop_cluster_node::cluster_components: +# The values are maps from each role that a node can have in a cluster: +# client, gateway_server, library, master, worker, standby +# to a role recognized by each puppet module's deploy class. +# +# Note that the code here will pass all these roles to all the deploy +# classes defined in every single Bigtop's puppet module. This is similar +# to how a visitor pattern works in OOP. One subtle ramification of this +# approach is that you should make sure that deploy classes from different +# modules do NOT accept same strings for role types. +# +# And if that wasn't enough of a head scratcher -- you also need to keep +# in mind that there's no hdfs key in the following map, even though it +# is a perfectly legal value for hadoop_cluster_node::cluster_components: +# The reason for this is that hdfs is treated as an alias for either +# hdfs-non-ha or hdfs-ha depending on whether HA for HDFS is either enabled +# or disabled. + $roles_map = { apex => { client => ["apex-client"], @@ -217,14 +237,17 @@ class node_with_components inherits hadoop_cluster_node { } $given_components = $components_array[0] ? { - "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]), + "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]) << "hdfs", default => $components_array, } $ha_dependent_components = $ha_enabled ? { - true => ["hdfs-ha"], - default => ["hdfs-non-ha"], + true => "hdfs-ha", + default => "hdfs-non-ha", + } + $components = member($given_components, "hdfs") ? { + true => delete($given_components, "hdfs") << $ha_dependent_components, + default => $given_components } - $components = concat($given_components, $ha_dependent_components) $master_role_types = ["master", "worker", "library"] $standby_role_types = ["standby", "library"] http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/bigtop-deploy/puppet/manifests/site.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/site.pp b/bigtop-deploy/puppet/manifests/site.pp index ce5857f..f35a4e0 100644 --- a/bigtop-deploy/puppet/manifests/site.pp +++ b/bigtop-deploy/puppet/manifests/site.pp @@ -69,7 +69,10 @@ if ($provision_repo) { case $::operatingsystem { /Debian/: { require apt - require apt::backports + class { 'apt::backports': + pin => 500, + } + Class['apt::backports'] -> Package <||> package { "jdk": name => $jdk_package_name, http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp index 30a68e2..7d8e48f 100644 --- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp @@ -45,7 +45,7 @@ class hadoop ($hadoop_security_authentication = "simple", } } - if ($hadoop::common_hdfs::ha != "disabled" and "standby-namenode" in $roles) { + if ("standby-namenode" in $roles and $hadoop::common_hdfs::ha != "disabled") { include hadoop::namenode } @@ -67,7 +67,7 @@ class hadoop ($hadoop_security_authentication = "simple", } } - if ($hadoop::common_hdfs::ha == "disabled" and "secondarynamenode" in $roles) { + if ("secondarynamenode" in $roles and $hadoop::common_hdfs::ha == "disabled") { include hadoop::secondarynamenode } http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/docker/pseudo-cluster/config/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/docker/pseudo-cluster/config/hieradata/site.yaml b/docker/pseudo-cluster/config/hieradata/site.yaml index a8ab29d..1d29859 100644 --- a/docker/pseudo-cluster/config/hieradata/site.yaml +++ b/docker/pseudo-cluster/config/hieradata/site.yaml @@ -3,8 +3,9 @@ hadoop::hadoop_storage_dirs: - /data/1 - /data/2 hadoop_cluster_node::cluster_components: - - hadoop + - hdfs - yarn + - mapreduce bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64 bigtop::jdk_package_name: openjdk-8-jdk http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/docker/sandbox/site.yaml.template.centos-6_hadoop ---------------------------------------------------------------------- diff --git a/docker/sandbox/site.yaml.template.centos-6_hadoop b/docker/sandbox/site.yaml.template.centos-6_hadoop index f38c4fb..554431e 100644 --- a/docker/sandbox/site.yaml.template.centos-6_hadoop +++ b/docker/sandbox/site.yaml.template.centos-6_hadoop @@ -16,5 +16,5 @@ bigtop::hadoop_head_node: "head.node.fqdn" hadoop::hadoop_storage_dirs: [/data/1, /data/2] bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64 -hadoop_cluster_node::cluster_components: [hadoop, yarn] +hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce] bigtop::jdk_package_name: java-1.8.0-openjdk-devel.x86_64 http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/docker/sandbox/site.yaml.template.debian-8_hadoop ---------------------------------------------------------------------- diff --git a/docker/sandbox/site.yaml.template.debian-8_hadoop b/docker/sandbox/site.yaml.template.debian-8_hadoop index 9a00c43..d9c7c7f 100644 --- a/docker/sandbox/site.yaml.template.debian-8_hadoop +++ b/docker/sandbox/site.yaml.template.debian-8_hadoop @@ -16,5 +16,5 @@ bigtop::hadoop_head_node: "head.node.fqdn" hadoop::hadoop_storage_dirs: [/data/1, /data/2] bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64 -hadoop_cluster_node::cluster_components: [hadoop, yarn] +hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce] bigtop::jdk_package_name: openjdk-8-jdk http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_centos6.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos6.yaml b/provisioner/docker/config_centos6.yaml index 1f06731..9b7b6ee 100644 --- a/provisioner/docker/config_centos6.yaml +++ b/provisioner/docker/config_centos6.yaml @@ -19,7 +19,7 @@ docker: repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"; distro: centos -components: [hadoop, yarn] +components: [hdfs, yarn, mapreduce] enable_local_repo: false -smoke_test_components: [mapreduce, pig] +smoke_test_components: [hdfs, yarn, mapreduce] jdk: "java-1.8.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_centos7.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml index 95724ff..3fe66e5 100644 --- a/provisioner/docker/config_centos7.yaml +++ b/provisioner/docker/config_centos7.yaml @@ -19,7 +19,7 @@ docker: repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"; distro: centos -components: [hadoop, yarn] +components: [hdfs, yarn, mapreduce] enable_local_repo: false -smoke_test_components: [mapreduce, pig] +smoke_test_components: [hdfs, yarn, mapreduce] jdk: "java-1.8.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_debian8.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_debian8.yaml b/provisioner/docker/config_debian8.yaml index ea8d150..d3e2399 100644 --- a/provisioner/docker/config_debian8.yaml +++ b/provisioner/docker/config_debian8.yaml @@ -19,7 +19,7 @@ docker: repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64"; distro: debian -components: [hadoop, yarn] +components: [hdfs, yarn, mapreduce] enable_local_repo: false -smoke_test_components: [mapreduce, pig] +smoke_test_components: [hdfs, yarn, mapreduce] jdk: "openjdk-8-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_ubuntu_trusty.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_ubuntu_trusty.yaml b/provisioner/docker/config_ubuntu_trusty.yaml deleted file mode 100644 index a56e97a..0000000 --- a/provisioner/docker/config_ubuntu_trusty.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_limit: "4g" - image: "bigtop/puppet:ubuntu-14.04" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"; -distro: debian -components: [hadoop, yarn] -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "openjdk-8-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_ubuntu_xenial.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_ubuntu_xenial.yaml b/provisioner/docker/config_ubuntu_xenial.yaml new file mode 100644 index 0000000..3b454fa --- /dev/null +++ b/provisioner/docker/config_ubuntu_xenial.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:ubuntu-16.04" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"; +distro: debian +components: [hdfs, yarn, mapreduce] +enable_local_repo: false +smoke_test_components: [hdfs, yarn, mapreduce] +jdk: "openjdk-8-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/vagrant/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml index aabc7bd..4974104 100644 --- a/provisioner/vagrant/vagrantconfig.yaml +++ b/provisioner/vagrant/vagrantconfig.yaml @@ -19,8 +19,8 @@ box: "puppetlabs/centos-7.2-64-nocm" repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"; num_instances: 1 distro: centos -components: [hadoop, yarn, hbase] +components: [hdfs, yarn, mapreduce] enable_local_repo: false run_smoke_tests: false -smoke_test_components: [mapreduce, pig] +smoke_test_components: [hdfs, yarn, mapreduce] jdk: "java-1.8.0-openjdk-devel.x86_64"
