Updated Branches: refs/heads/trunk c244d4fc4 -> 8871874f0
AMBARI-2897 Add component Zkfc to HDFS Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/8871874f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/8871874f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/8871874f Branch: refs/heads/trunk Commit: 8871874f087b96cad2dcd6003779969ded2603a7 Parents: c244d4f Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Aug 9 21:57:08 2013 +0300 Committer: Lisnichenko Dmitro <[email protected]> Committed: Wed Aug 14 18:43:14 2013 +0300 ---------------------------------------------------------------------- .../hdp-hadoop/manifests/hdfs/service_check.pp | 92 ++++++++++++++------ .../modules/hdp-hadoop/manifests/service.pp | 2 + .../puppet/modules/hdp-hadoop/manifests/zkfc.pp | 51 +++++++++++ .../puppet/modules/hdp/manifests/configfile.pp | 2 + .../main/puppet/modules/hdp/manifests/params.pp | 3 + .../main/python/ambari_agent/AmbariConfig.py | 4 +- .../src/main/python/ambari_agent/LiveStatus.py | 2 + .../java/org/apache/ambari/server/Role.java | 3 +- .../apache/ambari/server/utils/StageUtils.java | 1 + .../stacks/HDP/2.0.5/services/HDFS/metainfo.xml | 5 ++ .../HDPLocal/2.0.5/services/HDFS/metainfo.xml | 5 ++ 11 files changed, 140 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp index 7b49ed5..37d0eea 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp @@ -27,7 +27,7 @@ class hdp-hadoop::hdfs::service_check() $safemode_command = "dfsadmin -safemode get | grep OFF" $create_dir_cmd = "fs -mkdir ${dir} ; hadoop fs -chmod -R 777 ${dir}" - $test_dir_exists = "hadoop fs -test -e ${dir}" #TODO: may fix up fact that test needs explicit hadoop while omamnd does not + $test_dir_exists = "hadoop fs -test -e ${dir}" #TODO: may fix up fact that test needs explicit hadoop while command does not $cleanup_cmd = "fs -rm ${tmp_file}" #cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command $create_file_cmd = "${cleanup_cmd}; hadoop fs -put /etc/passwd ${tmp_file}" #TODO: inconsistent that second comamnd needs hadoop @@ -63,7 +63,6 @@ class hdp-hadoop::hdfs::service_check() } - #TODO: put in after testing # hdp-hadoop::exec-hadoop { 'hdfs::service_check::cleanup': # command => $cleanup_cmd, @@ -71,44 +70,58 @@ class hdp-hadoop::hdfs::service_check() # require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'], # before => Anchor['hdp-hadoop::hdfs::service_check::end'] #} - - if hdp_is_empty($hdp::params::journalnode_hosts) { - ##No journalnode hosts, just run hdfs test - hdp-hadoop::exec-hadoop { 'hdfs::service_check::test': - command => $test_cmd, - refreshonly => true, - user => $hdp::params::smokeuser, - require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'], - before => Anchor['hdp-hadoop::hdfs::service_check::end'] - } + + hdp-hadoop::exec-hadoop { 'hdfs::service_check::test': + command => $test_cmd, + refreshonly => true, + user => $hdp::params::smokeuser, + require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'], + before => Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:begin'] } - else { - ## Cluster has journalnode hosts, run hdfs test and test of journalnodes - hdp-hadoop::exec-hadoop { 'hdfs::service_check::test': - command => $test_cmd, - refreshonly => true, - user => $hdp::params::smokeuser, - require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'], - before => Class['hdp-hadoop::journalnode::service_check'] - } - + + anchor { 'hdp-hadoop::hdfs::service_check::journalnode_check:begin':} + + if hdp_is_empty($hdp::params::journalnode_hosts) { + ##No journalnode hosts + Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:begin'] -> + Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:end'] + + } else { + ## Cluster has journalnode hosts, run test of journalnodes $journalnode_hosts_comma_sep = hdp_comma_list_from_array($hdp::params::journalnode_hosts) - class { 'hdp-hadoop::journalnode::service_check': journalnode_hosts => $journalnode_hosts_comma_sep, - require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'], - before => Anchor['hdp-hadoop::hdfs::service_check::end'] + require => Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:begin'], + before => Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:end'] + } + } + + anchor { 'hdp-hadoop::hdfs::service_check::journalnode_check:end':} -> + anchor { 'hdp-hadoop::hdfs::service_check::zkfc_check:begin':} + + if hdp_is_empty($hdp::params::zkfc_hosts) { + ## No zkfc hosts + Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:begin'] -> + Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:end'] + } else { + ## Cluster has zkfc hosts, run test of local zkfc daemon if current host + ## is namenode. If namenode has not ZKFC installed, it is also considered + ## as a misconfiguration. + if ($hdp::params::is_namenode_master) { + class { 'hdp-hadoop::zkfc::service_check': + require => Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:begin'], + before => Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:end'] + } } } - - anchor{ 'hdp-hadoop::hdfs::service_check::end':} + anchor { 'hdp-hadoop::hdfs::service_check::zkfc_check:end':} -> + anchor{ 'hdp-hadoop::hdfs::service_check::end':} } class hdp-hadoop::journalnode::service_check($journalnode_hosts) { - $journalnode_port = $hdp::params::journalnode_port $smoke_test_user = $hdp::params::smokeuser @@ -132,3 +145,26 @@ class hdp-hadoop::journalnode::service_check($journalnode_hosts) } anchor{"hdp-hadoop::smoketest::begin":} -> File[$checkWebUIFilePath] -> Exec[$checkWebUIFilePath] -> anchor{"hdp-hadoop::smoketest::end":} } + +class hdp-hadoop::zkfc::service_check() inherits hdp-hadoop::params +{ + $hdfs_user = $hdp::params::hdfs_user + $pid_dir = "${hdp-hadoop::params::hadoop_pid_dir_prefix}/${hdfs_user}" + $pid_file = "${pid_dir}/hadoop-${hdfs_user}-zkfc.pid" + + # Here we check if pid file exists and if yes, then we run 'ps pid' command + # that returns 1 if process is not running + $check_zkfc_process_cmd = "ls ${pid_file} >/dev/null 2>&1 && ps `cat ${pid_file}` >/dev/null 2>&1" + + exec { $check_zkfc_process_cmd: + command => $check_zkfc_process_cmd, + tries => 3, + try_sleep => 5, + path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + logoutput => "true" + } + + anchor{"hdp-hadoop::zkfc::service_check::begin":} -> Exec[$check_zkfc_process_cmd] -> + anchor{"hdp-hadoop::zkfc::service_check::end":} + +} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp index 321e21a..0af21a9 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp @@ -57,6 +57,8 @@ define hdp-hadoop::service( } else { $daemon_cmd = "su - ${user} -c '${cmd} start ${name}'" } + # Here we check if pid file exists and if yes, then we run 'ps pid' command + # that returns 1 if process is not running $service_is_up = "ls ${pid_file} >/dev/null 2>&1 && ps `cat ${pid_file}` >/dev/null 2>&1" } elsif ($ensure == 'stopped') { if ($run_as_root == true) { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp new file mode 100644 index 0000000..3ec0055 --- /dev/null +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp @@ -0,0 +1,51 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +class hdp-hadoop::zkfc( + $service_state = $hdp::params::cluster_service_state, + $opts = {} +) inherits hdp-hadoop::params +{ + + Hdp-hadoop::Common<||>{service_states +> $service_state} + Hdp-hadoop::Package<||>{include_64_bit => true} + Hdp-hadoop::Configfile<||>{sizes +> 64} + + $hdfs_user = $hdp-hadoop::params::hdfs_user + + if ($service_state == 'no_op') { + } elsif ($service_state in ['running','stopped','installed_and_configured','uninstalled']) { + + #adds package, users and directories, and common hadoop configs + include hdp-hadoop::initialize + + hdp-hadoop::service{ 'zkfc': + ensure => $service_state, + user => $hdp-hadoop::params::hdfs_user, + create_pid_dir => true, + create_log_dir => true + } + + #top level does not need anchors + Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Service['zkfc'] -> Anchor['hdp-hadoop::end'] + } else { + hdp_fail("TODO not implemented yet: service_state = ${service_state}") + } +} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp index b768b1b..b7db742 100644 --- a/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp +++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp @@ -34,6 +34,7 @@ define hdp::configfile( $hs_host = $hdp::params::hs_host, $slave_hosts = $hdp::params::slave_hosts, $journalnode_hosts = $hdp::params::journalnode_hosts, + $zkfc_hosts = $hdp::params::zkfc_hosts, $mapred_tt_hosts = $hdp::params::mapred_tt_hosts, $all_hosts = $hdp::params::all_hosts, $hbase_rs_hosts = $hdp::params::hbase_rs_hosts, @@ -55,6 +56,7 @@ define hdp::configfile( $public_nm_hosts = $hdp::params::public_nm_hosts, $public_hs_host = $hdp::params::public_hs_host, $public_journalnode_hosts = $hdp::params::public_journalnode_hosts, + $public_zkfc_hosts = $hdp::params::public_zkfc_hosts, $public_jtnode_host = $hdp::params::public_jtnode_host, $public_hbase_master_hosts = $hdp::params::public_hbase_master_hosts, $public_zookeeper_hosts = $hdp::params::public_zookeeper_hosts, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp index 21e2251..1f91194 100644 --- a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp +++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp @@ -65,6 +65,7 @@ class hdp::params() $jtnode_host = hdp_default("jtnode_host") $slave_hosts = hdp_default("slave_hosts") $journalnode_hosts = hdp_default("journalnode_hosts") + $zkfc_hosts = hdp_default("zkfc_hosts") $nn_principal_str = hdp_default("hdfs-site/dfs.namenode.kerberos.principal", "nn/[email protected]") if ("_HOST" in $nn_principal_str and hdp_is_empty($namenode_host) == false) { @@ -156,6 +157,7 @@ class hdp::params() $public_nm_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$nm_hosts) $public_hs_host = hdp_host_attribute($hostAttributes,"publicfqdn",$hs_host) $public_journalnode_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$journalnode_hosts) + $public_zkfc_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zkfc_hosts) $public_jtnode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$jtnode_host) $public_hbase_master_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$hbase_master_hosts) $public_zookeeper_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zookeeper_hosts) @@ -172,6 +174,7 @@ class hdp::params() $public_nm_hosts = hdp_default("nm_hosts") $public_hs_host = hdp_default("hs_host") $public_journalnode_hosts = hdp_default("journalnode_hosts") + $public_zkfc_hosts = hdp_default("zkfc_hosts") $public_jtnode_host = hdp_default("jtnode_host") $public_hbase_master_hosts = hdp_default("hbase_master_hosts") $public_zookeeper_hosts = hdp_default("zookeeper_hosts") http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py index 567044d..d838d69 100644 --- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py +++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py @@ -136,7 +136,7 @@ rolesToClass = { 'YARN_SERVICE_CHECK': 'hdp-yarn::yarn::service_check', 'FLUME_SERVER': 'hdp-flume', 'JOURNALNODE': 'hdp-hadoop::journalnode', - 'JOURNALNODE_SERVICE_CHECK': 'hdp-hadoop::journalnode::service_check' + 'ZKFC': 'hdp-hadoop::zkfc' } serviceStates = { @@ -156,6 +156,7 @@ servicesToPidNames = { 'NODEMANAGER': 'yarn-{USER}-nodemanager.pid$', 'HISTORYSERVER': 'mapred-{USER}-historyserver.pid$', 'JOURNALNODE': 'hadoop-{USER}-journalnode.pid$', + 'ZKFC': 'hadoop-{USER}-zkfc.pid$', 'OOZIE_SERVER': 'oozie.pid', 'ZOOKEEPER_SERVER': 'zookeeper_server.pid', 'FLUME_SERVER': 'flume-node.pid', @@ -180,6 +181,7 @@ servicesToLinuxUser = { 'SECONDARY_NAMENODE': 'hdfs_user', 'DATANODE': 'hdfs_user', 'JOURNALNODE': 'hdfs_user', + 'ZKFC': 'hdfs_user', 'JOBTRACKER': 'mapred_user', 'TASKTRACKER': 'mapred_user', 'RESOURCEMANAGER': 'yarn_user', http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/python/ambari_agent/LiveStatus.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py index 44e0588..b41b6e3 100644 --- a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py +++ b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py @@ -45,6 +45,8 @@ class LiveStatus: "componentName" : "SECONDARY_NAMENODE"}, {"serviceName" : "HDFS", "componentName" : "JOURNALNODE"}, + {"serviceName" : "HDFS", + "componentName" : "ZKFC"}, {"serviceName" : "MAPREDUCE", "componentName" : "JOBTRACKER"}, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/java/org/apache/ambari/server/Role.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/Role.java b/ambari-server/src/main/java/org/apache/ambari/server/Role.java index 18f318c..8913f22 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/Role.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/Role.java @@ -83,5 +83,6 @@ public enum Role { HISTORYSERVER, TEZ_CLIENT, FLUME_SERVER, - JOURNALNODE + JOURNALNODE, + ZKFC } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index 59660f9..3262d92 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -90,6 +90,7 @@ public class StageUtils { componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts"); componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host"); componentToClusterInfoKeyMap.put("JOURNALNODE", "journalnode_hosts"); + componentToClusterInfoKeyMap.put("ZKFC", "zkfc_hosts"); componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts"); componentToClusterInfoKeyMap.put("FLUME_SERVER", "flume_hosts"); componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts"); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml index 95dc294..2ea1033 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml @@ -45,6 +45,11 @@ <name>JOURNALNODE</name> <category>SLAVE</category> </component> + + <component> + <name>ZKFC</name> + <category>SLAVE</category> + </component> </components> </metainfo> http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml index b695bf3..f6f7586 100644 --- a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml @@ -45,6 +45,11 @@ <name>JOURNALNODE</name> <category>SLAVE</category> </component> + + <component> + <name>ZKFC</name> + <category>SLAVE</category> + </component> </components>
