Updated Branches: refs/heads/trunk 81cd63d9e -> 67fd6fb7c
AMBARI-2970. NameNode fails to come up in HA mode. (swagle) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/67fd6fb7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/67fd6fb7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/67fd6fb7 Branch: refs/heads/trunk Commit: 67fd6fb7c489b44b83f8552f61e35b28aab5cbc5 Parents: 81cd63d Author: Siddharth Wagle <[email protected]> Authored: Wed Aug 21 11:36:12 2013 -0700 Committer: Siddharth Wagle <[email protected]> Committed: Wed Aug 21 11:39:13 2013 -0700 ---------------------------------------------------------------------- .../functions/hdp_hadoop_get_namenode_id.rb | 47 ++++++++++++++++++++ .../hdp-hadoop/manifests/hdfs/directory.pp | 21 ++++++--- .../puppet/modules/hdp-hadoop/manifests/init.pp | 13 ++++-- .../modules/hdp-hadoop/manifests/namenode.pp | 44 +++++++++++------- .../modules/hdp-hadoop/manifests/params.pp | 5 +++ 5 files changed, 103 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb new file mode 100644 index 0000000..719d1e6 --- /dev/null +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +#to get namenode service id in HA setup + +module Puppet::Parser::Functions + newfunction(:hdp_hadoop_get_namenode_id, :type => :rvalue) do |args| + namenode_id = "" + if args.length > 1 + # Get hdfs-site to lookup hostname properties + lookup_property = args[0] + siteName = args[1] + siteConfig = lookupvar("#{siteName}") + nn_ids_str = lookupvar("::hdp::params::dfs_ha_namenode_ids") + hostname = lookupvar("::hdp::params::hostname") + nn_ids = nn_ids_str.to_s.split(',') + + if nn_ids.length > 1 + nn_ids.each do |id| + lookup_key = lookup_property + "." + id.to_s.strip + property_val = siteConfig.fetch(lookup_key, "") + if property_val != "" and property_val.include? hostname + namenode_id = id + end + end + end + end + namenode_id.strip + end +end \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp index 453aeb3..78ac747 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp @@ -32,22 +32,31 @@ define hdp-hadoop::hdfs::directory( $namenode_safe_mode_off = "hadoop dfsadmin -safemode get|grep 'Safe mode is OFF'" $tries = 30 $try_sleep = 10 - - if ($service_state == 'running') { + if ($hdp::params::dfs_ha_enabled == true) { + $namenode_id = $hdp-hadoop::params::namenode_id + if (hdp_is_empty($namenode_id) == false) { + $dfs_check_nn_status_cmd = "hdfs haadmin -getServiceState $namenode_id | grep active > /dev/null" + } + } else { + $dfs_check_nn_status_cmd = "true" + } + if ($service_state == 'running') { if (hdp_get_major_stack_version($hdp::params::stack_version) >= 2) { $mkdir_cmd = "fs -mkdir -p ${name}" } else { $mkdir_cmd = "fs -mkdir ${name}" } + hdp-hadoop::exec-hadoop { $mkdir_cmd: command => $mkdir_cmd, - unless => $dir_exists, - onlyif => $namenode_safe_mode_off, + unless => "$dfs_check_nn_status_cmd && $dir_exists", + onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off", try_sleep => $try_sleep, tries => $tries } + if ($owner == unset) { $chown = "" } else { @@ -67,7 +76,7 @@ define hdp-hadoop::hdfs::directory( } hdp-hadoop::exec-hadoop {$chown_cmd : command => $chown_cmd, - onlyif => "$namenode_safe_mode_off && $dir_exists", + onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists", try_sleep => $try_sleep, tries => $tries } @@ -83,7 +92,7 @@ define hdp-hadoop::hdfs::directory( } hdp-hadoop::exec-hadoop {$chmod_cmd : command => $chmod_cmd, - onlyif => "$namenode_safe_mode_off && $dir_exists", + onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists", try_sleep => $try_sleep, tries => $tries } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp index bd63543..b9624c4 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp @@ -408,7 +408,8 @@ define hdp-hadoop::exec-hadoop( $try_sleep = undef, $user = undef, $logoutput = undef, - $onlyif = undef + $onlyif = undef, + $path = undef ) { include hdp-hadoop::params @@ -439,10 +440,14 @@ define hdp-hadoop::exec-hadoop( $kinit_if_needed = "" } - if ($echo_yes == true) { - $cmd = "yes Y | hadoop --config ${conf_dir} ${command}" + if ($path == undef) { + if ($echo_yes == true) { + $cmd = "yes Y | hadoop --config ${conf_dir} ${command}" + } else { + $cmd = "hadoop --config ${conf_dir} ${command}" + } } else { - $cmd = "hadoop --config ${conf_dir} ${command}" + $cmd = "${path} ${command}" } if ($kinit_if_needed != "") { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp index c203cf3..4ae78ae 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp @@ -63,7 +63,7 @@ class hdp-hadoop::namenode( group => $hdp::params::user_group } } - + hdp-hadoop::namenode::create_name_dirs { $dfs_name_dir: service_state => $service_state } @@ -82,20 +82,26 @@ class hdp-hadoop::namenode( } hdp-hadoop::namenode::create_app_directories { 'create_app_directories' : - service_state => $service_state + service_state => $service_state } hdp-hadoop::namenode::create_user_directories { 'create_user_directories' : - service_state => $service_state + service_state => $service_state } - #top level does not need anchors - Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Namenode::Create_name_dirs<||> -> Hdp-hadoop::Service['namenode'] -> - # Now, creating directories inside HDFS - Hdp-hadoop::Namenode::Create_app_directories<||> -> Hdp-hadoop::Namenode::Create_user_directories<||> -> Anchor['hdp-hadoop::end'] + Anchor['hdp-hadoop::begin'] -> + Hdp-hadoop::Namenode::Create_name_dirs<||> -> + Hdp-hadoop::Service['namenode'] -> + Hdp-hadoop::Namenode::Create_app_directories<||> -> + Hdp-hadoop::Namenode::Create_user_directories<||> -> + Anchor['hdp-hadoop::end'] + if ($service_state == 'running' and $format == true) { - Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Namenode::Create_name_dirs<||> -> - Class['hdp-hadoop::namenode::format'] -> Hdp-hadoop::Service['namenode'] -> Anchor['hdp-hadoop::end'] + Anchor['hdp-hadoop::begin'] -> + Hdp-hadoop::Namenode::Create_name_dirs<||> -> + Class['hdp-hadoop::namenode::format'] -> + Hdp-hadoop::Service['namenode'] -> + Anchor['hdp-hadoop::end'] } } else { hdp_fail("TODO not implemented yet: service_state = ${service_state}") @@ -128,10 +134,12 @@ define hdp-hadoop::namenode::create_app_directories($service_state) service_state => $service_state, owner => $hdp-hadoop::params::mapred_user } + hdp-hadoop::hdfs::directory{ '/mapred/system' : service_state => $service_state, owner => $hdp-hadoop::params::mapred_user } + Hdp-hadoop::Hdfs::Directory['/mapred'] -> Hdp-hadoop::Hdfs::Directory['/mapred/system'] if ($hdp::params::hbase_master_hosts != "") { @@ -140,12 +148,13 @@ define hdp-hadoop::namenode::create_app_directories($service_state) owner => $hdp::params::hbase_user, service_state => $service_state } + $hbase_staging_dir = $hdp::params::hbase_staging_dir - hdp-hadoop::hdfs::directory { $hbase_staging_dir: - owner => $hdp::params::hbase_user, - service_state => $service_state, - mode => '711', - } + hdp-hadoop::hdfs::directory { $hbase_staging_dir: + owner => $hdp::params::hbase_user, + service_state => $service_state, + mode => '711' + } } if ($hdp::params::hive_server_host != "") { @@ -224,7 +233,7 @@ define hdp-hadoop::namenode::create_user_directories($service_state) $hive_hdfs_user_dir = $hdp::params::hive_hdfs_user_dir $hive_dir_item="$hive_hdfs_user_dir," } else { - $hive_dir_item="" + $hive_dir_item="" } if ($hdp::params::oozie_server != "") { @@ -252,8 +261,9 @@ define hdp-hadoop::namenode::create_user_directories($service_state) #Get unique users directories set $users_dirs_set = hdp_set_from_comma_list($users_dir_list_comm_sep) - hdp-hadoop::namenode::create_user_directory{$users_dirs_set: - service_state => $service_state} + hdp-hadoop::namenode::create_user_directory{ $users_dirs_set: + service_state => $service_state + } } } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp index df3492e..3cc765c 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp @@ -202,4 +202,9 @@ class hdp-hadoop::params( $ambari_db_rca_username = hdp_default("ambari_db_rca_username", "mapred") $ambari_db_rca_password = hdp_default("ambari_db_rca_password", "mapred") + if ($hdp::params::dfs_ha_enabled == true) { + $nameservice = $hdp::params::dfs_ha_nameservices + $namenode_id = hdp_hadoop_get_namenode_id("dfs.namenode.rpc-address.${nameservice}", "hdfs-site") + } + }
