Updated Branches: refs/heads/trunk e883524cb -> 80a8850ca
AMBARI-2809. Add support for install, start and smoke test JournalNode slave Component in HDFS. Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/80a8850c Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/80a8850c Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/80a8850c Branch: refs/heads/trunk Commit: 80a8850cacd62871568605f02bc44b05094cc8bb Parents: e883524 Author: Alex <alex@ubuntu.(none)> Authored: Tue Aug 6 15:14:18 2013 +0300 Committer: Alex <alex@ubuntu.(none)> Committed: Tue Aug 6 15:16:07 2013 +0300 ---------------------------------------------------------------------- .../modules/hdp-hadoop/files/checkWebUI.py | 46 ++++++++++++++ .../hdp-hadoop/manifests/hdfs/service_check.pp | 64 +++++++++++++++++--- .../modules/hdp-hadoop/manifests/journalnode.pp | 60 ++++++++++++++++++ .../modules/hdp-hadoop/manifests/params.pp | 3 + .../puppet/modules/hdp/manifests/configfile.pp | 2 + .../main/puppet/modules/hdp/manifests/init.pp | 2 + .../main/puppet/modules/hdp/manifests/params.pp | 3 + .../main/python/ambari_agent/AmbariConfig.py | 6 +- .../src/main/python/ambari_agent/LiveStatus.py | 2 + .../java/org/apache/ambari/server/Role.java | 3 +- .../apache/ambari/server/utils/StageUtils.java | 1 + .../services/HDFS/configuration/hdfs-site.xml | 15 +++++ .../stacks/HDP/2.0.3/services/HDFS/metainfo.xml | 6 ++ 13 files changed, 203 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp-hadoop/files/checkWebUI.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/files/checkWebUI.py b/ambari-agent/src/main/puppet/modules/hdp-hadoop/files/checkWebUI.py new file mode 100644 index 0000000..2679e94 --- /dev/null +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/files/checkWebUI.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python2.6 + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import optparse +import urllib + +# +# Main. +# +def main(): + parser = optparse.OptionParser(usage="usage: %prog [options] component ") + parser.add_option("-u", "--url", dest="url", help="Host:Port for WEB UI to check it availability") + + (options, args) = parser.parse_args() + + url = options.url + + try: + httpCode = urllib.urlopen('http://' + url).getcode() + except Exception: + httpCode = 404 + + if httpCode != 200: + exit(1) + else: + exit(0) + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp index ace972d..511fabe 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp @@ -62,14 +62,7 @@ class hdp-hadoop::hdfs::service_check() notify => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'] } - hdp-hadoop::exec-hadoop { 'hdfs::service_check::test': - command => $test_cmd, - refreshonly => true, - user => $hdp::params::smokeuser, - require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'], - #notify => Hdp-hadoop::Exec-hadoop['hdfs::service_check::cleanup'] #TODO: put in after testing - before => Anchor['hdp-hadoop::hdfs::service_check::end'] #TODO: remove after testing - } + #TODO: put in after testing # hdp-hadoop::exec-hadoop { 'hdfs::service_check::cleanup': @@ -78,6 +71,61 @@ class hdp-hadoop::hdfs::service_check() # require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'], # before => Anchor['hdp-hadoop::hdfs::service_check::end'] #} + + if hdp_is_empty($hdp::params::journalnode_hosts) { + ##No journalnode hosts, just run hdfs test + hdp-hadoop::exec-hadoop { 'hdfs::service_check::test': + command => $test_cmd, + refreshonly => true, + user => $hdp::params::smokeuser, + require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'], + before => Anchor['hdp-hadoop::hdfs::service_check::end'] + } + } + else { + ## Cluster has journalnode hosts, run hdfs test and test of journalnodes + hdp-hadoop::exec-hadoop { 'hdfs::service_check::test': + command => $test_cmd, + refreshonly => true, + user => $hdp::params::smokeuser, + require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'], + before => Class['hdp-hadoop::journalnode::service_check'] + } + class { 'hdp-hadoop::journalnode::service_check': + journalnode_host => $hdp::params::journalnode_hosts, + require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'], + before => Anchor['hdp-hadoop::hdfs::service_check::end'] + } + } + + anchor{ 'hdp-hadoop::hdfs::service_check::end':} } + +class hdp-hadoop::journalnode::service_check($journalnode_host) +{ + + $journalnode_port = $hdp::params::journalnode_port + $smoke_test_user = $hdp::params::smokeuser + + $checkWebUIFileName = "checkWebUI.py" + $checkWebUIFilePath = "/tmp/$checkWebUIFileName" + + $checkWebUICmd = "su - ${smoke_test_user} -c 'python $checkWebUIFilePath -u $journalnode_host:$journalnode_port'" + + file { $checkWebUIFilePath: + ensure => present, + source => "puppet:///modules/hdp-hadoop/$checkWebUIFileName", + mode => '0755' + } + + exec { $checkWebUIFilePath: + command => $checkWebUICmd, + tries => 3, + try_sleep => 5, + path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + logoutput => "true" +} + anchor{"hdp-hadoop::smoketest::begin":} -> File[$checkWebUIFilePath] -> Exec[$checkWebUIFilePath] -> anchor{"hdp-hadoop::smoketest::end":} +} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/journalnode.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/journalnode.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/journalnode.pp new file mode 100644 index 0000000..3d5aaa6 --- /dev/null +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/journalnode.pp @@ -0,0 +1,60 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +class hdp-hadoop::journalnode( + $service_state = $hdp::params::cluster_service_state, + $opts = {} +) inherits hdp-hadoop::params +{ + + $hdp::params::service_exists['hdp-hadoop::journalnode'] = true + + Hdp-hadoop::Common<||>{service_states +> $service_state} + Hdp-hadoop::Package<||>{include_64_bit => true} + Hdp-hadoop::Configfile<||>{sizes +> 64} + + $jn_edits_dir = $hdp-hadoop::params::jn_edits_dir + $hdfs_user = $hdp-hadoop::params::hdfs_user + + if ($service_state == 'no_op') { + } elsif ($service_state in ['running','stopped','installed_and_configured','uninstalled']) { + + #adds package, users and directories, and common hadoop configs + include hdp-hadoop::initialize + + hdp::directory_recursive_create{ $jn_edits_dir: + service_state => $service_state, + force => true, + owner => $hdfs_user + } + + hdp-hadoop::service{ 'journalnode': + ensure => $service_state, + user => $hdp-hadoop::params::hdfs_user, + create_pid_dir => true, + create_log_dir => true + } + + #top level does not need anchors + Anchor['hdp-hadoop::begin'] -> Hdp::Directory_recursive_create[$jn_edits_dir] -> Hdp-hadoop::Service['journalnode'] -> Anchor['hdp-hadoop::end'] + } else { + hdp_fail("TODO not implemented yet: service_state = ${service_state}") + } +} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp index fe5efce..f1b96fc 100644 --- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp +++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp @@ -131,6 +131,9 @@ class hdp-hadoop::params( $dfs_support_append = hdp_default("hdfs-site/dfs.support.append",true) $dfs_webhdfs_enabled = hdp_default("hdfs-site/dfs.webhdfs.enabled",false) + + $jn_edits_dir = hdp_default("hdfs-site/dfs.journalnode.edits.dir", "/grid/0/hdfs/journal") + ######### mapred ####### http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp index 84d8309..b768b1b 100644 --- a/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp +++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp @@ -33,6 +33,7 @@ define hdp::configfile( $nm_hosts = $hdp::params::nm_hosts, $hs_host = $hdp::params::hs_host, $slave_hosts = $hdp::params::slave_hosts, + $journalnode_hosts = $hdp::params::journalnode_hosts, $mapred_tt_hosts = $hdp::params::mapred_tt_hosts, $all_hosts = $hdp::params::all_hosts, $hbase_rs_hosts = $hdp::params::hbase_rs_hosts, @@ -53,6 +54,7 @@ define hdp::configfile( $public_rm_host = $hdp::params::public_rm_host, $public_nm_hosts = $hdp::params::public_nm_hosts, $public_hs_host = $hdp::params::public_hs_host, + $public_journalnode_hosts = $hdp::params::public_journalnode_hosts, $public_jtnode_host = $hdp::params::public_jtnode_host, $public_hbase_master_hosts = $hdp::params::public_hbase_master_hosts, $public_zookeeper_hosts = $hdp::params::public_zookeeper_hosts, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp index 1a0bf32..7a53ed6 100644 --- a/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp +++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp @@ -39,10 +39,12 @@ class hdp( $namenode_port = hdp_get_port_from_url($hdfs-site["dfs.http.address"]) $snamenode_port = hdp_get_port_from_url($hdfs-site["dfs.secondary.http.address"]) $datanode_port = hdp_get_port_from_url($hdfs-site["dfs.datanode.http.address"]) + $journalnode_port = hdp_get_port_from_url($hdfs-site["dfs.journalnode.http-address"]) } else { $namenode_port = "50070" $snamenode_port = "50090" $datanode_port = "50075" + $journalnode_port = "8480" } if has_key($configuration, 'mapred-site') { http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp index 5a2b91b..b27bff5 100644 --- a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp +++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp @@ -64,6 +64,7 @@ class hdp::params() $snamenode_host = hdp_default("snamenode_host") $jtnode_host = hdp_default("jtnode_host") $slave_hosts = hdp_default("slave_hosts") + $journalnode_hosts = hdp_default("journalnode_hosts") $nn_principal_str = hdp_default("hdfs-site/dfs.namenode.kerberos.principal", "nn/[email protected]") if ("_HOST" in $nn_principal_str and hdp_is_empty($namenode_host) == false) { @@ -154,6 +155,7 @@ class hdp::params() $public_rm_host = hdp_host_attribute($hostAttributes,"publicfqdn",$rm_host) $public_nm_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$nm_hosts) $public_hs_host = hdp_host_attribute($hostAttributes,"publicfqdn",$hs_host) + $public_journalnode_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$journalnode_hosts) $public_jtnode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$jtnode_host) $public_hbase_master_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$hbase_master_hosts) $public_zookeeper_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zookeeper_hosts) @@ -169,6 +171,7 @@ class hdp::params() $public_rm_host = hdp_default("rm_host") $public_nm_hosts = hdp_default("nm_hosts") $public_hs_host = hdp_default("hs_host") + $public_journalnode_hosts = hdp_default("journalnode_hosts") $public_jtnode_host = hdp_default("jtnode_host") $public_hbase_master_hosts = hdp_default("hbase_master_hosts") $public_zookeeper_hosts = hdp_default("zookeeper_hosts") http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py index 9dad41a..567044d 100644 --- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py +++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py @@ -134,7 +134,9 @@ rolesToClass = { 'HISTORYSERVER_SERVICE_CHECK': 'hdp-yarn::historyserver::service_check', 'TEZ_CLIENT': 'hdp-tez::tez_client', 'YARN_SERVICE_CHECK': 'hdp-yarn::yarn::service_check', - 'FLUME_SERVER': 'hdp-flume' + 'FLUME_SERVER': 'hdp-flume', + 'JOURNALNODE': 'hdp-hadoop::journalnode', + 'JOURNALNODE_SERVICE_CHECK': 'hdp-hadoop::journalnode::service_check' } serviceStates = { @@ -153,6 +155,7 @@ servicesToPidNames = { 'RESOURCEMANAGER': 'yarn-{USER}-resourcemanager.pid$', 'NODEMANAGER': 'yarn-{USER}-nodemanager.pid$', 'HISTORYSERVER': 'mapred-{USER}-historyserver.pid$', + 'JOURNALNODE': 'hadoop-{USER}-journalnode.pid$', 'OOZIE_SERVER': 'oozie.pid', 'ZOOKEEPER_SERVER': 'zookeeper_server.pid', 'FLUME_SERVER': 'flume-node.pid', @@ -176,6 +179,7 @@ servicesToLinuxUser = { 'NAMENODE': 'hdfs_user', 'SECONDARY_NAMENODE': 'hdfs_user', 'DATANODE': 'hdfs_user', + 'JOURNALNODE': 'hdfs_user', 'JOBTRACKER': 'mapred_user', 'TASKTRACKER': 'mapred_user', 'RESOURCEMANAGER': 'yarn_user', http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-agent/src/main/python/ambari_agent/LiveStatus.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py index 087582d..44e0588 100644 --- a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py +++ b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py @@ -43,6 +43,8 @@ class LiveStatus: "componentName" : "NAMENODE"}, {"serviceName" : "HDFS", "componentName" : "SECONDARY_NAMENODE"}, + {"serviceName" : "HDFS", + "componentName" : "JOURNALNODE"}, {"serviceName" : "MAPREDUCE", "componentName" : "JOBTRACKER"}, http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-server/src/main/java/org/apache/ambari/server/Role.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/Role.java b/ambari-server/src/main/java/org/apache/ambari/server/Role.java index 82180dc..18f318c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/Role.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/Role.java @@ -82,5 +82,6 @@ public enum Role { YARN_CLIENT, HISTORYSERVER, TEZ_CLIENT, - FLUME_SERVER + FLUME_SERVER, + JOURNALNODE } http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index c4e9dcf..59660f9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -89,6 +89,7 @@ public class StageUtils { componentToClusterInfoKeyMap.put("RESOURCEMANAGER", "rm_host"); componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts"); componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host"); + componentToClusterInfoKeyMap.put("JOURNALNODE", "journalnode_hosts"); componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts"); componentToClusterInfoKeyMap.put("FLUME_SERVER", "flume_hosts"); componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts"); http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/hdfs-site.xml index 10d58c5..fcd6015 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/hdfs-site.xml @@ -455,5 +455,20 @@ don't exist, they will be created with this permission.</description> <value>30000</value> <description>Datanode is stale after not getting a heartbeat in this interval in ms</description> </property> + + <property> + <name>dfs.journalnode.http-address</name> + <value>0.0.0.0:8480</value> + <description>The address and port the JournalNode web UI listens on. + If the port is 0 then the server will start on a free port. </description> + </property> + + <property> + <name>dfs.journalnode.edits.dir</name> + <value>/grid/0/hdfs/journal</value> + <description>The path where the JournalNode daemon will store its local state. </description> + </property> + + </configuration> http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/80a8850c/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/metainfo.xml index 8514121..7eb4d8c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/metainfo.xml @@ -40,6 +40,12 @@ <name>HDFS_CLIENT</name> <category>CLIENT</category> </component> + + <component> + <name>JOURNALNODE</name> + <category>SLAVE</category> + </component> + </components>
