Updated Branches: refs/heads/branch-1.4.2 0b5dfa542 -> cc1b2e18d
AMBARI-3612. Support for client component to report status with configs (dsen) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/cc1b2e18 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/cc1b2e18 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/cc1b2e18 Branch: refs/heads/branch-1.4.2 Commit: cc1b2e18d116042068f86cec2fb1848b53ab2814 Parents: 0b5dfa5 Author: Dmitry Sen <[email protected]> Authored: Tue Oct 29 16:22:42 2013 +0200 Committer: Dmitry Sen <[email protected]> Committed: Tue Oct 29 16:22:42 2013 +0200 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/ActionQueue.py | 7 +- .../src/main/python/ambari_agent/LiveStatus.py | 65 +++++++++++----- ambari-agent/src/test/python/TestLiveStatus.py | 13 +++- .../ambari/server/agent/HeartbeatMonitor.java | 80 ++++++++++---------- .../server/agent/TestHeartbeatMonitor.java | 13 ++-- 5 files changed, 111 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cc1b2e18/ambari-agent/src/main/python/ambari_agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py index 2881997..a796368 100644 --- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py +++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py @@ -158,8 +158,11 @@ class ActionQueue(threading.Thread): if command.has_key('configurationTags'): configHandler.write_actual(command['configurationTags']) roleResult['configurationTags'] = command['configurationTags'] - - if command.has_key('roleCommand') and command['roleCommand'] == self.ROLE_COMMAND_START: + component = {'serviceName':command['serviceName'],'componentName':command['role']} + if command.has_key('roleCommand') and \ + (command['roleCommand'] == self.ROLE_COMMAND_START or \ + (command['roleCommand'] == self.ROLE_COMMAND_INSTALL \ + and component in LiveStatus.CLIENT_COMPONENTS)): configHandler.copy_to_component(command['role']) roleResult['configurationTags'] = configHandler.read_actual_component(command['role']) self.commandStatuses.put_command_status(command, roleResult) http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cc1b2e18/ambari-agent/src/main/python/ambari_agent/LiveStatus.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py index b41b6e3..6138584 100644 --- a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py +++ b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py @@ -36,6 +36,31 @@ class LiveStatus: "YARN", "MAPREDUCE2", "FLUME" ] + CLIENT_COMPONENTS = [ + {"serviceName" : "HBASE", + "componentName" : "HBASE_CLIENT"}, + {"serviceName" : "HDFS", + "componentName" : "HDFS_CLIENT"}, + {"serviceName" : "MAPREDUCE", + "componentName" : "MAPREDUCE_CLIENT"}, + {"serviceName" : "ZOOKEEPER", + "componentName" : "ZOOKEEPER_CLIENT"}, + {"serviceName" : "OOZIE", + "componentName" : "OOZIE_CLIENT"}, + {"serviceName" : "HCATALOG", + "componentName" : "HCAT"}, + {"serviceName" : "HIVE", + "componentName" : "HIVE_CLIENT"}, + {"serviceName" : "YARN", + "componentName" : "YARN_CLIENT"}, + {"serviceName" : "MAPREDUCE2", + "componentName" : "MAPREDUCE2_CLIENT"}, + {"serviceName" : "PIG", + "componentName" : "PIG"}, + {"serviceName" : "SQOOP", + "componentName" : "SQOOP"} + ] + COMPONENTS = [ {"serviceName" : "HDFS", "componentName" : "DATANODE"}, @@ -118,29 +143,35 @@ class LiveStatus: # Live status was stripped from heartbeat after revision e1718dd def build(self): - global SERVICES, COMPONENTS, LIVE_STATUS, DEAD_STATUS + global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS statusCheck = StatusCheck(AmbariConfig.servicesToPidNames, AmbariConfig.pidPathesVars, self.globalConfig, AmbariConfig.servicesToLinuxUser) livestatus = None - for component in self.COMPONENTS: - if component["serviceName"] == self.service and component["componentName"] == self.component: - serviceStatus = statusCheck.getStatus(component["componentName"]) + component = {"serviceName" : self.service, "componentName" : self.component} + if component in self.COMPONENTS + self.CLIENT_COMPONENTS : + # CLIENT components can't have status STARTED + if component in self.CLIENT_COMPONENTS: + status = self.DEAD_STATUS + else: + serviceStatus = statusCheck.getStatus(self.component) + if serviceStatus is None: - logger.warn("There is no service to pid mapping for " + component["componentName"]) + logger.warn("There is no service to pid mapping for " + self.component) status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS - livestatus ={"componentName" : component["componentName"], - "msg" : "", - "status" : status, - "clusterName" : self.cluster, - "serviceName" : self.service, - "stackVersion": self.versionsHandler. - read_stack_version(component["componentName"]) - } - active_config = self.actualConfigHandler.read_actual_component(component['componentName']) - if not active_config is None: - livestatus['configurationTags'] = active_config - break + + livestatus ={"componentName" : self.component, + "msg" : "", + "status" : status, + "clusterName" : self.cluster, + "serviceName" : self.service, + "stackVersion": self.versionsHandler. + read_stack_version(self.component) + } + active_config = self.actualConfigHandler.read_actual_component(self.component) + if not active_config is None: + livestatus['configurationTags'] = active_config + logger.debug("The live status for component " + str(self.component) +\ " of service " + str(self.service) + " is " + str(livestatus)) return livestatus http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cc1b2e18/ambari-agent/src/test/python/TestLiveStatus.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/TestLiveStatus.py b/ambari-agent/src/test/python/TestLiveStatus.py index 60cbfc2..977ae4d 100644 --- a/ambari-agent/src/test/python/TestLiveStatus.py +++ b/ambari-agent/src/test/python/TestLiveStatus.py @@ -23,6 +23,8 @@ from ambari_agent.LiveStatus import LiveStatus from ambari_agent.AmbariConfig import AmbariConfig import socket import os, sys, StringIO +from ambari_agent import ActualConfigHandler +from mock.mock import patch, MagicMock, call class TestLiveStatus(TestCase): @@ -36,7 +38,8 @@ class TestLiveStatus(TestCase): # enable stdout sys.stdout = sys.__stdout__ - def test_build(self): + @patch.object(ActualConfigHandler.ActualConfigHandler, "read_actual_component") + def test_build(self, read_actual_component_mock): for component in LiveStatus.COMPONENTS: config = AmbariConfig().getConfig() config.set('agent', 'prefix', "dummy_files") @@ -48,4 +51,10 @@ class TestLiveStatus(TestCase): if component['componentName'] == 'GANGLIA_SERVER': self.assertEquals(result['stackVersion'],'{"stackName":"HDP","stackVersion":"1.2.2"}', 'Livestatus should contain component stack version') - + + # Test build status for CLIENT component (in LiveStatus.CLIENT_COMPONENTS) + read_actual_component_mock.return_value = "some tags" + livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config) + result = livestatus.build() + self.assertTrue(len(result) > 0, 'Livestatus should not be empty') + self.assertTrue(result.has_key('configurationTags')) http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cc1b2e18/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java index f556ab2..6161c63 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java @@ -187,57 +187,55 @@ public class HeartbeatMonitor implements Runnable { Service service = cl.getService(sch.getServiceName()); ServiceComponent sc = service.getServiceComponent(sch .getServiceComponentName()); - // Do not send status commands for client components - if (!sc.isClientComponent()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Live status will include status of service " + serviceName + " of cluster " + cl.getClusterName()); - } + // Send status commands for any components + if (LOG.isDebugEnabled()) { + LOG.debug("Live status will include status of service " + serviceName + " of cluster " + cl.getClusterName()); + } - Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); + Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); - // get the cluster config for type 'global' - // apply service overrides, if the tag is not the same - // apply host overrides, if any + // get the cluster config for type 'global' + // apply service overrides, if the tag is not the same + // apply host overrides, if any - Config clusterConfig = cl.getDesiredConfigByType("global"); - if (null != clusterConfig) { - // cluster config for 'global' - Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); + Config clusterConfig = cl.getDesiredConfigByType("global"); + if (null != clusterConfig) { + // cluster config for 'global' + Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties()); - // apply service overrides, only if the tag is not the same (for when service configs are overrides) - Config svcConfig = service.getDesiredConfigs().get("global"); - if (null != svcConfig && !svcConfig.getVersionTag().equals(clusterConfig.getVersionTag())) { - props.putAll(svcConfig.getProperties()); - } + // apply service overrides, only if the tag is not the same (for when service configs are overrides) + Config svcConfig = service.getDesiredConfigs().get("global"); + if (null != svcConfig && !svcConfig.getVersionTag().equals(clusterConfig.getVersionTag())) { + props.putAll(svcConfig.getProperties()); + } - // apply host overrides, if any - Host host = fsm.getHost(hostname); - DesiredConfig dc = host.getDesiredConfigs(cl.getClusterId()).get("global"); - if (null != dc) { - Config hostConfig = cl.getConfig("global", dc.getVersion()); - if (null != hostConfig) { - props.putAll(hostConfig.getProperties()); - } + // apply host overrides, if any + Host host = fsm.getHost(hostname); + DesiredConfig dc = host.getDesiredConfigs(cl.getClusterId()).get("global"); + if (null != dc) { + Config hostConfig = cl.getConfig("global", dc.getVersion()); + if (null != hostConfig) { + props.putAll(hostConfig.getProperties()); } - - configurations.put("global", props); } - // HACK - if any service exists with global tag, and we have none, use - // that instead - if (configurations.isEmpty()) { - Config config = service.getDesiredConfigs().get("global"); - if (null != config) - configurations.put("global", new HashMap<String, String>(config.getProperties())); - } + configurations.put("global", props); + } - StatusCommand statusCmd = new StatusCommand(); - statusCmd.setClusterName(cl.getClusterName()); - statusCmd.setServiceName(serviceName); - statusCmd.setComponentName(sch.getServiceComponentName()); - statusCmd.setConfigurations(configurations); - cmds.add(statusCmd); + // HACK - if any service exists with global tag, and we have none, use + // that instead + if (configurations.isEmpty()) { + Config config = service.getDesiredConfigs().get("global"); + if (null != config) + configurations.put("global", new HashMap<String, String>(config.getProperties())); } + + StatusCommand statusCmd = new StatusCommand(); + statusCmd.setClusterName(cl.getClusterName()); + statusCmd.setServiceName(serviceName); + statusCmd.setComponentName(sch.getServiceComponentName()); + statusCmd.setConfigurations(configurations); + cmds.add(statusCmd); } } return cmds; http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/cc1b2e18/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java index fd45a48..43313ec 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java @@ -169,7 +169,7 @@ public class TestHeartbeatMonitor { } @Test - public void testNoStatusCommandForClientComponents() throws Exception { + public void testStatusCommandForAnyComponents() throws Exception { Clusters clusters = injector.getInstance(Clusters.class); clusters.addHost(hostname1); clusters.getHost(hostname1).setOsType("centos6"); @@ -235,9 +235,11 @@ public class TestHeartbeatMonitor { hb.setResponseId(12); handler.handleHeartBeat(hb); + // HeartbeatMonitor should generate StatusCommands for + // MASTER, SLAVE or CLIENT components List<StatusCommand> cmds = hm.generateStatusCommands(hostname1); assertTrue("HeartbeatMonitor should generate StatusCommands for host1", - cmds.size() == 3); + cmds.size() == 4); assertEquals("HDFS", cmds.get(0).getServiceName()); boolean containsDATANODEStatus = false; boolean containsNAMENODEStatus = false; @@ -255,11 +257,12 @@ public class TestHeartbeatMonitor { assertTrue(containsDATANODEStatus); assertTrue(containsNAMENODEStatus); assertTrue(containsSECONDARY_NAMENODEStatus); - assertFalse(containsHDFS_CLIENTStatus); + assertTrue(containsHDFS_CLIENTStatus); cmds = hm.generateStatusCommands(hostname2); - assertTrue("HeartbeatMonitor should not generate StatusCommands for host2" + - " because it has only client components", cmds.isEmpty()); + assertTrue("HeartbeatMonitor should generate StatusCommands for host2, " + + "even if it has only client components", cmds.size() == 1); + assertTrue(cmds.get(0).getComponentName().equals(Role.HDFS_CLIENT.name())); } @Test
