Author: mahadev
Date: Wed Jan 23 21:05:54 2013
New Revision: 1437734
URL: http://svn.apache.org/viewvc?rev=1437734&view=rev
Log:
AMBARI-1233. Directory permissions on httpd /var/www/cgi-bin should not be
touched by Ambari. (mahadev)
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-ganglia/manifests/server.pp
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/files/check_name_dir_status.php
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py
incubator/ambari/trunk/ambari-agent/src/test/python/TestActionQueue.py
incubator/ambari/trunk/ambari-agent/src/test/python/TestHeartbeat.py
incubator/ambari/trunk/ambari-agent/src/test/python/TestLiveStatus.py
incubator/ambari/trunk/ambari-agent/src/test/python/TestStatusCheck.py
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
Modified: incubator/ambari/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Wed Jan 23 21:05:54 2013
@@ -166,6 +166,13 @@ Trunk (unreleased changes):
AMBARI-1243. Remove unwanted import causing the builds to fail on linux.
(mahadev)
+ AMBARI-1233. Directory permissions on httpd /var/www/cgi-bin should not be
+ touched by Ambari. (mahadev)
+
+ AMBARI-1170. For live status checks we should only look at the run
+ directories that we get from the server (only for hadoop and its eco system)
+ and not all. (mahadev)
+
AMBARI-1.2.0 branch:
INCOMPATIBLE CHANGES
Modified:
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-ganglia/manifests/server.pp
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-ganglia/manifests/server.pp?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-ganglia/manifests/server.pp
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-ganglia/manifests/server.pp
Wed Jan 23 21:05:54 2013
@@ -130,7 +130,8 @@ class hdp-ganglia::server::files(
$rrd_py_path = $hdp::params::rrd_py_path [$hdp::params::hdp_os_type]
hdp::directory_recursive_create{$rrd_py_path:
- ensure => "directory"
+ ensure => "directory",
+ override_owner => false
}
$rrd_py_file_path = "${rrd_py_path}/rrd.py"
Modified:
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/files/check_name_dir_status.php
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/files/check_name_dir_status.php?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/files/check_name_dir_status.php
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp-nagios/files/check_name_dir_status.php
Wed Jan 23 21:05:54 2013
@@ -36,8 +36,8 @@
$json_array = json_decode($json_string, true);
$object = $json_array['beans'][0];
if ($object['NameDirStatuses'] == "") {
- echo "UNKNOWN: Namenode directory status not available via
http://<nn_host>:port/jmx url" . "\n";
- exit(3);
+ echo "WARNING: Namenode directory status not available via
http://".$host.":".$port."/jmx url" . "\n";
+ exit(1);
}
$NameDirStatuses = json_decode($object['NameDirStatuses'], true);
$failed_dir_count = count($NameDirStatuses['failed']);
Modified:
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp
Wed Jan 23 21:05:54 2013
@@ -183,7 +183,8 @@ define hdp::directory(
$mode = undef,
$ensure = directory,
$force = undef,
- $service_state = 'running'
+ $service_state = 'running',
+ $override_owner = false
)
{
if (($service_state == 'uninstalled') and ($wipeoff_data == true)) {
@@ -195,13 +196,21 @@ define hdp::directory(
force => $force
}
} elsif ($service_state != 'uninstalled') {
- file { $name :
- ensure => present,
- owner => $owner,
- group => $group,
- mode => $mode,
- force => $force
- }
+ if $override_owner == true {
+ file { $name :
+ ensure => present,
+ owner => $owner,
+ group => $group,
+ mode => $mode,
+ force => $force
+ }
+ } else {
+ file { $name :
+ ensure => present,
+ mode => $mode,
+ force => $force
+ }
+ }
}
}
#TODO: check on -R flag and use of recurse
@@ -212,7 +221,8 @@ define hdp::directory_recursive_create(
$context_tag = undef,
$ensure = directory,
$force = undef,
- $service_state = 'running'
+ $service_state = 'running',
+ $override_owner = true
)
{
@@ -227,7 +237,8 @@ define hdp::directory_recursive_create(
mode => $mode,
ensure => $ensure,
force => $force,
- service_state => $service_state
+ service_state => $service_state,
+ override_owner => $override_owner
}
Hdp::Exec["mkdir -p ${name}"] -> Hdp::Directory[$name]
}
Modified:
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
Wed Jan 23 21:05:54 2013
@@ -109,8 +109,9 @@ class ActionQueue(threading.Thread):
cluster = command['clusterName']
service = command['serviceName']
component = command['componentName']
+ globalConfig = command['configurations']['global']
try:
- livestatus = LiveStatus(cluster, service, component)
+ livestatus = LiveStatus(cluster, service, component, globalConfig)
result = livestatus.build()
logger.info("Got live status for component " + component + " of
service " + str(service) +\
" of cluster " + str(cluster) + "\n" +
pprint.pformat(result))
Modified:
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
Wed Jan 23 21:05:54 2013
@@ -35,7 +35,6 @@ secured_url_port=8441
prefix=/tmp/ambari-agent
[services]
-pidLookupPath=/var/run/
[stack]
installprefix=/tmp
@@ -148,6 +147,28 @@ servicesToPidNames = {
'MYSQL_SERVER': 'mysqld.pid'
}
+pidPathesVars = [
+ {'var' : 'hadoop_pid_dir_prefix',
+ 'defaultValue' : '/var/run/hadoop'},
+ {'var' : 'hadoop_pid_dir_prefix',
+ 'defaultValue' : '/var/run/hadoop'},
+ {'var' : 'ganglia_runtime_dir',
+ 'defaultValue' : '/var/run/ganglia/hdp'},
+ {'var' : 'hbase_pid_dir',
+ 'defaultValue' : '/var/run/hbase'},
+ {'var' : '',
+ 'defaultValue' : '/var/run/nagios'},
+ {'var' : 'zk_pid_dir',
+ 'defaultValue' : '/var/run/zookeeper'},
+ {'var' : 'oozie_pid_dir',
+ 'defaultValue' : '/var/run/oozie'},
+ {'var' : 'hcat_pid_dir',
+ 'defaultValue' : '/var/run/webhcat'},
+ {'var' : 'hive_pid_dir',
+ 'defaultValue' : '/var/run/hive'},
+ {'var' : 'mysqld_pid_dir',
+ 'defaultValue' : '/var/run/mysqld'}
+]
class AmbariConfig:
def getConfig(self):
@@ -170,6 +191,10 @@ class AmbariConfig:
global servicesToPidNames
return servicesToPidNames
+ def getPidPathesVars(self):
+ global pidPathesVars
+ return pidPathesVars
+
def setConfig(customConfig):
global config
Modified:
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
Wed Jan 23 21:05:54 2013
@@ -91,10 +91,11 @@ class LiveStatus:
LIVE_STATUS = "STARTED"
DEAD_STATUS = "INSTALLED"
- def __init__(self, cluster, service, component):
+ def __init__(self, cluster, service, component, globalConfig):
self.cluster = cluster
self.service = service
self.component = component
+ self.globalConfig = globalConfig
def belongsToService(self, component):
@@ -104,9 +105,7 @@ class LiveStatus:
# Live status was stripped from heartbeat after revision e1718dd
def build(self):
global SERVICES, COMPONENTS, LIVE_STATUS, DEAD_STATUS
- pidLookupPath = AmbariConfig.config.get('services','pidLookupPath')
-
- statusCheck = StatusCheck(pidLookupPath, AmbariConfig.servicesToPidNames)
+ statusCheck = StatusCheck(AmbariConfig.servicesToPidNames,
AmbariConfig.pidPathesVars, self.globalConfig)
livestatus = None
for component in self.COMPONENTS:
if component["serviceName"] == self.service and
component["componentName"] == self.component:
Modified:
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py
(original)
+++
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py
Wed Jan 23 21:05:54 2013
@@ -28,15 +28,9 @@ import re
logger = logging.getLogger()
-
-
-
class StatusCheck:
-
- def get_pair(self, line):
- key, sep, value = line.strip().partition("=")
- return key, value
-
+
+
def listFiles(self, dir):
basedir = dir
logger.debug("Files in " + os.path.abspath(dir) + ": ")
@@ -45,9 +39,9 @@ class StatusCheck:
if os.path.isdir(dir):
for item in os.listdir(dir):
if os.path.isfile(item) and item.endswith('.pid'):
- self.pidFilesDict[item.split(os.sep).pop()] = item
+ self.pidFilesDict[item.split(os.sep).pop()] = os.getcwd() +
os.sep + item
else:
- subdirlist.append(os.path.join(basedir, item))
+ subdirlist.append(os.path.join(basedir, item))
for subdir in subdirlist:
self.listFiles(subdir)
else:
@@ -55,19 +49,35 @@ class StatusCheck:
self.pidFilesDict[dir.split(os.sep).pop()] = dir
except OSError as e:
logger.info(e.strerror + ' to ' + e.filename)
+
+ def fillDirValues(self):
+ try:
+ for pidVar in self.pidPathesVars:
+ pidVarName = pidVar['var']
+ pidDefaultvalue = pidVar['defaultValue']
+ if self.globalConfig.has_key(pidVarName):
+ self.pidPathes.append(self.globalConfig[pidVarName])
+ else:
+ self.pidPathes.append(pidDefaultvalue)
+ except Exception as e:
+ logger.error("Error while filling directories values " + str(e))
+
+ def __init__(self, serviceToPidDict, pidPathesVars, globalConfig):
- def __init__(self, path, serviceToPidDict):
- if not os.path.isdir(path):
- raise ValueError("Path argument must be valid directory")
-
- self.path = path
self.serToPidDict = serviceToPidDict
+ self.pidPathesVars = pidPathesVars
+ self.pidPathes = []
self.sh = shellRunner()
self.pidFilesDict = {}
- self.listFiles(self.path)
-
+ self.globalConfig = globalConfig
+
+ self.fillDirValues()
+
+ for pidPath in self.pidPathes:
+ self.listFiles(pidPath)
def getIsLive(self, pidPath):
+
if not pidPath:
return False
@@ -92,7 +102,7 @@ class StatusCheck:
try:
pidPath = None
pidPattern = self.serToPidDict[serviceCode]
- logger.info( 'pidPattern: ' + pidPattern)
+ logger.info('pidPattern: ' + pidPattern)
except KeyError as e:
logger.warn('There is no mapping for ' + serviceCode)
return None
Modified: incubator/ambari/trunk/ambari-agent/src/test/python/TestActionQueue.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/test/python/TestActionQueue.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/test/python/TestActionQueue.py
(original)
+++ incubator/ambari/trunk/ambari-agent/src/test/python/TestActionQueue.py Wed
Jan 23 21:05:54 2013
@@ -63,7 +63,8 @@ class TestActionQueue(TestCase):
'clusterHostInfo': "clusterHostInfo",
'roleCommand': "roleCommand",
'configurations': "configurations",
- 'commandType': "EXECUTION_COMMAND"
+ 'commandType': "EXECUTION_COMMAND",
+ 'configurations':{'global' : {}}
}
actionQueue.put(command)
@@ -121,4 +122,4 @@ class FakeExecutor():
"exitcode": 0,
"stdout": "returned stdout",
"stderr": "returned stderr"
- }
\ No newline at end of file
+ }
Modified: incubator/ambari/trunk/ambari-agent/src/test/python/TestHeartbeat.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/test/python/TestHeartbeat.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/test/python/TestHeartbeat.py
(original)
+++ incubator/ambari/trunk/ambari-agent/src/test/python/TestHeartbeat.py Wed
Jan 23 21:05:54 2013
@@ -29,7 +29,6 @@ import time
class TestHeartbeat(TestCase):
-
def test_build(self):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
heartbeat = Heartbeat(actionQueue)
@@ -55,7 +54,8 @@ class TestHeartbeat(TestCase):
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "",
- "componentName" : "DATANODE"
+ "componentName" : "DATANODE",
+ 'configurations':{'global' : {}}
}
actionQueue.put(statusCommand)
actionQueue.start()
@@ -76,7 +76,8 @@ class TestHeartbeat(TestCase):
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "",
- "componentName" : "DATANODE"
+ "componentName" : "DATANODE",
+ 'configurations':{'global' : {}}
}
actionQueue.put(statusCommand)
time.sleep(0.1)
@@ -102,7 +103,8 @@ class TestHeartbeat(TestCase):
'stderr' : 'none',
'exitCode' : 777,
'serviceName' : "serviceName",
- 'status' : 'IN_PROGRESS'
+ 'status' : 'IN_PROGRESS',
+ 'configurations':{'global' : {}}
}
heartbeat = Heartbeat(actionQueue)
result = heartbeat.build(100)
Modified: incubator/ambari/trunk/ambari-agent/src/test/python/TestLiveStatus.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/test/python/TestLiveStatus.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/test/python/TestLiveStatus.py
(original)
+++ incubator/ambari/trunk/ambari-agent/src/test/python/TestLiveStatus.py Wed
Jan 23 21:05:54 2013
@@ -27,7 +27,7 @@ import os
class TestLiveStatus(TestCase):
def test_build(self):
for component in LiveStatus.COMPONENTS:
- livestatus = LiveStatus('', component['serviceName'],
component['componentName'])
+ livestatus = LiveStatus('', component['serviceName'],
component['componentName'], {})
result = livestatus.build()
print "LiveStatus of {0}: {1}".format(component['serviceName'],
str(result))
self.assertEquals(len(result) > 0, True, 'Livestatus should not be
empty')
Modified: incubator/ambari/trunk/ambari-agent/src/test/python/TestStatusCheck.py
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/test/python/TestStatusCheck.py?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/test/python/TestStatusCheck.py
(original)
+++ incubator/ambari/trunk/ambari-agent/src/test/python/TestStatusCheck.py Wed
Jan 23 21:05:54 2013
@@ -48,8 +48,15 @@ class TestStatusCheck(TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
- self.tmpdict = { COMPONENT_LIVE : COMPONENT_LIVE_PID,
- COMPONENT_DEAD : COMPONENT_DEAD_PID}
+ self.serviceToPidDict = {
+ COMPONENT_LIVE : COMPONENT_LIVE_PID,
+ COMPONENT_DEAD : COMPONENT_DEAD_PID
+ }
+
+ self.pidPathesVars = [
+ {'var' : '',
+ 'defaultValue' : self.tmpdir}
+ ]
self.sh = shellRunner()
@@ -70,11 +77,7 @@ class TestStatusCheck(TestCase):
dead_pid_file.close()
#Init status checker
- self.statusCheck = StatusCheck(self.tmpdir, self.tmpdict)
-
- # Ensure that status checker throws exceptions on invalid params
- def test_exceptions(self):
- self.assertRaises(ValueError,StatusCheck,"tmp","tmp")
+ self.statusCheck = StatusCheck(self.serviceToPidDict,self.pidPathesVars,{})
# Ensure that status checker return True for running process
def test_live(self):
Modified:
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
(original)
+++
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
Wed Jan 23 21:05:54 2013
@@ -18,7 +18,10 @@
package org.apache.ambari.server.agent;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.ActionManager;
@@ -145,10 +148,23 @@ public class HeartbeatMonitor implements
LOG.debug("Live status will include status of service " +
serviceName +
" of cluster " + cl.getClusterName());
}
+
+ Map<String, Config> configs = sch.getDesiredConfigs();
+
+ Map<String, Map<String, String>> configurations =
+ new TreeMap<String, Map<String, String>>();
+
+ for (Config config : configs.values()) {
+ if (config.getType().equals("global"))
+ configurations.put(config.getType(),
+ config.getProperties());
+ }
+
StatusCommand statusCmd = new StatusCommand();
statusCmd.setClusterName(cl.getClusterName());
statusCmd.setServiceName(serviceName);
statusCmd.setComponentName(sch.getServiceComponentName());
+ statusCmd.setConfigurations(configurations);
cmds.add(statusCmd);
}
}
Modified:
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
URL:
http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java?rev=1437734&r1=1437733&r2=1437734&view=diff
==============================================================================
---
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
(original)
+++
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
Wed Jan 23 21:05:54 2013
@@ -18,6 +18,7 @@
package org.apache.ambari.server.agent;
import java.util.List;
+import java.util.Map;
import org.codehaus.jackson.annotate.JsonProperty;
@@ -33,6 +34,7 @@ public class StatusCommand extends Agent
private String clusterName;
private String serviceName;
private String componentName;
+ private Map<String, Map<String, String>> configurations;
@JsonProperty("clusterName")
public String getClusterName() {
@@ -63,4 +65,14 @@ public class StatusCommand extends Agent
public void setComponentName(String componentName) {
this.componentName = componentName;
}
+
+ @JsonProperty("configurations")
+ public Map<String, Map<String, String>> getConfigurations() {
+ return configurations;
+ }
+
+ @JsonProperty("configurations")
+ public void setConfigurations(Map<String, Map<String, String>>
configurations) {
+ this.configurations = configurations;
+ }
}
