AMBARI-18733. Perf: Allow running multiple Ambari Agents on the same host (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4e96cf51 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4e96cf51 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4e96cf51 Branch: refs/heads/branch-feature-AMBARI-18634 Commit: 4e96cf51c5a796c7efc8415c7106ca5d071678a2 Parents: fca6f1c Author: Alejandro Fernandez <[email protected]> Authored: Wed Nov 2 14:59:39 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Wed Nov 2 14:59:39 2016 -0700 ---------------------------------------------------------------------- ambari-agent/conf/unix/ambari-agent | 12 +++++- ambari-agent/conf/windows/service_wrapper.py | 1 + .../main/python/ambari_agent/AmbariConfig.py | 44 +++++++++++++++----- .../src/main/python/ambari_agent/Facter.py | 21 +++++++--- .../src/main/python/ambari_agent/Hardware.py | 8 ++-- .../src/main/python/ambari_agent/Heartbeat.py | 5 ++- .../src/main/python/ambari_agent/Register.py | 2 +- .../src/main/python/ambari_agent/main.py | 28 +++++++++++-- .../test/python/ambari_agent/TestCheckWebUI.py | 2 + .../test/python/ambari_agent/TestHardware.py | 30 +++++++------ .../src/test/python/ambari_agent/TestMain.py | 4 +- 11 files changed, 116 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/conf/unix/ambari-agent ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/unix/ambari-agent b/ambari-agent/conf/unix/ambari-agent index 30897dd..0f791ca 100755 --- a/ambari-agent/conf/unix/ambari-agent +++ b/ambari-agent/conf/unix/ambari-agent @@ -33,9 +33,17 @@ case "${1:-}" in ;; esac +export HOME_DIR="" +if [ "$#" = 3 ] && [ $2 = "--home" ] ; then + export HOME_DIR=$3 + echo "Allow running multiple agents on this host; will use custom Home Dir: $HOME_DIR" +fi + +export CONFIG_FILE="$HOME_DIR/etc/ambari-agent/conf/ambari-agent.ini" + get_agent_property() { property_name="$1" - value=$(awk -F "=" "/^$property_name/ {print \$2}" /etc/ambari-agent/conf/ambari-agent.ini) + value=$(awk -F "=" "/^$property_name/ {print \$2}" $CONFIG_FILE) echo $value } @@ -51,7 +59,7 @@ valid_path() { export PATH=/usr/sbin:/sbin:/usr/lib/ambari-server/*:$PATH -export AMBARI_CONF_DIR=/etc/ambari-server/conf:$PATH +export AMBARI_CONF_DIR=$HOME_DIR/etc/ambari-server/conf:$PATH # Because Ambari rpm unpacks modules here on all systems export PYTHONPATH=/usr/lib/python2.6/site-packages:${PYTHONPATH:-} http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/conf/windows/service_wrapper.py ---------------------------------------------------------------------- diff --git a/ambari-agent/conf/windows/service_wrapper.py b/ambari-agent/conf/windows/service_wrapper.py index d031c34..ac9e22d 100644 --- a/ambari-agent/conf/windows/service_wrapper.py +++ b/ambari-agent/conf/windows/service_wrapper.py @@ -124,6 +124,7 @@ def ctrlHandler(ctrlType): # def setup(options): config = AmbariConfig() + # TODO AMBARI-18733, need to read home_dir to get correct config file. configFile = config.getConfigFile() updateConfigServerHostname(configFile, options.host_name) http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py index 78b34e5..6e4d74a 100644 --- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py +++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py @@ -191,7 +191,7 @@ class AmbariConfig: @staticmethod @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) - def getConfigFile(): + def getConfigFile(home_dir=""): if 'AMBARI_AGENT_CONF_DIR' in os.environ: return os.path.join(os.environ['AMBARI_AGENT_CONF_DIR'], "ambari-agent.ini") else: @@ -199,32 +199,56 @@ class AmbariConfig: @staticmethod @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) - def getConfigFile(): + def getConfigFile(home_dir=""): + """ + Get the configuration file path. + :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path. + :return: Configuration file path. + """ if 'AMBARI_AGENT_CONF_DIR' in os.environ: return os.path.join(os.environ['AMBARI_AGENT_CONF_DIR'], "ambari-agent.ini") else: - return os.path.join(os.sep, "etc", "ambari-agent", "conf", "ambari-agent.ini") + # home_dir may be an empty string + return os.path.join(os.sep, home_dir, "etc", "ambari-agent", "conf", "ambari-agent.ini") + # TODO AMBARI-18733, change usages of this function to provide the home_dir. @staticmethod - def getLogFile(): + def getLogFile(home_dir=""): + """ + Get the log file path. + :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path. + :return: Log file path. + """ if 'AMBARI_AGENT_LOG_DIR' in os.environ: return os.path.join(os.environ['AMBARI_AGENT_LOG_DIR'], "ambari-agent.log") else: - return os.path.join(os.sep, "var", "log", "ambari-agent", "ambari-agent.log") - + return os.path.join(os.sep, home_dir, "var", "log", "ambari-agent", "ambari-agent.log") + + # TODO AMBARI-18733, change usages of this function to provide the home_dir. @staticmethod - def getAlertsLogFile(): + def getAlertsLogFile(home_dir=""): + """ + Get the alerts log file path. + :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path. + :return: Alerts log file path. + """ if 'AMBARI_AGENT_LOG_DIR' in os.environ: return os.path.join(os.environ['AMBARI_AGENT_LOG_DIR'], "ambari-agent.log") else: - return os.path.join(os.sep, "var", "log", "ambari-agent", "ambari-alerts.log") + return os.path.join(os.sep, home_dir, "var", "log", "ambari-agent", "ambari-alerts.log") + # TODO AMBARI-18733, change usages of this function to provide the home_dir. @staticmethod - def getOutFile(): + def getOutFile(home_dir=""): + """ + Get the out file path. + :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path. + :return: Out file path. + """ if 'AMBARI_AGENT_LOG_DIR' in os.environ: return os.path.join(os.environ['AMBARI_AGENT_LOG_DIR'], "ambari-agent.out") else: - return os.path.join(os.sep, "var", "log", "ambari-agent", "ambari-agent.out") + return os.path.join(os.sep, home_dir, "var", "log", "ambari-agent", "ambari-agent.out") def has_option(self, section, option): return self.config.has_option(section, option) http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Facter.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Facter.py b/ambari-agent/src/main/python/ambari_agent/Facter.py index a1f815d..3643ff7 100644 --- a/ambari-agent/src/main/python/ambari_agent/Facter.py +++ b/ambari-agent/src/main/python/ambari_agent/Facter.py @@ -53,10 +53,18 @@ def run_os_command(cmd): class Facter(object): - def __init__(self): - self.config = self.resolve_ambari_config() - + def __init__(self, config): + """ + Initialize the configs, which can be provided if using multiple Agents per host. + :param config: Agent configs. None if will use the default location. + """ + self.config = config if config is not None else self.resolve_ambari_config() + def resolve_ambari_config(self): + """ + Resolve the default Ambari Agent configs. + :return: The default configs. + """ try: config = AmbariConfig() if os.path.exists(AmbariConfig.getConfigFile()): @@ -370,8 +378,8 @@ class FacterLinux(Facter): GET_UPTIME_CMD = "cat /proc/uptime" GET_MEMINFO_CMD = "cat /proc/meminfo" - def __init__(self): - super(FacterLinux,self).__init__() + def __init__(self, config): + super(FacterLinux,self).__init__(config) self.DATA_IFCONFIG_SHORT_OUTPUT = FacterLinux.setDataIfConfigShortOutput() self.DATA_UPTIME_OUTPUT = FacterLinux.setDataUpTimeOutput() self.DATA_MEMINFO_OUTPUT = FacterLinux.setMemInfoOutput() @@ -547,7 +555,8 @@ class FacterLinux(Facter): def main(argv=None): - print Facter().facterInfo() + config = None + print Facter(config).facterInfo() if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Hardware.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Hardware.py b/ambari-agent/src/main/python/ambari_agent/Hardware.py index a65c1cd..3c94d28 100644 --- a/ambari-agent/src/main/python/ambari_agent/Hardware.py +++ b/ambari-agent/src/main/python/ambari_agent/Hardware.py @@ -42,11 +42,12 @@ class Hardware: IGNORE_ROOT_MOUNTS = ["proc", "dev", "sys"] IGNORE_DEVICES = ["proc", "tmpfs", "cgroup", "mqueue", "shm"] - def __init__(self): + def __init__(self, config): self.hardware = { 'mounts': Hardware.osdisks() } - self.hardware.update(Facter().facterInfo()) + self.config = config + self.hardware.update(Facter(self.config).facterInfo()) @classmethod def _parse_df_line(cls, line): @@ -169,7 +170,8 @@ def main(): from resource_management.core.logger import Logger Logger.initialize_logger() - print Hardware().get() + config = None + print Hardware(config).get() if __name__ == '__main__': main() http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Heartbeat.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py index 82ea9b6..1e05aae 100644 --- a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py +++ b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py @@ -111,8 +111,9 @@ def main(argv=None): from ambari_agent.Controller import Controller cfg = AmbariConfig() - if os.path.exists(AmbariConfig.getConfigFile()): - cfg.read(AmbariConfig.getConfigFile()) + config_file_path = AmbariConfig.getConfigFile(home_dir="") + if os.path.exists(config_file_path): + cfg.read(config_file_path) else: raise Exception("No config found, use default") http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Register.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/Register.py b/ambari-agent/src/main/python/ambari_agent/Register.py index 2d85b03..0c811c6 100644 --- a/ambari-agent/src/main/python/ambari_agent/Register.py +++ b/ambari-agent/src/main/python/ambari_agent/Register.py @@ -31,8 +31,8 @@ class Register: """ Registering with the server. Get the hardware profile and declare success for now """ def __init__(self, config): - self.hardware = Hardware() self.config = config + self.hardware = Hardware(self.config) def build(self, version, id='-1'): global clusterId, clusterDefinitionRevision, firstContact http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/main.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/main.py b/ambari-agent/src/main/python/ambari_agent/main.py index 8cbb839..04e0c2c 100644 --- a/ambari-agent/src/main/python/ambari_agent/main.py +++ b/ambari-agent/src/main/python/ambari_agent/main.py @@ -72,7 +72,12 @@ alerts_logger = logging.getLogger('ambari_alerts') formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s" agentPid = os.getpid() + +# Global variables to be set later. +home_dir = "" + config = AmbariConfig.AmbariConfig() +# TODO AMBARI-18733, remove this global variable and calculate it based on home_dir once it is set. configFile = config.getConfigFile() two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY @@ -110,7 +115,8 @@ def add_syslog_handler(logger): def update_log_level(config): # Setting loglevel based on config file global logger - log_cfg_file = os.path.join(os.path.dirname(AmbariConfig.AmbariConfig.getConfigFile()), "logging.conf") + global home_dir + log_cfg_file = os.path.join(os.path.dirname(AmbariConfig.AmbariConfig.getConfigFile(home_dir)), "logging.conf") if os.path.exists(log_cfg_file): logging.config.fileConfig(log_cfg_file) # create logger @@ -132,11 +138,15 @@ def update_log_level(config): logger.info("Default loglevel=DEBUG") -# ToDo: move that function inside AmbariConfig +# TODO AMBARI-18733, move inside AmbariConfig def resolve_ambari_config(): + """ + Load the configurations. + In production, home_dir will be "". When running multiple Agents per host, each agent will have a unique path. + """ global config - configPath = os.path.abspath(AmbariConfig.AmbariConfig.getConfigFile()) - + global home_dir + configPath = os.path.abspath(AmbariConfig.AmbariConfig.getConfigFile(home_dir)) try: if os.path.exists(configPath): config.read(configPath) @@ -254,9 +264,11 @@ def stop_agent(): sys.exit(0) def reset_agent(options): + global home_dir try: # update agent config file agent_config = ConfigParser.ConfigParser() + # TODO AMBARI-18733, calculate configFile based on home_dir agent_config.read(configFile) server_host = agent_config.get('server', 'hostname') new_host = options[2] @@ -302,13 +314,17 @@ def run_threads(server_hostname, heartbeat_stop_callback): # we need this for windows os, where no sigterm available def main(heartbeat_stop_callback=None): global config + global home_dir + parser = OptionParser() parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False) parser.add_option("-e", "--expected-hostname", dest="expected_hostname", action="store", help="expected hostname of current host. If hostname differs, agent will fail", default=None) + parser.add_option("--home", dest="home_dir", action="store", help="Home directory", default="") (options, args) = parser.parse_args() expected_hostname = options.expected_hostname + home_dir = options.home_dir logging_level = logging.DEBUG if options.verbose else logging.INFO @@ -318,6 +334,10 @@ def main(heartbeat_stop_callback=None): setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level) Logger.initialize_logger('resource_management', logging_level=logging_level) + if home_dir != "": + # When running multiple Ambari Agents on this host for simulation, each one will use a unique home directory. + Logger.info("Agent is using Home Dir: %s" % str(home_dir)) + # use the host's locale for numeric formatting try: locale.setlocale(locale.LC_ALL, '') http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py b/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py index 4980477..0cbc90e 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py +++ b/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py @@ -24,6 +24,8 @@ import sys from mock.mock import MagicMock, patch +# Needed to import checkWebUI.py +sys.path.append("../../../../ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files") import checkWebUI class TestMain(unittest.TestCase): http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/test/python/ambari_agent/TestHardware.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py index 5c014db..038b2f8 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py @@ -52,7 +52,8 @@ class TestHardware(TestCase): def test_build(self, get_os_version_mock, get_os_type_mock): get_os_type_mock.return_value = "suse" get_os_version_mock.return_value = "11" - hardware = Hardware() + config = None + hardware = Hardware(config) result = hardware.get() osdisks = hardware.osdisks() for dev_item in result['mounts']: @@ -196,7 +197,8 @@ class TestHardware(TestCase): hostname_mock.return_value = 'ambari' get_os_type_mock.return_value = "suse" get_os_version_mock.return_value = "11" - result = Facter().facterInfo() + config = None + result = Facter(config).facterInfo() self.assertEquals(result['hostname'], "ambari") self.assertEquals(result['domain'], "apache.org") @@ -211,7 +213,8 @@ class TestHardware(TestCase): facter_setDataUpTimeOutput_mock.return_value = "262813.00 123.45" get_os_type_mock.return_value = "suse" get_os_version_mock.return_value = "11" - result = Facter().facterInfo() + config = None + result = Facter(config).facterInfo() self.assertEquals(result['uptime_seconds'], '262813') self.assertEquals(result['uptime_hours'], '73') @@ -236,7 +239,8 @@ SwapFree: 1598676 kB get_os_type_mock.return_value = "suse" get_os_version_mock.return_value = "11" - result = Facter().facterInfo() + config = None + result = Facter(config).facterInfo() self.assertEquals(result['memorysize'], 1832392) self.assertEquals(result['memorytotal'], 1832392) @@ -261,7 +265,8 @@ SwapFree: 1598676 kB get_os_type_mock.return_value = "suse" get_os_version_mock.return_value = "11" - result = Facter().facterInfo() + config = None + result = Facter(config).facterInfo() self.assertTrue(inet_ntoa_mock.called) self.assertTrue(get_ip_address_by_ifname_mock.called) @@ -287,7 +292,8 @@ SwapFree: 1598676 kB get_os_type_mock.return_value = "suse" get_os_version_mock.return_value = "11" - result = Facter().facterInfo() + config = None + result = Facter(config).facterInfo() self.assertTrue(get_ip_address_by_ifname_mock.called) self.assertEquals(result['netmask'], None) @@ -301,22 +307,23 @@ SwapFree: 1598676 kB get_os_version_mock.return_value = "11" get_os_family_mock.return_value = "redhat" - result = Facter().facterInfo() + config = None + result = Facter(config).facterInfo() self.assertEquals(result['operatingsystem'], 'some_type_of_os') self.assertEquals(result['osfamily'], 'redhat') get_os_family_mock.return_value = "ubuntu" - result = Facter().facterInfo() + result = Facter(config).facterInfo() self.assertEquals(result['operatingsystem'], 'some_type_of_os') self.assertEquals(result['osfamily'], 'ubuntu') get_os_family_mock.return_value = "suse" - result = Facter().facterInfo() + result = Facter(config).facterInfo() self.assertEquals(result['operatingsystem'], 'some_type_of_os') self.assertEquals(result['osfamily'], 'suse') get_os_family_mock.return_value = "My_new_family" - result = Facter().facterInfo() + result = Facter(config).facterInfo() self.assertEquals(result['operatingsystem'], 'some_type_of_os') self.assertEquals(result['osfamily'], 'My_new_family') @@ -352,8 +359,7 @@ SwapFree: 1598676 kB json_data.items.return_value = [('key', 'value')] json_data.__getitem__.return_value = 'value' - facter = Facter() - facter.config = config + facter = Facter(config) result = facter.getSystemResourceOverrides() isdir.assert_called_with('/etc/custom_resource_overrides') http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/test/python/ambari_agent/TestMain.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestMain.py b/ambari-agent/src/test/python/ambari_agent/TestMain.py index ffb5d83..6f38410 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestMain.py +++ b/ambari-agent/src/test/python/ambari_agent/TestMain.py @@ -96,10 +96,12 @@ class TestMain(unittest.TestCase): setLevel_mock.assert_called_with(logging.DEBUG) + @patch("os.path.exists") @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) @patch.object(main.logger, "setLevel") @patch("logging.basicConfig") - def test_update_log_level(self, basicConfig_mock, setLevel_mock): + def test_update_log_level(self, basicConfig_mock, setLevel_mock, os_path_exists_mock): + os_path_exists_mock.return_value = False config = AmbariConfig().getConfig() # Testing with default setup (config file does not contain loglevel entry)
