Repository: ambari Updated Branches: refs/heads/branch-1.7.0 f2e62d3a6 -> 0e0e51fba
AMBARI-7762. Confirm Hosts page is lagging.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e0e51fb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e0e51fb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e0e51fb Branch: refs/heads/branch-1.7.0 Commit: 0e0e51fba62710666a512dc33d81e50f5f8f1d1c Parents: f2e62d3 Author: Vitaly Brodetskyi <[email protected]> Authored: Mon Oct 13 19:52:31 2014 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Mon Oct 13 19:52:31 2014 +0300 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/HostCleanup.py | 51 +++++++++++++++++--- .../test/python/ambari_agent/TestHostCleanup.py | 35 +++++++++++--- 2 files changed, 73 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0e0e51fb/ambari-agent/src/main/python/ambari_agent/HostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py index 0840378..570c320 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py @@ -27,16 +27,16 @@ import subprocess import logging import shutil import platform +import fnmatch import ConfigParser import optparse import shlex import datetime -import AmbariConfig +from AmbariConfig import AmbariConfig from pwd import getpwnam from ambari_commons import OSCheck logger = logging.getLogger() -configFile = "/etc/ambari-agent/conf/ambari-agent.ini" PACKAGE_ERASE_CMD = { "redhat": "yum erase -y {0}", @@ -67,6 +67,9 @@ REPOS_KEY = "repo_list" DIR_SECTION = "directories" ADDITIONAL_DIRS = "additional_directories" DIR_KEY = "dir_list" +CACHE_FILES_PATTERN = { + 'alerts': ['*.json'] +} PROCESS_SECTION = "processes" PROCESS_KEY = "proc_list" ALT_SECTION = "alternatives" @@ -86,9 +89,9 @@ PACKAGES_BLACK_LIST = ["ambari-server", "ambari-agent"] class HostCleanup: def resolve_ambari_config(self): try: - config = AmbariConfig.AmbariConfig() - if os.path.exists(configFile): - config.read(configFile) + config = AmbariConfig() + if os.path.exists(AmbariConfig.CONFIG_FILE): + config.read(AmbariConfig.CONFIG_FILE) else: raise Exception("No config found, use default") @@ -229,6 +232,33 @@ class HostCleanup: return out + + def do_clear_cache(self, cache_root, dir_map=None): + """ + Clear cache dir according to provided root directory + + cache_root - root dir for cache directory + dir_map - should be used only for recursive calls + """ + global CACHE_FILES_PATTERN + file_map = CACHE_FILES_PATTERN if dir_map is None else dir_map + remList = [] + + # Build remove list according to masks + for folder in file_map: + if isinstance(file_map[folder], list): # here is list of file masks/files + for mask in file_map[folder]: + remList += self.get_files_in_dir("%s/%s" % (cache_root, folder), mask) + elif isinstance(file_map[folder], dict): # here described sub-folder + remList += self.do_clear_cache("%s/%s" % (cache_root, folder), file_map[folder]) + + if dir_map is not None: # push result list back as this is call from stack + return remList + else: # root call, so we have final list + self.do_erase_files_silent(remList) + + + # Alternatives exist as a stack of symlinks under /var/lib/alternatives/$name # Script expects names of the alternatives as input # We find all the symlinks using command, #] alternatives --display $name @@ -276,7 +306,7 @@ class HostCleanup: logger.error("Unable to kill process with pid: " + pid + ", " + stderrdata) return 0 - def get_files_in_dir(self, dirPath): + def get_files_in_dir(self, dirPath, filemask = None): fileList = [] if dirPath: if os.path.exists(dirPath): @@ -285,7 +315,11 @@ class HostCleanup: for link in listdir: path = dirPath + os.sep + link if not os.path.islink(path) and not os.path.isdir(path): - fileList.append(path) + if filemask is not None: + if fnmatch.fnmatch(path, filemask): + fileList.append(path) + else: + fileList.append(path) return fileList @@ -532,6 +566,9 @@ def main(): if propMap: h.do_cleanup(propMap) + if os.path.exists(config.get('agent', 'cache_dir')): + h.do_clear_cache(config.get('agent', 'cache_dir')) + logger.info('Clean-up completed. The output is at %s' % (str(options.outputfile))) http://git-wip-us.apache.org/repos/asf/ambari/blob/0e0e51fb/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py index cdd5c36..c958412 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py @@ -117,6 +117,7 @@ class TestHostCleanup(TestCase): self.silent = silent self.java_home = java_home + @patch.object(HostCleanup.HostCleanup, 'do_clear_cache') @patch.object(HostCleanup, 'get_YN_input') @patch.object(HostCleanup.HostCleanup, 'do_cleanup') @patch.object(HostCleanup.HostCleanup, 'is_current_user_root') @@ -126,7 +127,7 @@ class TestHostCleanup(TestCase): @patch.object(logging, 'FileHandler') @patch.object(optparse.OptionParser, 'parse_args') def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, - set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock): + set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock): parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False, False, 'java_home'), []) file_handler_mock.return_value = logging.FileHandler('') # disable creating real file @@ -144,7 +145,27 @@ class TestHostCleanup(TestCase): read_host_check_file_mock.assert_called_with('/someinputfile') self.assertTrue(get_yn_input_mock.called) + @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir') + @patch.object(HostCleanup.HostCleanup, 'do_erase_files_silent') + def test_clear_cache(self, erase_files_mock, get_files_mock): + old_data = HostCleanup.CACHE_FILES_PATTERN + + HostCleanup.CACHE_FILES_PATTERN = { + 'somedir': ['*.txt'] + } + + files_list = ['/tmp/somedir/test.txt'] + get_files_mock.return_value = files_list + + self.hostcleanup.do_clear_cache('/tmp') + + get_files_mock.assert_called_with('/tmp/somedir', '*.txt') + erase_files_mock.assert_called_with(files_list) + + HostCleanup.CACHE_FILES_PATTERN = old_data + + @patch.object(HostCleanup.HostCleanup, 'do_clear_cache') @patch.object(HostCleanup, 'get_YN_input') @patch.object(HostCleanup.HostCleanup, 'do_cleanup') @patch.object(HostCleanup.HostCleanup, 'is_current_user_root') @@ -154,7 +175,7 @@ class TestHostCleanup(TestCase): @patch.object(logging, 'FileHandler') @patch.object(optparse.OptionParser, 'parse_args') def test_options_silent(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, - set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock): + set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock): parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False, True, 'java_home'), []) file_handler_mock.return_value = logging.FileHandler('') # disable creating real file @@ -172,6 +193,7 @@ class TestHostCleanup(TestCase): read_host_check_file_mock.assert_called_with('/someinputfile') self.assertFalse(get_yn_input_mock.called) + @patch.object(HostCleanup.HostCleanup, 'do_clear_cache') @patch.object(HostCleanup.HostCleanup, 'get_additional_dirs') @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives') @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') @@ -185,7 +207,7 @@ class TestHostCleanup(TestCase): do_erase_dir_silent_method, do_erase_files_silent_method, do_kill_processes_method, get_os_type_method, find_repo_files_for_repos_method, - do_erase_alternatives_method, get_additional_dirs_method): + do_erase_alternatives_method, get_additional_dirs_method, clear_cache_mock): out = StringIO.StringIO() sys.stdout = out get_additional_dirs_method.return_value = ['/tmp/hadoop-nagios','/tmp/hsperfdata_007'] @@ -216,7 +238,7 @@ class TestHostCleanup(TestCase): sys.stdout = sys.__stdout__ - + @patch.object(HostCleanup.HostCleanup, 'do_clear_cache') @patch.object(HostCleanup.HostCleanup, 'do_delete_by_owner') @patch.object(HostCleanup.HostCleanup, 'get_user_ids') @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives') @@ -232,7 +254,7 @@ class TestHostCleanup(TestCase): do_erase_files_silent_method, do_kill_processes_method, get_os_type_method, find_repo_files_for_repos_method, do_erase_alternatives_method, get_user_ids_method, - do_delete_by_owner_method): + do_delete_by_owner_method, clear_cache_mock): global SKIP_LIST oldSkipList = HostCleanup.SKIP_LIST @@ -260,6 +282,7 @@ class TestHostCleanup(TestCase): HostCleanup.SKIP_LIST = oldSkipList sys.stdout = sys.__stdout__ + @patch.object(HostCleanup.HostCleanup, 'do_clear_cache') @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') @patch.object(OSCheck, "get_os_type") @patch.object(HostCleanup.HostCleanup, 'do_kill_processes') @@ -271,7 +294,7 @@ class TestHostCleanup(TestCase): do_delete_users_method, do_erase_dir_silent_method, do_erase_files_silent_method, do_kill_processes_method, - get_os_type_method, find_repo_files_for_repos_method): + get_os_type_method, find_repo_files_for_repos_method, clear_cache_mock): out = StringIO.StringIO() sys.stdout = out
