Updated Branches: refs/heads/trunk 6028540df -> 4c2e398e0
AMBARI-3047. Enhance host clean up to handle tmp files and folders. Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/4c2e398e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/4c2e398e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/4c2e398e Branch: refs/heads/trunk Commit: 4c2e398e0b2123048c7f8c3af9b49e82aec1562c Parents: 6028540 Author: Sumit Mohanty <[email protected]> Authored: Wed Aug 28 15:01:02 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Aug 28 15:01:02 2013 -0700 ---------------------------------------------------------------------- .../ambari_agent/HostCheckReportFileHandler.py | 3 ++ .../src/main/python/ambari_agent/HostCleanup.py | 48 +++++++++++++++++++- .../src/main/python/ambari_agent/HostInfo.py | 20 +------- .../python/TestHostCheckReportFileHandler.py | 3 ++ ambari-agent/src/test/python/TestHostCleanup.py | 38 ++++++++++++++-- ambari-agent/src/test/python/TestHostInfo.py | 12 ----- 6 files changed, 89 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py index be0de89..d46ab64 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py @@ -47,10 +47,13 @@ class HostCheckReportFileHandler: if 'existingUsers' in hostInfo.keys(): items = [] + items2 = [] for itemDetail in hostInfo['existingUsers']: items.append(itemDetail['name']) + items2.append(itemDetail['homeDir']) config.add_section('users') config.set('users', 'usr_list', ','.join(items)) + config.set('users', 'usr_homedir_list', ','.join(items2)) if 'alternatives' in hostInfo.keys(): items = [] http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/main/python/ambari_agent/HostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py index 409a115..7b63707 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py @@ -30,6 +30,7 @@ import shlex import sys import datetime import AmbariConfig +from pwd import getpwnam logger = logging.getLogger() configFile = "/etc/ambari-agent/conf/ambari-agent.ini" @@ -37,6 +38,7 @@ configFile = "/etc/ambari-agent/conf/ambari-agent.ini" PACKAGE_ERASE_CMD_RHEL = "yum erase -y {0}" PACKAGE_ERASE_CMD_SUSE = "zypper -n -q remove {0}" USER_ERASE_CMD = "userdel -rf {0}" +GROUP_ERASE_CMD = "groupdel {0}" PROC_KILL_CMD = "kill -9 {0}" ALT_DISP_CMD = "alternatives --display {0}" ALT_ERASE_CMD = "alternatives --remove {0} {1}" @@ -51,6 +53,8 @@ PACKAGE_SECTION = "packages" PACKAGE_KEY = "pkg_list" USER_SECTION = "users" USER_KEY = "usr_list" +USER_HOMEDIR_KEY = "usr_homedir_list" +USER_HOMEDIR_SECTION = "usr_homedir" REPO_SECTION = "repositories" REPOS_KEY = "repo_list" DIR_SECTION = "directories" @@ -59,6 +63,8 @@ PROCESS_SECTION = "processes" PROCESS_KEY = "proc_list" ALT_SECTION = "alternatives" ALT_KEYS = ["symlink_list", "target_list"] +HADOOP_GROUP = "hadoop" +FOLDER_LIST = ["/tmp"] # resources that should not be cleaned REPOSITORY_BLACK_LIST = ["ambari.repo"] @@ -83,11 +89,14 @@ class HostCleanup: if argMap: packageList = argMap.get(PACKAGE_SECTION) userList = argMap.get(USER_SECTION) + homeDirList = argMap.get(USER_HOMEDIR_SECTION) dirList = argMap.get(DIR_SECTION) repoList = argMap.get(REPO_SECTION) procList = argMap.get(PROCESS_SECTION) alt_map = argMap.get(ALT_SECTION) + if userList and not USER_SECTION in SKIP_LIST: + userIds = self.get_user_ids(userList) if procList and not PROCESS_SECTION in SKIP_LIST: logger.info("\n" + "Killing pid's: " + str(procList) + "\n") self.do_kill_processes(procList) @@ -97,6 +106,8 @@ class HostCleanup: if userList and not USER_SECTION in SKIP_LIST: logger.info("\n" + "Deleting users: " + str(userList)) self.do_delete_users(userList) + self.do_erase_dir_silent(homeDirList) + self.do_delete_by_owner(userIds, FOLDER_LIST) if dirList and not DIR_SECTION in SKIP_LIST: logger.info("\n" + "Deleting directories: " + str(dirList)) self.do_erase_dir_silent(dirList) @@ -139,6 +150,12 @@ class HostCleanup: logger.warn("Cannot read user list: " + str(sys.exc_info()[0])) try: + if config.has_option(USER_SECTION, USER_HOMEDIR_KEY): + propertyMap[USER_HOMEDIR_SECTION] = config.get(USER_SECTION, USER_HOMEDIR_KEY).split(',') + except: + logger.warn("Cannot read user homedir list: " + str(sys.exc_info()[0])) + + try: if config.has_option(REPO_SECTION, REPOS_KEY): propertyMap[REPO_SECTION] = config.get(REPO_SECTION, REPOS_KEY).split(',') except: @@ -327,6 +344,34 @@ class HostCleanup: logger.info("File doesn't exists: " + path) return 0 + def do_delete_group(self): + groupDelCommand = GROUP_ERASE_CMD.format(HADOOP_GROUP) + (returncode, stdoutdata, stderrdata) = self.run_os_command(groupDelCommand) + if returncode != 0: + logger.warn("Cannot delete group : " + HADOOP_GROUP + ", " + stderrdata) + else: + logger.info("Successfully deleted group: " + HADOOP_GROUP) + + def do_delete_by_owner(self, userIds, folders): + for folder in folders: + for filename in os.listdir(folder): + fileToCheck = os.path.join(folder, filename) + stat = os.stat(fileToCheck) + if stat.st_uid in userIds: + self.do_erase_dir_silent([fileToCheck]) + logger.info("Deleting file/folder: " + fileToCheck) + + def get_user_ids(self, userList): + userIds = [] + if userList: + for user in userList: + if user: + try: + userIds.append(getpwnam(user).pw_uid) + except Exception: + logger.warn("Cannot find user : " + user) + return userIds + def do_delete_users(self, userList): if userList: for user in userList: @@ -337,6 +382,7 @@ class HostCleanup: logger.warn("Cannot delete user : " + user + ", " + stderrdata) else: logger.info("Successfully deleted user: " + user) + self.do_delete_group() return 0 def is_current_user_root(self): @@ -401,7 +447,7 @@ def main(): help="log file to store results.", metavar="FILE") parser.add_option("-k", "--skip", dest="skip", help="(packages|users|directories|repositories|processes|alternatives)." +\ - " separator = ,") + " Use , as separator. Use empty string to clean all (by default users are skipped)") (options, args) = parser.parse_args() # set output file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/main/python/ambari_agent/HostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py index 4805717..00b77ff 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py @@ -41,7 +41,8 @@ class HostInfo: "hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios", "oozie", "sqoop", "hue", "zookeeper", "mapred", "hdfs", "flume", "ambari_qa", "hadoop_deploy", "rrdcached", "hcat", "ambari-qa", - "sqoop-ambari-qa", "sqoop-ambari_qa" + "sqoop-ambari-qa", "sqoop-ambari_qa", "webhcat", "hadoop-hdfs", "hadoop-yarn", + "hadoop-mapreduce" ] # List of live services checked for on the host, takes a map of plan strings @@ -219,22 +220,6 @@ class HostInfo: except: pass - def checkFoldersBasedOnNames(self, basePaths, projectNames, existingUsers, dirs): - foldersToIgnore = [] - for user in existingUsers: - foldersToIgnore.append(user['homeDir']) - try: - for dirName in basePaths: - for project in projectNames: - path = dirName.strip() + project.strip() - if not path in foldersToIgnore and os.path.exists(path): - obj = {} - obj['type'] = self.dirType(path) - obj['name'] = path - dirs.append(obj) - except: - pass - def javaProcs(self, list): try: pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] @@ -321,7 +306,6 @@ class HostInfo: dirs = [] self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs) - self.checkFoldersBasedOnNames(self.DIRNAME_PATTERNS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs) dict['stackFoldersAndFiles'] = dirs installedPackages = [] http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py b/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py index 6ace6e4..1256717 100644 --- a/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py +++ b/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py @@ -71,6 +71,7 @@ class TestHostCheckReportFileHandler(TestCase): configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE) configValidator.read(configPath) users = configValidator.get('users', 'usr_list') + users = configValidator.get('users', 'usr_homedir_list') self.assertEquals(users, '') names = configValidator.get('alternatives', 'symlink_list') targets = configValidator.get('alternatives', 'target_list') @@ -124,7 +125,9 @@ class TestHostCheckReportFileHandler(TestCase): configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE) configValidator.read(configPath) users = configValidator.get('users', 'usr_list') + homedirs = configValidator.get('users', 'usr_homedir_list') self.assertEquals(users, 'user1') + self.assertEquals(homedirs, '/var/log') names = configValidator.get('alternatives', 'symlink_list') targets = configValidator.get('alternatives', 'target_list') http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/test/python/TestHostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/TestHostCleanup.py b/ambari-agent/src/test/python/TestHostCleanup.py index 0bb513a..a3928be 100644 --- a/ambari-agent/src/test/python/TestHostCleanup.py +++ b/ambari-agent/src/test/python/TestHostCleanup.py @@ -42,6 +42,7 @@ PROCESS_KEY = "proc_list" ALT_SECTION = "alternatives" ALT_KEYS = ["symlink_list", "target_list"] ALT_ERASE_CMD = "alternatives --remove {0} {1}" +USER_HOMEDIR_SECTION = "usr_homedir" class TestHostCleanup(TestCase): @@ -148,7 +149,7 @@ created = 2013-07-02 20:39:22.162757""" REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'], PROCESS_SECTION:['abcd', 'pqrst'], ALT_SECTION:{ALT_KEYS[0]:['alt1','alt2'], ALT_KEYS[1]:[ - 'dir1']}} + 'dir1']}, USER_HOMEDIR_SECTION:['decf']} get_os_type_method.return_value = 'redhat' find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst'] @@ -160,7 +161,8 @@ created = 2013-07-02 20:39:22.162757""" self.assertTrue(do_erase_packages_method.called) self.assertTrue(do_kill_processes_method.called) self.assertTrue(do_erase_alternatives_method.called) - do_erase_dir_silent_method.assert_called_once_with(['abcd', 'pqrst']) + calls = [call(['decf']), call(['abcd', 'pqrst'])] + do_erase_dir_silent_method.assert_has_calls(calls) do_erase_packages_method.assert_called_once_with(['abcd', 'pqrst']) do_erase_files_silent_method.assert_called_once_with(['abcd', 'pqrst']) do_delete_users_method.assert_called_once_with(['abcd', 'pqrst']) @@ -172,6 +174,8 @@ created = 2013-07-02 20:39:22.162757""" sys.stdout = sys.__stdout__ + @patch.object(HostCleanup.HostCleanup, 'do_delete_by_owner') + @patch.object(HostCleanup.HostCleanup, 'get_user_ids') @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives') @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') @patch.object(HostCleanup.HostCleanup, 'get_os_type') @@ -184,7 +188,8 @@ created = 2013-07-02 20:39:22.162757""" do_erase_dir_silent_method, do_erase_files_silent_method, do_kill_processes_method, get_os_type_method, find_repo_files_for_repos_method, - do_erase_alternatives_method): + do_erase_alternatives_method, get_user_ids_method, + do_delete_by_owner_method): out = StringIO.StringIO() sys.stdout = out @@ -198,6 +203,8 @@ created = 2013-07-02 20:39:22.162757""" self.hostcleanup.do_cleanup(propertyMap) + self.assertFalse(do_delete_by_owner_method.called) + self.assertFalse(get_user_ids_method.called) self.assertFalse(do_delete_users_method.called) self.assertTrue(do_erase_dir_silent_method.called) self.assertTrue(do_erase_files_silent_method.called) @@ -235,12 +242,35 @@ created = 2013-07-02 20:39:22.162757""" self.assertFalse(do_erase_files_silent_method.called) self.assertFalse(do_erase_packages_method.called) self.assertTrue(do_kill_processes_method.called) - do_erase_dir_silent_method.assert_called_once_with(['abcd', 'pqrst']) + calls = [call(None), call(['abcd', 'pqrst'])] + do_erase_dir_silent_method.assert_has_calls(calls) do_delete_users_method.assert_called_once_with(['abcd', 'pqrst']) do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst']) sys.stdout = sys.__stdout__ + @patch.object(HostCleanup.HostCleanup, 'do_erase_dir_silent') + @patch("os.stat") + @patch("os.path.join") + @patch("os.listdir") + def test_do_delete_by_owner(self, listdir_mock, join_mock, stat_mock, do_erase_dir_silent_method): + listdir_mock.return_value = ["k", "j"] + join_mock.return_value = "path" + response = MagicMock() + response.st_uid = 1 + stat_mock.return_value = response + self.hostcleanup.do_delete_by_owner([1, 2], ["a"]) + self.assertTrue(do_erase_dir_silent_method.called) + calls = [call(["path"]), call(["path"])] + do_erase_dir_silent_method.assert_has_calls(calls) + + @patch.object(HostCleanup.HostCleanup, 'run_os_command') + def test_do_delete_users(self, run_os_command_mock): + run_os_command_mock.return_value = (1, "", "") + self.hostcleanup.do_delete_users(["a", "b"]) + self.assertTrue(run_os_command_mock.called) + calls = [call('userdel -rf a'), call('userdel -rf b'), call('groupdel hadoop')] + run_os_command_mock.assert_has_calls(calls) @patch("ConfigParser.RawConfigParser") @patch("__builtin__.open") http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/test/python/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/TestHostInfo.py b/ambari-agent/src/test/python/TestHostInfo.py index 778b9f3..3bda116 100644 --- a/ambari-agent/src/test/python/TestHostInfo.py +++ b/ambari-agent/src/test/python/TestHostInfo.py @@ -196,18 +196,6 @@ class TestHostInfo(TestCase): packageAnalyzer.allInstalledPackages(installedPackages) self.assertEqual(installedPackages, []) - @patch('os.path.exists') - def test_checkFoldersBasedOnNames(self, path_mock): - path_mock.return_value = True - hostInfo = HostInfo() - results = [] - existingUsers = [{'name':'a1', 'homeDir':'/home/a1'}, {'name':'b1', 'homeDir':'/home/b1'}] - hostInfo.checkFoldersBasedOnNames(["/etc/conf", "/var/lib_", "/home/"], ["a1", "b1"], existingUsers, results) - self.assertEqual(4, len(results)) - names = [i['name'] for i in results] - for item in ['/etc/confa1', '/var/lib_a1', '/etc/confb1', '/var/lib_b1']: - self.assertTrue(item in names) - @patch('os.path.exists') def test_checkFolders(self, path_mock):
