Updated Branches: refs/heads/trunk acfbf348a -> 76daacf0f
AMBARI-3189. Review host check filters for various OS types. Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/76daacf0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/76daacf0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/76daacf0 Branch: refs/heads/trunk Commit: 76daacf0f7de1734cb63c2b0a06bd7c70f7412ae Parents: acfbf34 Author: Sumit Mohanty <[email protected]> Authored: Wed Sep 11 23:14:35 2013 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Sep 11 23:14:35 2013 -0700 ---------------------------------------------------------------------- .../src/main/python/ambari_agent/HostCleanup.py | 66 ++++++++++++++------ .../src/main/python/ambari_agent/HostInfo.py | 20 ++++-- ambari-agent/src/test/python/TestHostCleanup.py | 17 ++--- ambari-agent/src/test/python/TestHostInfo.py | 48 +++++++++++++- .../apache/ambari/server/agent/AgentEnv.java | 50 --------------- 5 files changed, 118 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/76daacf0/ambari-agent/src/main/python/ambari_agent/HostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py index 7b63707..6d07691 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py @@ -45,7 +45,7 @@ ALT_ERASE_CMD = "alternatives --remove {0} {1}" REPO_PATH_RHEL = "/etc/yum.repos.d" REPO_PATH_SUSE = "/etc/zypp/repos.d/" -SKIP_LIST = ["users"] +SKIP_LIST = [] HOST_CHECK_FILE_NAME = "hostcheck.result" OUTPUT_FILE_NAME = "hostcleanup.result" @@ -70,8 +70,8 @@ FOLDER_LIST = ["/tmp"] REPOSITORY_BLACK_LIST = ["ambari.repo"] PACKAGES_BLACK_LIST = ["ambari-server", "ambari-agent"] -class HostCleanup: +class HostCleanup: def resolve_ambari_config(self): try: config = AmbariConfig.config @@ -124,7 +124,8 @@ class HostCleanup: def read_host_check_file(self, config_file_path): propertyMap = {} try: - with open(config_file_path, 'r'): pass + with open(config_file_path, 'r'): + pass except Exception, e: logger.error("Host check result not found at: " + str(config_file_path)) return None @@ -196,7 +197,7 @@ class HostCleanup: out = p2.communicate()[0] logger.debug('alternatives --display ' + alt_name + '\n, out = ' + out) except: - logger.warn('Cannot process alternative named: ' + alt_name + ',' +\ + logger.warn('Cannot process alternative named: ' + alt_name + ',' + \ 'error: ' + str(sys.exc_info()[0])) return out @@ -222,8 +223,8 @@ class HostCleanup: for entry in alternates: if entry: alt_path = entry.split()[0] - logger.debug('Erasing alternative named: ' + alt_name + ', '\ - 'path: ' + alt_path) + logger.debug('Erasing alternative named: ' + alt_name + ', ' \ + 'path: ' + alt_path) command = ALT_ERASE_CMD.format(alt_name, alt_path) (returncode, stdoutdata, stderrdata) = self.run_os_command(command) @@ -308,7 +309,7 @@ class HostCleanup: logger.warn("Unsupported OS type, cannot remove package.") if command != '': - logger.debug('Executing: '+ str(command)) + logger.debug('Executing: ' + str(command)) (returncode, stdoutdata, stderrdata) = self.run_os_command(command) if returncode != 0: logger.warn("Erasing packages failed: " + stderrdata) @@ -390,7 +391,7 @@ class HostCleanup: def get_os_type(self): os_info = platform.linux_distribution( - None, None, None, ['SuSE', 'redhat' ], 0 + None, None, None, ['SuSE', 'redhat'], 0 ) return os_info[0].lower() @@ -403,9 +404,9 @@ class HostCleanup: if type(cmd) == str: cmd = shlex.split(cmd) process = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE ) (stdoutdata, stderrdata) = process.communicate() return process.returncode, stdoutdata, stderrdata @@ -429,6 +430,27 @@ def backup_file(filePath): logger.warn('Could not backup file "%s": %s' % (str(filePath, e))) return 0 + +def get_YN_input(prompt, default): + yes = set(['yes', 'ye', 'y']) + no = set(['no', 'n']) + return get_choice_string_input(prompt, default, yes, no) + + +def get_choice_string_input(prompt, default, firstChoice, secondChoice): + choice = raw_input(prompt).lower() + if choice in firstChoice: + return True + elif choice in secondChoice: + return False + elif choice is "": # Just enter pressed + return default + else: + print "input not recognized, please try again: " + return get_choice_string_input(prompt, default, firstChoice, secondChoice) + pass + + def main(): h = HostCleanup() config = h.resolve_ambari_config() @@ -438,16 +460,16 @@ def main(): parser = optparse.OptionParser() parser.add_option("-v", "--verbose", dest="verbose", action="store_false", - default=False, help="output verbosity.") + default=False, help="output verbosity.") parser.add_option("-f", "--file", dest="inputfile", - default=hostCheckFilePath, - help="host check result file to read.", metavar="FILE") + default=hostCheckFilePath, + help="host check result file to read.", metavar="FILE") parser.add_option("-o", "--out", dest="outputfile", - default=hostCheckResultPath, - help="log file to store results.", metavar="FILE") + default=hostCheckResultPath, + help="log file to store results.", metavar="FILE") parser.add_option("-k", "--skip", dest="skip", - help="(packages|users|directories|repositories|processes|alternatives)." +\ - " Use , as separator. Use empty string to clean all (by default users are skipped)") + help="(packages|users|directories|repositories|processes|alternatives)." + \ + " Use , as separator.") (options, args) = parser.parse_args() # set output file @@ -469,11 +491,17 @@ def main(): global SKIP_LIST SKIP_LIST = options.skip.split(',') - is_root = h.is_current_user_root() if not is_root: raise RuntimeError('HostCleanup needs to be run as root.') + if "users" not in SKIP_LIST: + delete_users = get_YN_input('You have elected to remove all users as well. If it is not intended then use ' + 'option --skip \"users\". Do you want to continue [y/n] (n)', False) + if not delete_users: + print 'Exiting. Use option --skip="users" to skip deleting users' + sys.exit(1) + hostcheckfile = options.inputfile propMap = h.read_host_check_file(hostcheckfile) http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/76daacf0/ambari-agent/src/main/python/ambari_agent/HostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py index 00b77ff..56cbae1 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py @@ -80,7 +80,7 @@ class HostInfo: # Additional packages to look for (search packages that start with these) ADDITIONAL_PACKAGES = [ "rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb", "libconfuse", "ambari-log4j", - "hadoop", "zookeeper" + "hadoop", "zookeeper", "oozie", "webhcat" ] # ignore packages from repos whose names start with these strings @@ -263,8 +263,15 @@ class HostInfo: return self.current_umask else: return self.current_umask - - + + + def get_os_type(self): + os_info = platform.linux_distribution( + None, None, None, ['SuSE', 'redhat' ], 0 + ) + return os_info[0].lower() + + """ Return various details about the host componentsMapped: indicates if any components are mapped to this host commandsInProgress: indicates if any commands are in progress @@ -278,17 +285,18 @@ class HostInfo: dict['hostHealth']['diskStatus'] = [self.osdiskAvailableSpace("/")] - dict['rpms'] = [] - liveSvcs = [] self.checkLiveServices(self.DEFAULT_LIVE_SERVICES, liveSvcs) dict['hostHealth']['liveServices'] = liveSvcs dict['umask'] = str(self.getUMask()) + # detailed host check is not available for Suse + isSuse = 'suse' == self.get_os_type() + # If commands are in progress or components are already mapped to this host # Then do not perform certain expensive host checks - if componentsMapped or commandsInProgress: + if componentsMapped or commandsInProgress or isSuse: dict['existingRepos'] = [self.RESULT_UNAVAILABLE] dict['installedPackages'] = [] dict['alternatives'] = [] http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/76daacf0/ambari-agent/src/test/python/TestHostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/TestHostCleanup.py b/ambari-agent/src/test/python/TestHostCleanup.py index a3928be..401aee9 100644 --- a/ambari-agent/src/test/python/TestHostCleanup.py +++ b/ambari-agent/src/test/python/TestHostCleanup.py @@ -103,18 +103,21 @@ created = 2013-07-02 20:39:22.162757""" self.inputfile = inputfile self.skip = skip self.verbose = False - - @patch.object(HostCleanup.HostCleanup, 'do_cleanup') + + @patch.object(HostCleanup, 'get_YN_input') + @patch.object(HostCleanup.HostCleanup, 'do_cleanup') @patch.object(HostCleanup.HostCleanup, 'is_current_user_root') @patch.object(logging.FileHandler, 'setFormatter') @patch.object(HostCleanup.HostCleanup,'read_host_check_file') @patch.object(logging,'basicConfig') @patch.object(logging, 'FileHandler') @patch.object(optparse.OptionParser, 'parse_args') - def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, set_formatter_mock, user_root_mock, do_cleanup_mock): + def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, + set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock): parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False), []) file_handler_mock.return_value = logging.FileHandler('') # disable creating real file user_root_mock.return_value = True + get_yn_input_mock.return_value = True HostCleanup.main() # test --out @@ -140,9 +143,6 @@ created = 2013-07-02 20:39:22.162757""" do_erase_files_silent_method, do_kill_processes_method, get_os_type_method, find_repo_files_for_repos_method, do_erase_alternatives_method): - global SKIP_LIST - oldSkipList = HostCleanup.SKIP_LIST - HostCleanup.SKIP_LIST = [] out = StringIO.StringIO() sys.stdout = out propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'], @@ -170,7 +170,6 @@ created = 2013-07-02 20:39:22.162757""" do_erase_alternatives_method.assert_called_once_with({ALT_KEYS[0]:['alt1', 'alt2'], ALT_KEYS[1]:['dir1']}) - HostCleanup.SKIP_LIST = oldSkipList sys.stdout = sys.__stdout__ @@ -191,6 +190,9 @@ created = 2013-07-02 20:39:22.162757""" do_erase_alternatives_method, get_user_ids_method, do_delete_by_owner_method): + global SKIP_LIST + oldSkipList = HostCleanup.SKIP_LIST + HostCleanup.SKIP_LIST = ["users"] out = StringIO.StringIO() sys.stdout = out propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'], @@ -211,6 +213,7 @@ created = 2013-07-02 20:39:22.162757""" self.assertTrue(do_erase_packages_method.called) self.assertTrue(do_kill_processes_method.called) self.assertTrue(do_erase_alternatives_method.called) + HostCleanup.SKIP_LIST = oldSkipList sys.stdout = sys.__stdout__ @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos') http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/76daacf0/ambari-agent/src/test/python/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/TestHostInfo.py b/ambari-agent/src/test/python/TestHostInfo.py index 3bda116..dc18f42 100644 --- a/ambari-agent/src/test/python/TestHostInfo.py +++ b/ambari-agent/src/test/python/TestHostInfo.py @@ -230,6 +230,51 @@ class TestHostInfo(TestCase): self.assertTrue(newlist[1]['status'], "Invalid home directory") + @patch.object(HostInfo, 'get_os_type') + @patch('os.umask') + @patch.object(HostInfo, 'osdiskAvailableSpace') + @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') + @patch.object(PackagesAnalyzer, 'allAvailablePackages') + @patch.object(PackagesAnalyzer, 'allInstalledPackages') + @patch.object(PackagesAnalyzer, 'getPackageDetails') + @patch.object(PackagesAnalyzer, 'getInstalledPkgsByNames') + @patch.object(PackagesAnalyzer, 'getInstalledPkgsByRepo') + @patch.object(PackagesAnalyzer, 'getInstalledRepos') + @patch.object(HostInfo, 'checkUsers') + @patch.object(HostInfo, 'checkLiveServices') + @patch.object(HostInfo, 'javaProcs') + @patch.object(HostInfo, 'checkFolders') + @patch.object(HostInfo, 'etcAlternativesConf') + @patch.object(HostInfo, 'hadoopVarRunCount') + @patch.object(HostInfo, 'hadoopVarLogCount') + def test_hostinfo_register_suse(self, hvlc_mock, hvrc_mock, eac_mock, cf_mock, jp_mock, + cls_mock, cu_mock, gir_mock, gipbr_mock, gipbn_mock, + gpd_mock, aip_mock, aap_mock, whcf_mock, odas_mock, + os_umask_mock, get_os_type_mock): + hvlc_mock.return_value = 1 + hvrc_mock.return_value = 1 + gipbr_mock.return_value = ["pkg1"] + gipbn_mock.return_value = ["pkg2"] + gpd_mock.return_value = ["pkg1", "pkg2"] + odas_mock.return_value = [{'name':'name1'}] + get_os_type_mock.return_value = "suse" + + hostInfo = HostInfo() + dict = {} + hostInfo.register(dict, False, False) + self.assertFalse(gir_mock.called) + self.assertFalse(gpd_mock.called) + self.assertFalse(aip_mock.called) + self.assertFalse(aap_mock.called) + self.assertTrue(odas_mock.called) + self.assertTrue(os_umask_mock.called) + self.assertFalse(whcf_mock.called) + + self.assertTrue(0 == len(dict['installedPackages'])) + self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth']) + + + @patch.object(HostInfo, 'get_os_type') @patch('os.umask') @patch.object(HostInfo, 'osdiskAvailableSpace') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') @@ -249,13 +294,14 @@ class TestHostInfo(TestCase): def test_hostinfo_register(self, hvlc_mock, hvrc_mock, eac_mock, cf_mock, jp_mock, cls_mock, cu_mock, gir_mock, gipbr_mock, gipbn_mock, gpd_mock, aip_mock, aap_mock, whcf_mock, odas_mock, - os_umask_mock): + os_umask_mock, get_os_type_mock): hvlc_mock.return_value = 1 hvrc_mock.return_value = 1 gipbr_mock.return_value = ["pkg1"] gipbn_mock.return_value = ["pkg2"] gpd_mock.return_value = ["pkg1", "pkg2"] odas_mock.return_value = [{'name':'name1'}] + get_os_type_mock.return_value = "redhat" hostInfo = HostInfo() dict = {} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/76daacf0/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentEnv.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentEnv.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentEnv.java index 97b1f82..6c62783 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentEnv.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/AgentEnv.java @@ -32,11 +32,6 @@ public class AgentEnv { private Directory[] stackFoldersAndFiles = new Directory[0]; /** - * Various RPM package versions. - */ - private Rpm[] rpms = new Rpm[0]; - - /** * Directories that match name <code>/etc/alternatives/*conf</code> */ private Alternative[] alternatives = new Alternative[0]; @@ -79,14 +74,6 @@ public class AgentEnv { stackFoldersAndFiles = dirs; } - public void setRpms(Rpm[] rpm) { - rpms = rpm; - } - - public Rpm[] getRpms() { - return rpms; - } - public void setExistingUsers(ExistingUser[] users) { existingUsers = users; } @@ -199,43 +186,6 @@ public class AgentEnv { } } - /** - * Represents information about rpm-installed packages - */ - public static class Rpm { - @SerializedName("name") - private String rpmName; - @SerializedName("installed") - private boolean rpmInstalled = false; - @SerializedName("version") - private String rpmVersion; - - public void setName(String name) { - rpmName = name; - } - - public String getName() { - return rpmName; - } - - public void setInstalled(boolean installed) { - rpmInstalled = installed; - } - - public boolean isInstalled() { - return rpmInstalled; - } - - public void setVersion(String version) { - rpmVersion = version; - } - - @JsonSerialize(include=Inclusion.NON_NULL) - public String getVersion() { - return rpmVersion; - } - } - public static class PackageDetail { @SerializedName("name") private String pkgName;
