Repository: ambari Updated Branches: refs/heads/trunk 7dfb0e4a5 -> 29fc87f74
AMBARI-9795. [Monarch] Cluster create failed with timeout on the client side (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/29fc87f7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/29fc87f7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/29fc87f7 Branch: refs/heads/trunk Commit: 29fc87f74d3dd0bb3def7ece52d4da1c54fd8fff Parents: 7dfb0e4 Author: Andrew Onishuk <[email protected]> Authored: Thu Feb 26 12:22:06 2015 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Thu Feb 26 12:22:06 2015 +0200 ---------------------------------------------------------------------- ambari-agent/pom.xml | 1 + .../ambari_agent/HostCheckReportFileHandler.py | 86 ++++++++++++++------ .../src/main/python/ambari_agent/HostCleanup.py | 19 ++++- .../src/main/python/ambari_agent/HostInfo.py | 62 -------------- .../TestHostCheckReportFileHandler.py | 23 +++--- .../test/python/ambari_agent/TestHostCleanup.py | 16 ++-- .../test/python/ambari_agent/TestHostInfo.py | 30 ------- .../libraries/functions/packages_analyzer.py | 12 +++ .../custom_actions/scripts/check_host.py | 73 ++++++++++++++++- .../test/python/custom_actions/TestCheckHost.py | 7 +- 10 files changed, 186 insertions(+), 143 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index fbc5029..0952456 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -657,6 +657,7 @@ <artifactId>apache-rat-plugin</artifactId> <configuration> <excludes> + <exclude>src/test/python/tmp_hostcheck.result</exclude> <exclude>src/examples/*</exclude> <exclude>src/test/python/dummy*.txt</exclude> <exclude>src/test/python/ambari_agent/dummy_files/*</exclude> http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py index bf24730..9c0f1d0 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py @@ -22,6 +22,7 @@ import datetime import os.path import logging import traceback +from AmbariConfig import AmbariConfig import ConfigParser; logger = logging.getLogger() @@ -29,12 +30,58 @@ logger = logging.getLogger() class HostCheckReportFileHandler: HOST_CHECK_FILE = "hostcheck.result" + HOST_CHECK_CUSTOM_ACTIONS_FILE = "hostcheck_custom_actions.result" - def __init__(self, config): + def __init__(self, config=None): self.hostCheckFilePath = None - if config is not None: - hostCheckFileDir = config.get('agent', 'prefix') - self.hostCheckFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_FILE) + + if config is None: + config = self.resolve_ambari_config() + + hostCheckFileDir = config.get('agent', 'prefix') + self.hostCheckFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_FILE) + self.hostCheckCustomActionsFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_CUSTOM_ACTIONS_FILE) + + def resolve_ambari_config(self): + try: + config = AmbariConfig() + if os.path.exists(AmbariConfig.getConfigFile()): + config.read(AmbariConfig.getConfigFile()) + else: + raise Exception("No config found, use default") + + except Exception, err: + logger.warn(err) + return config + + def writeHostChecksCustomActionsFile(self, structuredOutput): + if self.hostCheckCustomActionsFilePath is None: + return + + try: + logger.info("Host check custom action report at " + self.hostCheckCustomActionsFilePath) + config = ConfigParser.RawConfigParser() + config.add_section('metadata') + config.set('metadata', 'created', str(datetime.datetime.now())) + + if 'installed_packages' in structuredOutput.keys(): + items = [] + for itemDetail in structuredOutput['installed_packages']: + items.append(itemDetail['name']) + config.add_section('packages') + config.set('packages', 'pkg_list', ','.join(map(str, items))) + + if 'existing_repos' in structuredOutput.keys(): + config.add_section('repositories') + config.set('repositories', 'repo_list', ','.join(structuredOutput['existing_repos'])) + + self.removeFile(self.hostCheckCustomActionsFilePath) + self.touchFile(self.hostCheckCustomActionsFilePath) + with open(self.hostCheckCustomActionsFilePath, 'wb') as configfile: + config.write(configfile) + except Exception, err: + logger.error("Can't write host check file at %s :%s " % (self.hostCheckFilePath, err.message)) + traceback.print_exc() def writeHostCheckFile(self, hostInfo): if self.hostCheckFilePath is None: @@ -81,33 +128,22 @@ class HostCheckReportFileHandler: config.add_section('processes') config.set('processes', 'proc_list', ','.join(map(str, items))) - if 'installedPackages' in hostInfo.keys(): - items = [] - for itemDetail in hostInfo['installedPackages']: - items.append(itemDetail['name']) - config.add_section('packages') - config.set('packages', 'pkg_list', ','.join(map(str, items))) - - if 'existingRepos' in hostInfo.keys(): - config.add_section('repositories') - config.set('repositories', 'repo_list', ','.join(hostInfo['existingRepos'])) - - self.removeFile() - self.touchFile() + self.removeFile(self.hostCheckFilePath) + self.touchFile(self.hostCheckFilePath) with open(self.hostCheckFilePath, 'wb') as configfile: config.write(configfile) except Exception, err: logger.error("Can't write host check file at %s :%s " % (self.hostCheckFilePath, err.message)) traceback.print_exc() - def removeFile(self): - if os.path.isfile(self.hostCheckFilePath): - logger.info("Removing old host check file at %s" % self.hostCheckFilePath) - os.remove(self.hostCheckFilePath) + def removeFile(self, path): + if os.path.isfile(path): + logger.info("Removing old host check file at %s" % path) + os.remove(path) - def touchFile(self): - if not os.path.isfile(self.hostCheckFilePath): - logger.info("Creating host check file at %s" % self.hostCheckFilePath) - open(self.hostCheckFilePath, 'w').close() + def touchFile(self, path): + if not os.path.isfile(path): + logger.info("Creating host check file at %s" % path) + open(path, 'w').close() http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/src/main/python/ambari_agent/HostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py index 398502e..e4a4754 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py +++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py @@ -55,7 +55,9 @@ ALT_ERASE_CMD = "alternatives --remove {0} {1}" REPO_PATH_RHEL = "/etc/yum.repos.d" REPO_PATH_SUSE = "/etc/zypp/repos.d/" SKIP_LIST = [] +TMP_HOST_CHECK_FILE_NAME = "tmp_hostcheck.result" HOST_CHECK_FILE_NAME = "hostcheck.result" +HOST_CHECK_CUSTOM_ACTIONS_FILE = "hostcheck_custom_actions.result" OUTPUT_FILE_NAME = "hostcleanup.result" PACKAGE_SECTION = "packages" @@ -523,13 +525,15 @@ def main(): config = h.resolve_ambari_config() hostCheckFileDir = config.get('agent', 'prefix') hostCheckFilePath = os.path.join(hostCheckFileDir, HOST_CHECK_FILE_NAME) + hostCheckCustomActionsFilePath = os.path.join(hostCheckFileDir, HOST_CHECK_CUSTOM_ACTIONS_FILE) + hostCheckFilesPaths = hostCheckFilePath + "," + hostCheckCustomActionsFilePath hostCheckResultPath = os.path.join(hostCheckFileDir, OUTPUT_FILE_NAME) parser = optparse.OptionParser() parser.add_option("-v", "--verbose", dest="verbose", action="store_false", default=False, help="output verbosity.") - parser.add_option("-f", "--file", dest="inputfile", - default=hostCheckFilePath, + parser.add_option("-f", "--file", dest="inputfiles", + default=hostCheckFilesPaths, help="host check result file to read.", metavar="FILE") parser.add_option("-o", "--out", dest="outputfile", default=hostCheckResultPath, @@ -575,8 +579,15 @@ def main(): print 'Exiting. Use option --skip="users" to skip deleting users' sys.exit(1) - hostcheckfile = options.inputfile - propMap = h.read_host_check_file(hostcheckfile) + hostcheckfile, hostcheckfileca = options.inputfiles.split(",") + + with open(TMP_HOST_CHECK_FILE_NAME, "wb") as tmp_f: + with open(hostcheckfile, "rb") as f1: + with open(hostcheckfileca, "rb") as f2: + tmp_f.write(f1.read()) + tmp_f.write(f2.read()) + + propMap = h.read_host_check_file(TMP_HOST_CHECK_FILE_NAME) if propMap: h.do_cleanup(propMap) http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/src/main/python/ambari_agent/HostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py index dc0834a..516221d 100644 --- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py +++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py @@ -147,33 +147,6 @@ class HostInfoLinux(HostInfo): "/hadoop", "/usr/hdp" ] - # Packages that are used to find repos (then repos are used to find other packages) - PACKAGES = [ - "hadoop_2_2_*", "hadoop-2-2-.*", "zookeeper_2_2_*", "zookeeper-2-2-.*", - "hadoop", "zookeeper", "webhcat", "*-manager-server-db", "*-manager-daemons" - ] - - # Additional packages to look for (search packages that start with these) - ADDITIONAL_PACKAGES = [ - "rrdtool", "rrdtool-python", "ganglia", "gmond", "gweb", "libconfuse", - "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat" - ] - - # ignore packages from repos whose names start with these strings - IGNORE_PACKAGES_FROM_REPOS = [ - "ambari", "installed" - ] - - # ignore required packages - IGNORE_PACKAGES = [ - "epel-release" - ] - - # ignore repos from the list of repos to be cleaned - IGNORE_REPOS = [ - "ambari", "HDP-UTILS" - ] - DEFAULT_SERVICE_NAME = "ntpd" SERVICE_STATUS_CMD = "%s %s status" % (SERVICE_CMD, DEFAULT_SERVICE_NAME) @@ -202,8 +175,6 @@ class HostInfoLinux(HostInfo): result['name'] = fields[0] result['homeDir'] = fields[5] result['status'] = "Available" - if not os.path.exists(homeDir): - result['status'] = "Invalid home directory" results.append(result) def checkFolders(self, basePaths, projectNames, existingUsers, dirs): @@ -248,18 +219,6 @@ class HostInfoLinux(HostInfo): pass pass - def getReposToRemove(self, repos, ignoreList): - reposToRemove = [] - for repo in repos: - addToRemoveList = True - for ignoreRepo in ignoreList: - if packages_analyzer.nameMatch(ignoreRepo, repo): - addToRemoveList = False - continue - if addToRemoveList: - reposToRemove.append(repo) - return reposToRemove - def getTransparentHugePage(self): # This file exist only on redhat 6 thp_regex = "\[(.+)\]" @@ -323,8 +282,6 @@ class HostInfoLinux(HostInfo): # If commands are in progress or components are already mapped to this host # Then do not perform certain expensive host checks if componentsMapped or commandsInProgress: - dict['existingRepos'] = [self.RESULT_UNAVAILABLE] - dict['installedPackages'] = [] dict['alternatives'] = [] dict['stackFoldersAndFiles'] = [] dict['existingUsers'] = [] @@ -342,23 +299,6 @@ class HostInfoLinux(HostInfo): self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs) dict['stackFoldersAndFiles'] = dirs - installedPackages = [] - availablePackages = [] - packages_analyzer.allInstalledPackages(installedPackages) - packages_analyzer.allAvailablePackages(availablePackages) - - repos = [] - packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages, - self.IGNORE_PACKAGES_FROM_REPOS, repos) - packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages) - additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( - self.ADDITIONAL_PACKAGES, installedPackages) - allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) - dict['installedPackages'] = packages_analyzer.getPackageDetails(installedPackages, allPackages) - - repos = self.getReposToRemove(repos, self.IGNORE_REPOS) - dict['existingRepos'] = repos - self.reportFileHandler.writeHostCheckFile(dict) pass @@ -469,8 +409,6 @@ class HostInfoWindows(HostInfo): # If commands are in progress or components are already mapped to this host # Then do not perform certain expensive host checks if componentsMapped or commandsInProgress: - dict['existingRepos'] = [self.RESULT_UNAVAILABLE] - dict['installedPackages'] = [] dict['alternatives'] = [] dict['stackFoldersAndFiles'] = [] dict['existingUsers'] = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py index eb7efb2..d56ad8f 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py @@ -84,12 +84,6 @@ class TestHostCheckReportFileHandler(TestCase): procs = configValidator.get('processes', 'proc_list') self.assertEquals(procs, '') - pkgs = configValidator.get('packages', 'pkg_list') - self.assertEquals(pkgs, '') - - repos = configValidator.get('repositories', 'repo_list') - self.assertEquals(repos, '') - time = configValidator.get('metadata', 'created') self.assertTrue(time != None) @@ -114,11 +108,6 @@ class TestHostCheckReportFileHandler(TestCase): {'pid':355,'hadoop':True,'command':'some command','user':'root'}, {'pid':455,'hadoop':True,'command':'some command','user':'hdfs'} ] - dict['installedPackages'] = [ - {'name':'hadoop','version':'3.2.3','repoName':'HDP'}, - {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'} - ] - dict['existingRepos'] = ['HDP', 'HDP-epel'] handler.writeHostCheckFile(dict) configValidator = ConfigParser.RawConfigParser() @@ -140,6 +129,18 @@ class TestHostCheckReportFileHandler(TestCase): procs = configValidator.get('processes', 'proc_list') self.chkItemsEqual(procs, ['455', '355']) + + dict['installed_packages'] = [ + {'name':'hadoop','version':'3.2.3','repoName':'HDP'}, + {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'} + ] + dict['existing_repos'] = ['HDP', 'HDP-epel'] + + handler.writeHostChecksCustomActionsFile(dict) + configValidator = ConfigParser.RawConfigParser() + configPath_ca = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_CUSTOM_ACTIONS_FILE) + configValidator.read(configPath_ca) + pkgs = configValidator.get('packages', 'pkg_list') self.chkItemsEqual(pkgs, ['hadoop', 'hadoop-lib']) http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py index f43784c..734db5d 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py @@ -112,9 +112,9 @@ class TestHostCleanup(TestCase): sys.stdout = sys.__stdout__ class HostCleanupOptions: - def __init__(self, outputfile, inputfile, skip, verbose, silent, java_home): + def __init__(self, outputfile, inputfiles, skip, verbose, silent, java_home): self.outputfile = outputfile - self.inputfile = inputfile + self.inputfiles = inputfiles self.skip = skip self.verbose = verbose self.silent = silent @@ -131,7 +131,9 @@ class TestHostCleanup(TestCase): @patch.object(optparse.OptionParser, 'parse_args') def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock): - parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False, + open('/tmp/someinputfile1', 'a').close() + open('/tmp/someinputfile2', 'a').close() + parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/tmp/someinputfile1,/tmp/someinputfile2', '', False, False, 'java_home'), []) file_handler_mock.return_value = logging.FileHandler('') # disable creating real file user_root_mock.return_value = True @@ -145,7 +147,7 @@ class TestHostCleanup(TestCase): #test --verbose logging_mock.assert_called_with(level=logging.INFO) # test --in - read_host_check_file_mock.assert_called_with('/someinputfile') + read_host_check_file_mock.assert_called_with('tmp_hostcheck.result') self.assertTrue(get_yn_input_mock.called) @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir') @@ -179,7 +181,9 @@ class TestHostCleanup(TestCase): @patch.object(optparse.OptionParser, 'parse_args') def test_options_silent(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock, set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock): - parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False, + open('/tmp/someinputfile1', 'a').close() + open('/tmp/someinputfile2', 'a').close() + parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/tmp/someinputfile1,/tmp/someinputfile2', '', False, True, 'java_home'), []) file_handler_mock.return_value = logging.FileHandler('') # disable creating real file user_root_mock.return_value = True @@ -193,7 +197,7 @@ class TestHostCleanup(TestCase): #test --verbose logging_mock.assert_called_with(level=logging.INFO) # test --in - read_host_check_file_mock.assert_called_with('/someinputfile') + read_host_check_file_mock.assert_called_with('tmp_hostcheck.result') self.assertFalse(get_yn_input_mock.called) @patch.object(HostCleanup.HostCleanup, 'do_clear_cache') http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py index c1b93d6..fcab6b8 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py @@ -83,20 +83,6 @@ class TestHostInfo(TestCase): self.assertTrue(installedPackages[6][1], "11-38.13.9") @not_for_platform(PLATFORM_WINDOWS) - def test_getReposToRemove(self): - l1 = ["Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15", "Ambari 1.x", "HDP"] - l2 = ["Ambari", "HDP-UTIL"] - hostInfo = HostInfoLinux() - l3 = hostInfo.getReposToRemove(l1, l2) - self.assertTrue(1, len(l3)) - self.assertEqual(l3[0], "HDP") - - l1 = ["AMBARI.dev-1.x", "HDP-1.3.0"] - l3 = hostInfo.getReposToRemove(l1, l2) - self.assertTrue(1, len(l3)) - self.assertEqual(l3[0], "HDP-1.3.0") - - @not_for_platform(PLATFORM_WINDOWS) def test_perform_package_analysis(self): installedPackages = [ ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"], @@ -286,14 +272,9 @@ class TestHostInfo(TestCase): dict = {} hostInfo.register(dict, False, False) self.assertTrue(cit_mock.called) - self.assertTrue(gir_mock.called) - self.assertTrue(gpd_mock.called) - self.assertTrue(aip_mock.called) - self.assertTrue(aap_mock.called) self.assertTrue(os_umask_mock.called) self.assertTrue(whcf_mock.called) - self.assertTrue(0 < len(dict['installedPackages'])) self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth']) @not_for_platform(PLATFORM_WINDOWS) @@ -342,25 +323,14 @@ class TestHostInfo(TestCase): hostInfo = HostInfoLinux() dict = {} hostInfo.register(dict, False, False) - self.assertTrue(gir_mock.called) - self.assertTrue(gpd_mock.called) - self.assertTrue(aip_mock.called) self.assertTrue(cit_mock.called) self.assertEqual(1, cit_mock.call_count) - for existingPkg in ["pkg1", "pkg2"]: - self.assertTrue(existingPkg in dict['installedPackages']) - args, kwargs = gpd_mock.call_args_list[0] - for existingPkg in ["pkg1", "pkg2"]: - self.assertTrue(existingPkg in args[1]) - def verifyReturnedValues(self, dict): hostInfo = HostInfoLinux() self.assertEqual(dict['alternatives'], []) self.assertEqual(dict['stackFoldersAndFiles'], []) self.assertEqual(dict['existingUsers'], []) - self.assertEqual(dict['existingRepos'][0], hostInfo.RESULT_UNAVAILABLE) - self.assertEqual(dict['installedPackages'], []) self.assertTrue(dict['iptablesIsRunning']) @patch("os.path.exists") http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py index 888387c..55c1eb8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py @@ -261,3 +261,15 @@ def getPackageDetails(installedPackages, foundPackages): pkgDetail['repoName'] = installedPackage[2] packageDetails.append(pkgDetail) return packageDetails + +def getReposToRemove(repos, ignoreList): + reposToRemove = [] + for repo in repos: + addToRemoveList = True + for ignoreRepo in ignoreList: + if nameMatch(ignoreRepo, repo): + addToRemoveList = False + continue + if addToRemoveList: + reposToRemove.append(repo) + return reposToRemove http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-server/src/main/resources/custom_actions/scripts/check_host.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py index ff2eabd..b0fcfb6 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py @@ -24,16 +24,20 @@ import os import subprocess import socket +from resource_management.libraries.functions import packages_analyzer from ambari_commons import os_utils from ambari_commons.os_check import OSCheck, OSConst from ambari_commons.inet_utils import download_file from resource_management import Script, Execute, format from ambari_agent.HostInfo import HostInfo +from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler CHECK_JAVA_HOME = "java_home_check" CHECK_DB_CONNECTION = "db_connection_check" CHECK_HOST_RESOLUTION = "host_resolution_check" CHECK_LAST_AGENT_ENV = "last_agent_env_check" +CHECK_INSTALLED_PACKAGES = "installed_packages" +CHECK_EXISTING_REPOS = "existing_repos" DB_MYSQL = "mysql" DB_ORACLE = "oracle" @@ -52,9 +56,42 @@ JDBC_DRIVER_SYMLINK_MSSQL = "sqljdbc4.jar" JDBC_AUTH_SYMLINK_MSSQL = "sqljdbc_auth.dll" class CheckHost(Script): + # Packages that are used to find repos (then repos are used to find other packages) + PACKAGES = [ + "hadoop_2_2_*", "hadoop-2-2-.*", "zookeeper_2_2_*", "zookeeper-2-2-.*", + "hadoop", "zookeeper", "webhcat", "*-manager-server-db", "*-manager-daemons" + ] + + + # ignore packages from repos whose names start with these strings + IGNORE_PACKAGES_FROM_REPOS = [ + "ambari", "installed" + ] + + + # ignore required packages + IGNORE_PACKAGES = [ + "epel-release" + ] + + # Additional packages to look for (search packages that start with these) + ADDITIONAL_PACKAGES = [ + "rrdtool", "rrdtool-python", "ganglia", "gmond", "gweb", "libconfuse", + "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat" + ] + + # ignore repos from the list of repos to be cleaned + IGNORE_REPOS = [ + "ambari", "HDP-UTILS" + ] + + def __init__(self): + self.reportFileHandler = HostCheckReportFileHandler() + def actionexecute(self, env): config = Script.get_config() tmp_dir = Script.get_tmp_dir() + report_file_handler_dict = {} #print "CONFIG: " + str(config) @@ -93,9 +130,43 @@ class CheckHost(Script): except Exception, exception : print "There was an unknown error while checking last host environment details: " + str(exception) structured_output[CHECK_LAST_AGENT_ENV] = {"exit_code" : 1, "message": str(exception)} - + + # CHECK_INSTALLED_PACKAGES and CHECK_EXISTING_REPOS required to run together for + # reasons of not doing the same common work twice for them as it takes some time, especially on Ubuntu. + if CHECK_INSTALLED_PACKAGES in check_execute_list and CHECK_EXISTING_REPOS in check_execute_list: + try : + installed_packages, repos = self.execute_existing_repos_and_installed_packages_check(config) + structured_output[CHECK_INSTALLED_PACKAGES] = installed_packages + structured_output[CHECK_EXISTING_REPOS] = repos + except Exception, exception : + print "There was an unknown error while checking installed packages and existing repositories: " + str(exception) + structured_output[CHECK_INSTALLED_PACKAGES] = {"exit_code" : 1, "message": str(exception)} + structured_output[CHECK_EXISTING_REPOS] = {"exit_code" : 1, "message": str(exception)} + + # this is necessary for HostCleanup to know later what were the results. + self.reportFileHandler.writeHostChecksCustomActionsFile(structured_output) + self.put_structured_out(structured_output) + def execute_existing_repos_and_installed_packages_check(self, config): + installedPackages = [] + availablePackages = [] + packages_analyzer.allInstalledPackages(installedPackages) + packages_analyzer.allAvailablePackages(availablePackages) + + repos = [] + packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages, + self.IGNORE_PACKAGES_FROM_REPOS, repos) + packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages) + additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( + self.ADDITIONAL_PACKAGES, installedPackages) + allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) + + installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages) + repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS) + + return installedPackages, repos + def execute_java_home_available_check(self, config): print "Java home check started." http://git-wip-us.apache.org/repos/asf/ambari/blob/29fc87f7/ambari-server/src/test/python/custom_actions/TestCheckHost.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestCheckHost.py b/ambari-server/src/test/python/custom_actions/TestCheckHost.py index 105a0b0..2cd5c7a 100644 --- a/ambari-server/src/test/python/custom_actions/TestCheckHost.py +++ b/ambari-server/src/test/python/custom_actions/TestCheckHost.py @@ -55,7 +55,8 @@ class TestCheckHost(TestCase): checkHost = CheckHost() checkHost.actionexecute(None) - self.assertEquals(os_isfile_mock.call_args[0][0], 'test_java_home/bin/java') + print os_isfile_mock.call_args + self.assertEquals(os_isfile_mock.call_args[0][0], '/tmp/ambari-agent/hostcheck_custom_actions.result') self.assertEquals(structured_out_mock.call_args[0][0], {'java_home_check': {'message': 'Java home exists!', 'exit_code': 0}}) # test, java home doesn't exist @@ -64,7 +65,7 @@ class TestCheckHost(TestCase): checkHost.actionexecute(None) - self.assertEquals(os_isfile_mock.call_args[0][0], 'test_java_home/bin/java') + self.assertEquals(os_isfile_mock.call_args[0][0], '/tmp/ambari-agent/hostcheck_custom_actions.result') self.assertEquals(structured_out_mock.call_args[0][0], {'java_home_check': {"message": "Java home doesn't exist!", "exit_code" : 1}}) @@ -282,8 +283,6 @@ class TestCheckHost(TestCase): self.assertTrue('alternatives' in last_agent_env_check_result['last_agent_env_check']) self.assertTrue('umask' in last_agent_env_check_result['last_agent_env_check']) self.assertTrue('stackFoldersAndFiles' in last_agent_env_check_result['last_agent_env_check']) - self.assertTrue('existingRepos' in last_agent_env_check_result['last_agent_env_check']) - self.assertTrue('installedPackages' in last_agent_env_check_result['last_agent_env_check']) self.assertTrue('existingUsers' in last_agent_env_check_result['last_agent_env_check']) # try it now with errors
