Repository: ambari Updated Branches: refs/heads/branch-2.4 6ffa3f8fa -> c80dd0dfc refs/heads/trunk 8c1564e08 -> 0c10fb190
AMBARI-17120. Ambari warns the user about a bunch of rpm packages that are installed on barebone CentOS 7 (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0c10fb19 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0c10fb19 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0c10fb19 Branch: refs/heads/trunk Commit: 0c10fb1909741aa9255978340be2c3656ded0d3b Parents: 8c1564e Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Wed Jun 8 18:53:41 2016 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Wed Jun 8 18:53:41 2016 +0300 ---------------------------------------------------------------------- .../test/python/ambari_agent/TestHostInfo.py | 46 ------------------- .../resource_management/TestPackagesAnalyzer.py | 48 +++++++++++++++++++- .../libraries/functions/packages_analyzer.py | 11 ++--- .../custom_actions/scripts/check_host.py | 8 ++-- 4 files changed, 55 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0c10fb19/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py index 71ee23f..616fd5b 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py @@ -75,52 +75,6 @@ class TestHostInfo(TestCase): self.assertTrue(installedPackages[3][2], "HDP") self.assertTrue(installedPackages[6][1], "11-38.13.9") - def test_perform_package_analysis(self): - installedPackages = [ - ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"], - ["hadoop-b", "2.3", "HDP-epel"], ["epel", "3.1", "HDP-epel"], ["epel-2", "3.1", "HDP-epel"], - ["hadoop-c", "2.3", "Ambari"], ["ambari-s", "3.1", "Ambari"], - ["ganglia", "2.3", "GANGLIA"], ["rrd", "3.1", "RRD"], - ["keeper-1", "2.3", "GANGLIA"], ["keeper-2", "3.1", "base"],["def-def.x86", "2.2", "DEF.3"], - ["def.1", "1.2", "NewDEF"] - ] - availablePackages = [ - ["hadoop-d", "2.3", "HDP"], ["zk-2", "3.1", "HDP"], ["pig", "3.1", "HDP"], - ["epel-3", "2.3", "HDP-epel"], ["hadoop-e", "3.1", "HDP-epel"], - ["ambari-a", "3.1", "Ambari"], - ["keeper-3", "3.1", "base"] - ] - - packagesToLook = ["webhcat", "hadoop", "*-def"] - reposToIgnore = ["ambari"] - additionalPackages = ["ganglia", "rrd"] - - repos = [] - packages_analyzer.getInstalledRepos(packagesToLook, installedPackages + availablePackages, reposToIgnore, repos) - self.assertEqual(3, len(repos)) - expected = ["HDP", "HDP-epel", "DEF.3"] - for repo in expected: - self.assertTrue(repo in repos) - - packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages) - self.assertEqual(5, len(packagesInstalled)) - expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"] - for repo in expected: - self.assertTrue(repo in packagesInstalled) - - additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( - additionalPackages, installedPackages) - self.assertEqual(2, len(additionalPkgsInstalled)) - expected = ["ganglia", "rrd"] - for additionalPkg in expected: - self.assertTrue(additionalPkg in additionalPkgsInstalled) - - allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) - self.assertEqual(7, len(allPackages)) - expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "ganglia", "rrd", "def-def.x86"] - for package in expected: - self.assertTrue(package in allPackages) - @patch.object(OSCheck, 'get_os_family') @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock): http://git-wip-us.apache.org/repos/asf/ambari/blob/0c10fb19/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py b/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py index d9ddb38..73484f3 100644 --- a/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py +++ b/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py @@ -37,4 +37,50 @@ class TestPackagesAnalyzer(TestCase): checked_call_mock.return_value = (0, '0.0.1-SNAPSHOT','') result = packages_analyzer.getInstalledPackageVersion("package1") self.assertEqual(result, '0.0.1-SNAPSHOT') - self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)]) \ No newline at end of file + self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)]) + + def test_perform_package_analysis(self): + installedPackages = [ + ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"], + ["hadoop-b", "2.3", "HDP-epel"], ["epel", "3.1", "HDP-epel"], ["epel-2", "3.1", "HDP-epel"], + ["hadoop-c", "2.3", "Ambari"], ["ambari-s", "3.1", "Ambari"], + ["ganglia", "2.3", "GANGLIA"], ["rrd", "3.1", "RRD"], + ["keeper-1", "2.3", "GANGLIA"], ["keeper-2", "3.1", "base"],["def-def.x86", "2.2", "DEF.3"], + ["def.1", "1.2", "NewDEF"] + ] + availablePackages = [ + ["hadoop-d", "2.3", "HDP"], ["zk-2", "3.1", "HDP"], ["pig", "3.1", "HDP"], + ["epel-3", "2.3", "HDP-epel"], ["hadoop-e", "3.1", "HDP-epel"], + ["ambari-a", "3.1", "Ambari"], + ["keeper-3", "3.1", "base"] + ] + + packagesToLook = ["^webhcat.*$", "^hadoop.*$", "^.+-def.*$"] + reposToIgnore = ["ambari"] + additionalPackages = ["ganglia", "rrd"] + + repos = [] + packages_analyzer.getInstalledRepos(packagesToLook, installedPackages + availablePackages, reposToIgnore, repos) + self.assertEqual(3, len(repos)) + expected = ["HDP", "HDP-epel", "DEF.3"] + for repo in expected: + self.assertTrue(repo in repos) + + packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages) + self.assertEqual(5, len(packagesInstalled)) + expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"] + for repo in expected: + self.assertTrue(repo in packagesInstalled) + + additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( + additionalPackages, installedPackages) + self.assertEqual(2, len(additionalPkgsInstalled)) + expected = ["ganglia", "rrd"] + for additionalPkg in expected: + self.assertTrue(additionalPkg in additionalPkgsInstalled) + + allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) + self.assertEqual(7, len(allPackages)) + expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "ganglia", "rrd", "def-def.x86"] + for package in expected: + self.assertTrue(package in allPackages) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0c10fb19/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py index d4b48ef..7a21958 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py @@ -18,6 +18,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +import re import sys import logging import subprocess @@ -191,17 +192,13 @@ def getInstalledRepos(hintPackages, allPackages, ignoreRepos, repoList): Gets all installed repos by name based on repos that provide any package contained in hintPackages Repos starting with value in ignoreRepos will not be returned + hintPackages must be regexps. """ allRepos = [] for hintPackage in hintPackages: for item in allPackages: - if 0 == item[0].find(hintPackage): - if not item[2] in allRepos: - allRepos.append(item[2]) - elif hintPackage[0] == '*': - if item[0].find(hintPackage[1:]) > 0: - if not item[2] in allRepos: - allRepos.append(item[2]) + if re.match(hintPackage, item[0]) and not item[2] in allRepos: + allRepos.append(item[2]) for repo in allRepos: ignore = False http://git-wip-us.apache.org/repos/asf/ambari/blob/0c10fb19/ambari-server/src/main/resources/custom_actions/scripts/check_host.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py index 46b8199..004d5ef 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py @@ -81,10 +81,10 @@ THP_FILE_UBUNTU = "/sys/kernel/mm/transparent_hugepage/enabled" class CheckHost(Script): # Package prefixes that are used to find repos (then repos are used to find other packages) PACKAGES = [ - "hadoop", "zookeeper", "webhcat", "oozie", "ambari", "*-manager-server-db", - "*-manager-daemons", "mahout", "spark", "falcon", "hbase", "kafka", "knox", - "slider", "sqoop", "storm", "flume","hcatalog", "phoenix", "ranger", "accumulo", "hive_*", - "pig_", "pig-", "pig." # there's a default 'pigz' package which we should avoid + "^hadoop.*$", "^zookeeper.*$", "^webhcat.*$", "^oozie.*$", "^ambari.*$", "^.+-manager-server-db.*$", + "^.+-manager-daemons.*$", "^mahout[_\-]\d.*$", "^spark.*$", "^falcon.*$", "^hbase.*$", "^kafka.*$", "^knox.*$", + "^slider.*$", "^sqoop.*$", "^storm.*$", "^flume.*$","^hcatalog.*$", "^phoenix.*$", "^ranger.*$", "^accumulo.*$", "^hive_.*$", + "^pig[_\-.].*$" # there's a default 'pigz' package which we should avoid ]
