AMBARI-21398 Backend need to distinguish repo_version records by using the Stack Name (dgrinenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/33879743 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/33879743 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/33879743 Branch: refs/heads/branch-feature-AMBARI-21450 Commit: 338797435c8e6b489cdec12f0320b753a157e9e6 Parents: 13fadd9 Author: Dmytro Grinenko <hapyles...@apache.org> Authored: Thu Aug 17 13:02:53 2017 +0300 Committer: Dmytro Grinenko <hapyles...@apache.org> Committed: Thu Aug 17 13:02:53 2017 +0300 ---------------------------------------------------------------------- .../test/python/ambari_agent/TestHostInfo.py | 256 +-- .../resource_management/TestPackagesAnalyzer.py | 141 -- .../python/resource_management/TestScript.py | 23 +- .../src/main/python/ambari_commons/shell.py | 144 +- .../core/providers/__init__.py | 42 +- .../core/providers/package/__init__.py | 247 ++- .../core/providers/package/apt.py | 289 +++- .../core/providers/package/yumrpm.py | 223 ++- .../core/providers/package/zypper.py | 147 +- .../libraries/functions/packages_analyzer.py | 356 ----- .../libraries/script/script.py | 22 +- .../custom_actions/scripts/check_host.py | 26 +- .../custom_actions/scripts/install_packages.py | 60 +- .../scripts/remove_previous_stacks.py | 7 +- .../custom_actions/TestInstallPackages.py | 1488 ++++++++++-------- .../custom_actions/TestRemoveStackVersion.py | 40 +- .../stacks/2.0.6/HBASE/test_hbase_master.py | 93 +- 17 files changed, 1987 insertions(+), 1617 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py index 28e7d2b..cb20f4e 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py +++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py @@ -39,113 +39,12 @@ from ambari_agent.Hardware import Hardware from ambari_agent.AmbariConfig import AmbariConfig from resource_management.core import shell from resource_management.core.system import System -from resource_management.libraries.functions import packages_analyzer + @not_for_platform(PLATFORM_WINDOWS) @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) class TestHostInfo(TestCase): - @patch.object(OSCheck, 'get_os_family') - @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') - def test_analyze_zypper_out(self, spwt_mock, get_os_family_mock): - get_os_family_mock.return_value = 'suse' - stringToRead = """Refreshing service 'susecloud'. - Loading repository data... - Reading installed packages... - - S | Name | Type | Version | Arch | Repository - --+-----------------------------------+---------+------------------------+--------+---------------------- - i | ConsoleKit | package | 0.2.10-64.65.1 | x86_64 | SLES11-SP1-Updates - i | gweb | package | 2.2.0-99 | noarch | Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15 - i | hadoop | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP - i | hadoop-libhdfs | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP - i | ambari-server | package | 1.2.4.9-1 | noarch | Ambari 1.x - i | hdp_mon_ganglia_addons | package | 1.2.4.9-1 | noarch | Ambari 1.x - i | Minimal | pattern | 11-38.13.9 | x86_64 | SLES11-SP1""" - result = {} - result['out'] = stringToRead - result['err'] = "" - result['retCode'] = 0 - - spwt_mock.return_value = result - installedPackages = [] - packages_analyzer.allInstalledPackages(installedPackages) - self.assertEqual(7, len(installedPackages)) - self.assertTrue(installedPackages[1][0], "gweb") - self.assertTrue(installedPackages[3][2], "HDP") - self.assertTrue(installedPackages[6][1], "11-38.13.9") - - @patch.object(OSCheck, 'get_os_family') - @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') - def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock): - get_os_family_mock.return_value = 'redhat' - stringToRead = """Loaded plugins: amazon-id, product-id, rhui-lb, security, subscription-manager - Updating certificate-based repositories. - Installed Packages - AMBARI.dev.noarch 1.x-1.el6 installed - PyXML.x86_64 0.8.4-19.el6 @koji-override-0 - Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch - 3-7.el6 @koji-override-0 - hcatalog.noarch 0.11.0.1.3.0.0-107.el6 - @HDP-1.3.0 - hesiod.x86_64 3.1.0-19.el6 @koji-override-0/$releasever - hive.noarch 0.11.0.1.3.0.0-107.el6 - @HDP-1.3.0 - oracle-server-db.x86 1.3.17-2 - @Oracle-11g - ambari-log4j.noarch 1.2.5.9-1 @AMBARI.dev-1.x - libconfuse.x86_64 2.7-4.el6 @HDP-epel""" - result = {} - result['out'] = stringToRead - result['err'] = "" - result['retCode'] = 0 - - subprocessWithTimeout_mock.return_value = result - installedPackages = [] - packages_analyzer.allInstalledPackages(installedPackages) - self.assertEqual(9, len(installedPackages)) - for package in installedPackages: - self.assertTrue(package[0] in ["AMBARI.dev", "PyXML", "oracle-server-db", - "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US", - "hcatalog", "hesiod", "hive", "ambari-log4j", "libconfuse"]) - self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6", - "0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1", "2.7-4.el6"]) - self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0", - "koji-override-0/$releasever", "AMBARI.dev-1.x", "Oracle-11g", "HDP-epel"]) - - packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise", "hesiod", "hive"], - installedPackages) - self.assertEqual(4, len(packages)) - expected = ["AMBARI.dev", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US", - "hesiod", "hive"] - for package in expected: - self.assertTrue(package in packages) - - detailedPackages = packages_analyzer.getPackageDetails(installedPackages, packages) - self.assertEqual(4, len(detailedPackages)) - for package in detailedPackages: - self.assertTrue(package['version'] in ["1.x-1.el6", "3-7.el6", "3.1.0-19.el6", - "0.11.0.1.3.0.0-107.el6"]) - self.assertTrue(package['repoName'] in ["installed", "koji-override-0", "HDP-1.3.0", - "koji-override-0/$releasever"]) - self.assertFalse(package['repoName'] in ["AMBARI.dev-1.x"]) - - @patch.object(OSCheck, 'get_os_family') - @patch('resource_management.libraries.functions.packages_analyzer.subprocessWithTimeout') - def test_analyze_yum_output_err(self, subprocessWithTimeout_mock, get_os_family_mock): - get_os_family_mock.return_value = OSConst.REDHAT_FAMILY - - result = {} - result['out'] = "" - result['err'] = "" - result['retCode'] = 1 - - subprocessWithTimeout_mock.return_value = result - installedPackages = [] - packages_analyzer.allInstalledPackages(installedPackages) - self.assertEqual(installedPackages, []) - - @patch('os.path.exists') def test_checkFolders(self, path_mock): path_mock.return_value = True @@ -184,12 +83,7 @@ class TestHostInfo(TestCase): @patch.object(OSCheck, "get_os_type") @patch('os.umask') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') - @patch('resource_management.libraries.functions.packages_analyzer.allAvailablePackages') - @patch('resource_management.libraries.functions.packages_analyzer.allInstalledPackages') - @patch('resource_management.libraries.functions.packages_analyzer.getPackageDetails') - @patch('resource_management.libraries.functions.packages_analyzer.getInstalledPkgsByNames') - @patch('resource_management.libraries.functions.packages_analyzer.getInstalledPkgsByRepo') - @patch('resource_management.libraries.functions.packages_analyzer.getInstalledRepos') + @patch("resource_management.core.providers.get_provider") @patch.object(HostInfoLinux, 'checkUsers') @patch.object(HostInfoLinux, 'checkLiveServices') @patch.object(HostInfoLinux, 'javaProcs') @@ -200,14 +94,19 @@ class TestHostInfo(TestCase): @patch.object(HostInfoLinux, 'checkFirewall') @patch.object(HostInfoLinux, 'checkUnlimitedJce') def test_hostinfo_register_suse(self, jce_mock, cit_mock, hvlc_mock, hvrc_mock, eac_mock, cf_mock, jp_mock, - cls_mock, cu_mock, gir_mock, gipbr_mock, gipbn_mock, - gpd_mock, aip_mock, aap_mock, whcf_mock, os_umask_mock, get_os_type_mock): + cls_mock, cu_mock, get_packages_mock, whcf_mock, os_umask_mock, get_os_type_mock): + + m = MagicMock() + + m.get_package_details.return_value = ["pkg1", "pkg2"] + m.get_installed_pkgs_by_names.return_value = ["pkg2"] + m.get_installed_pkgs_by_repo.return_value = ["pkg1"] + + get_packages_mock.return_value = m + cit_mock.return_value = True hvlc_mock.return_value = 1 hvrc_mock.return_value = 1 - gipbr_mock.return_value = ["pkg1"] - gipbn_mock.return_value = ["pkg2"] - gpd_mock.return_value = ["pkg1", "pkg2"] get_os_type_mock.return_value = "suse" hostInfo = HostInfoLinux() @@ -223,12 +122,7 @@ class TestHostInfo(TestCase): @patch.object(OSCheck, "get_os_type") @patch('os.umask') @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile') - @patch('resource_management.libraries.functions.packages_analyzer.allAvailablePackages') - @patch('resource_management.libraries.functions.packages_analyzer.allInstalledPackages') - @patch('resource_management.libraries.functions.packages_analyzer.getPackageDetails') - @patch('resource_management.libraries.functions.packages_analyzer.getInstalledPkgsByNames') - @patch('resource_management.libraries.functions.packages_analyzer.getInstalledPkgsByRepo') - @patch('resource_management.libraries.functions.packages_analyzer.getInstalledRepos') + @patch("resource_management.core.providers.get_provider") @patch.object(HostInfoLinux, 'checkUsers') @patch.object(HostInfoLinux, 'checkLiveServices') @patch.object(HostInfoLinux, 'javaProcs') @@ -239,14 +133,18 @@ class TestHostInfo(TestCase): @patch.object(HostInfoLinux, 'checkFirewall') @patch.object(HostInfoLinux, 'getTransparentHugePage') def test_hostinfo_register(self, get_transparentHuge_page_mock, cit_mock, hvlc_mock, hvrc_mock, eac_mock, cf_mock, jp_mock, - cls_mock, cu_mock, gir_mock, gipbr_mock, gipbn_mock, - gpd_mock, aip_mock, aap_mock, whcf_mock, os_umask_mock, get_os_type_mock): + cls_mock, cu_mock, get_packages_mock, whcf_mock, os_umask_mock, get_os_type_mock): + m = MagicMock() + + m.get_package_details.return_value = ["pkg1", "pkg2"] + m.get_installed_pkgs_by_names.return_value = ["pkg2"] + m.get_installed_pkgs_by_repo.return_value = ["pkg1"] + + get_packages_mock.return_value = m + cit_mock.return_value = True hvlc_mock.return_value = 1 hvrc_mock.return_value = 1 - gipbr_mock.return_value = ["pkg1"] - gipbn_mock.return_value = ["pkg2"] - gpd_mock.return_value = ["pkg1", "pkg2"] get_os_type_mock.return_value = "redhat" hostInfo = HostInfoLinux() @@ -413,7 +311,7 @@ class TestHostInfo(TestCase): self.assertEquals(result[0]['status'], 'Unhealthy') self.assertEquals(result[0]['name'], 'service1 or service2') - self.assertEquals(result[0]['desc'], 'out\nout') + self.assertEquals(result[0]['desc'], 'out{0}out'.format(os.linesep)) msg = 'thrown by shell call' shell_call.side_effect = Exception(msg) @@ -538,112 +436,6 @@ class TestHostInfo(TestCase): os_path_isfile_mock.return_value = False self.assertEqual("", hostInfo.getTransparentHugePage()) - @staticmethod - def _add_packages_available(command, arg): - arg.append(["hadoop_2_2_0_1_885", "1.0", "HDP-2.2"]) - arg.append(["hadooplzo_2_2_0_1_885", "1.0", "HDP-2.2"]) - arg.append(["hadoop_2_2_0_1_885-libhdfs", "1.0", "HDP-2.2"]) - - @staticmethod - def _add_packages_lookUpYum(command, key, arg): - TestHostInfo._add_packages_available(command, arg) - - @patch("ambari_commons.os_check.OSCheck.is_suse_family") - @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family") - @patch("ambari_commons.os_check.OSCheck.is_redhat_family") - @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages") - def test_get_available_packages_in_repos_suse(self, lookUpZypperPackages, is_redhat_family, is_ubuntu_family, - is_suse_family_mock): - is_suse_family_mock.return_value = True - is_redhat_family.return_value = False - is_ubuntu_family.return_value = False - lookUpZypperPackages.side_effect = TestHostInfo._add_packages_available - - command_json = { - "repositoryFile": { - "stackName": "HDP", - "repoVersionId": 1, - "repoVersion": "2", - "repositories": [ - { - "repoName": "HDP", - "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, - "repoId": "HDP-2.2" - } - ] - } - } - - available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( - command_json['repositoryFile']['repositories']) - - self.assertEqual(available_packages_in_repos, - ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"]) - - @patch("ambari_commons.os_check.OSCheck.is_suse_family") - @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family") - @patch("ambari_commons.os_check.OSCheck.is_redhat_family") - @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages") - def test_get_available_packages_in_repos_rhel(self, lookUpYumPackages, is_redhat_family, is_ubuntu_family, - is_suse_family_mock): - is_suse_family_mock.return_value = False - is_redhat_family.return_value = True - is_ubuntu_family.return_value = False - lookUpYumPackages.side_effect = TestHostInfo._add_packages_lookUpYum - - command_json = { - "repositoryFile": { - "stackName": "HDP", - "repoVersionId": 1, - "repoVersion": "2", - "repositories": [ - { - "repoName": "HDP", - "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, - "repoId": "HDP-2.2" - } - ] - } - } - - available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( - command_json['repositoryFile']['repositories']) - - self.assertEqual(available_packages_in_repos, - ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs", "hadoop_2_2_0_1_885", - "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"]) - - @patch("ambari_commons.os_check.OSCheck.is_suse_family") - @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family") - @patch("ambari_commons.os_check.OSCheck.is_redhat_family") - @patch("resource_management.libraries.functions.packages_analyzer._lookUpAptPackages") - def test_get_available_packages_in_repos_ubuntu(self, lookUpAptPackages, is_redhat_family, is_ubuntu_family, - is_suse_family_mock): - is_suse_family_mock.return_value = False - is_redhat_family.return_value = False - is_ubuntu_family.return_value = True - lookUpAptPackages.side_effect = TestHostInfo._add_packages_available - - command_json = { - "repositoryFile": { - "stackName": "HDP", - "repoVersionId": 1, - "repoVersion": "2", - "repositories": [ - { - "repoName": "HDP", - "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0";, - "repoId": "HDP-2.2" - } - ] - } - } - - available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( - command_json['repositoryFile']['repositories']) - - self.assertEqual(available_packages_in_repos, - ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"]) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py b/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py deleted file mode 100644 index f898919..0000000 --- a/ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py +++ /dev/null @@ -1,141 +0,0 @@ -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' -from unittest import TestCase -from mock.mock import patch, MagicMock, call -from ambari_commons.os_check import OSCheck, OSConst -from resource_management.core.exceptions import Fail -from resource_management.libraries.functions import packages_analyzer - -class TestPackagesAnalyzer(TestCase): - @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call") - @patch.object(OSCheck, "is_ubuntu_family") - def test_get_installed_package_version_ubuntu(self, is_ubuntu_family_mock, checked_call_mock): - is_ubuntu_family_mock.return_value = True - checked_call_mock.return_value = (0, '1.2.3','') - result = packages_analyzer.getInstalledPackageVersion("package1") - self.assertEqual(result, '1.2.3') - self.assertEqual(checked_call_mock.call_args_list, [call("dpkg -s package1 | grep Version | awk '{print $2}'", stderr=-1)]) - - @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call") - @patch.object(OSCheck, "is_ubuntu_family") - def test_get_installed_package_version_centos_suse(self, is_ubuntu_family_mock, checked_call_mock): - is_ubuntu_family_mock.return_value = False - checked_call_mock.return_value = (0, '0.0.1-SNAPSHOT','') - result = packages_analyzer.getInstalledPackageVersion("package1") - self.assertEqual(result, '0.0.1-SNAPSHOT') - self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)]) - - @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call") - @patch.object(OSCheck, "is_in_family") - @patch.object(packages_analyzer, "Logger") - def test_vefify_dependency_suse(self, logger_mock, in_family_patch_mock, checked_call_mock): - test_cases = [ - { - "name": "SUSE Case", - "os": OSConst.SUSE_FAMILY, - "cheked_call_result": "5 new packages to install" - }, - { - "name": "REDHAT Case", - "os": OSConst.REDHAT_FAMILY, - "cheked_call_result": "Error:" - }, - { - "name": "UBUNTU Case", - "os": OSConst.UBUNTU_FAMILY, - "cheked_call_result": "E:" - } - ] - - for test_case in test_cases: - in_family_patch_mock.side_effect = lambda current_family, family: family == test_case["os"] - checked_call_mock.return_value = (0, "OK.") - - # happy scenario - self.assertTrue(packages_analyzer.verifyDependencies(), "test_verify_dependency failed on '%s'" % test_case["name"]) - - # unhappy scenario - checked_call_mock.return_value = (0, test_case["cheked_call_result"]) - self.assertFalse(packages_analyzer.verifyDependencies(), "test_verify_dependency failed on '%s'" % test_case["name"]) - - try: - in_family_patch_mock.side_effect = lambda current_family, family: False - packages_analyzer.verifyDependencies() - self.assertTrue(False, "Wrong handling of unknown operation system") - except Fail: - pass - - @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call") - @patch.object(OSCheck, "is_in_family") - def test_vefify_dependency_redhat(self, in_family_patch_mock, checked_call_mock): - in_family_patch_mock.side_effect = lambda current_family, family: family == OSConst.REDHAT_FAMILY - checked_call_mock.return_value = (0, "OK.") - pass - - @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call") - @patch.object(OSCheck, "is_in_family") - def test_vefify_dependency_ubuntu(self, in_family_patch_mock, checked_call_mock): - in_family_patch_mock.side_effect = lambda current_family, family: family == OSConst.UBUNTU_FAMILY - checked_call_mock.return_value = (0, "OK.") - pass - - def test_perform_package_analysis(self): - installedPackages = [ - ["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"], - ["hadoop-b", "2.3", "HDP-epel"], ["epel", "3.1", "HDP-epel"], ["epel-2", "3.1", "HDP-epel"], - ["hadoop-c", "2.3", "Ambari"], ["ambari-s", "3.1", "Ambari"], - ["ganglia", "2.3", "GANGLIA"], ["rrd", "3.1", "RRD"], - ["keeper-1", "2.3", "GANGLIA"], ["keeper-2", "3.1", "base"],["def-def.x86", "2.2", "DEF.3"], - ["def.1", "1.2", "NewDEF"] - ] - availablePackages = [ - ["hadoop-d", "2.3", "HDP"], ["zk-2", "3.1", "HDP"], ["pig", "3.1", "HDP"], - ["epel-3", "2.3", "HDP-epel"], ["hadoop-e", "3.1", "HDP-epel"], - ["ambari-a", "3.1", "Ambari"], - ["keeper-3", "3.1", "base"] - ] - - packagesToLook = ["^webhcat.*$", "^hadoop.*$", "^.+-def.*$"] - reposToIgnore = ["ambari"] - additionalPackages = ["ganglia", "rrd"] - - repos = [] - packages_analyzer.getInstalledRepos(packagesToLook, installedPackages + availablePackages, reposToIgnore, repos) - self.assertEqual(3, len(repos)) - expected = ["HDP", "HDP-epel", "DEF.3"] - for repo in expected: - self.assertTrue(repo in repos) - - packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages) - self.assertEqual(5, len(packagesInstalled)) - expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"] - for repo in expected: - self.assertTrue(repo in packagesInstalled) - - additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( - additionalPackages, installedPackages) - self.assertEqual(2, len(additionalPkgsInstalled)) - expected = ["ganglia", "rrd"] - for additionalPkg in expected: - self.assertTrue(additionalPkg in additionalPkgsInstalled) - - allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) - self.assertEqual(7, len(allPackages)) - expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "ganglia", "rrd", "def-def.x86"] - for package in expected: - self.assertTrue(package in allPackages) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-agent/src/test/python/resource_management/TestScript.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestScript.py b/ambari-agent/src/test/python/resource_management/TestScript.py index d531314..75726d6 100644 --- a/ambari-agent/src/test/python/resource_management/TestScript.py +++ b/ambari-agent/src/test/python/resource_management/TestScript.py @@ -21,7 +21,7 @@ import StringIO import sys, pprint from resource_management.libraries.script import Script from resource_management.core.environment import Environment -from mock.mock import patch +from mock.mock import patch, MagicMock from stacks.utils.RMFTestCase import * import logging @@ -63,25 +63,28 @@ class TestScript(RMFTestCase): # Testing config without any keys with Environment(".", test_mode=True) as env: - script = Script() - Script.config = no_packages_config - script.install_packages(env) + with patch("resource_management.libraries.script.get_provider", return_value=MagicMock()): + script = Script() + Script.config = no_packages_config + script.install_packages(env) resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, "[]") # Testing empty package list with Environment(".", test_mode=True) as env: - script = Script() - Script.config = empty_config - script.install_packages(env) + with patch("resource_management.libraries.script.get_provider", return_value=MagicMock()): + script = Script() + Script.config = empty_config + script.install_packages(env) resource_dump = pprint.pformat(env.resource_list) self.assertEquals(resource_dump, "[]") # Testing installing of a list of packages with Environment(".", test_mode=True) as env: - script = Script() - Script.config = dummy_config - script.install_packages("env") + with patch("resource_management.libraries.script.get_provider", return_value=MagicMock()): + script = Script() + Script.config = dummy_config + script.install_packages("env") resource_dump = pprint.pformat(env.resource_list) self.assertEqual(resource_dump, '[Package[\'hbase\'], Package[\'yet-another-package\']]') http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/ambari_commons/shell.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/shell.py b/ambari-common/src/main/python/ambari_commons/shell.py index 8d26599..021e495 100644 --- a/ambari-common/src/main/python/ambari_commons/shell.py +++ b/ambari-common/src/main/python/ambari_commons/shell.py @@ -1,6 +1,6 @@ # !/usr/bin/env python -''' +""" Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -16,29 +16,27 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -''' +""" import logging import subprocess import os -import tempfile import signal -import sys import threading import time -import traceback -import pprint -import platform +from contextlib import contextmanager from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl logger = logging.getLogger() -shellRunner = None threadLocal = threading.local() gracefull_kill_delay = 5 # seconds between SIGTERM and SIGKILL +# default timeout for async invoked processes +TIMEOUT_SECONDS = 300 + tempFiles = [] @@ -63,6 +61,7 @@ class _dict_to_object: def kill_process_with_children(parent_pid): shellRunnerWindows().run(["taskkill", "/F", "/T", "/PID", "{0}".format(parent_pid)]) + class shellRunner(object): def run(self, script, user=None): pass @@ -70,6 +69,135 @@ class shellRunner(object): def runPowershell(self, file=None, script_block=None, args=[]): raise NotImplementedError() + +def launch_subprocess(command): + """ + Process launch helper + + :param command Command to execute + :type command list[str]|str + :return Popen object + """ + is_shell = not isinstance(command, (list, tuple)) + return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=is_shell, close_fds=True) + + +def watchdog_func(event, cmd, exec_timeout): + """ + Watchdog function for subprocess executors + + :type event Event + :type cmd Popen + :type exec_timeout int + + + Usage example: + event = threading.Event() + + cmd = Popen(...) + + thread = threading.Thread(target=watchdog_func, args=(event, cmd, execution_timeout,)) + thread.start() + + ....cmd.communicate() or any another processing.... + + event.set() + thread.join() + ......result code.... + """ + event.wait(exec_timeout) + if cmd.returncode is None: + logger.error("Task timed out and will be killed") + kill_process_with_children(cmd.pid) + + +def subprocess_with_timeout(command, execution_timeout=None): + """ + Run command with limited time for execution, after timeout command would be killed + + :param command Command to execute + :param execution_timeout execution time limit in seconds. Defaulting to TIMEOUT_SECONDS global constant + + :type command list[str]|str + :type execution_timeout int + :rtype dict + """ + event = threading.Event() + + if execution_timeout is None: + execution_timeout = TIMEOUT_SECONDS + + os_stat = launch_subprocess(command) + logger.debug("Launching watchdog thread") + + event.clear() + + thread = threading.Thread(target=watchdog_func, args=(event, os_stat, execution_timeout,)) + thread.start() + + out, err = os_stat.communicate() + + result = { + "out": out, + "err": err, + "retCode": os_stat.returncode + } + + event.set() + thread.join() + return result + + +@contextmanager +def process_executor(command, timeout=None, error_callback=None): + """ + Context manager for command execution + + :type command list|str + :type timeout None|int + :type error_callback func + + :return stdout stream + + Usage example: + + Option 1. Basic + with process_executor(["ls", "-la]) as stdout: + for line in stdout: + print line + + Option 2. Extended + def error_handler(command, error_log, exit_code): + print "Command '{}' failed".format(command) + print "Exit Code: {} StdOut: {} \n".format(exit_code, "\n".join(error_log)) + + with process_executor(["ls", "-la], timeout=10, error_callback=error_handler) as stdout: + for line in stdout: + print line + + """ + if not timeout: + timeout = TIMEOUT_SECONDS + + event = threading.Event() + cmd = launch_subprocess(command) + + thread = threading.Thread(target=watchdog_func, args=(event, cmd, timeout,)) + thread.start() + + yield cmd.stdout + + exit_code = cmd.poll() + event.set() + thread.join() + + if exit_code is None: + kill_process_with_children(cmd.pid) + + if error_callback and exit_code and exit_code > 0: + error_callback(command, cmd.stderr.readlines(), exit_code) + + @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) class shellRunnerWindows(shellRunner): # Run any command http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/core/providers/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/__init__.py index ac6ee16..2091b03 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/__init__.py @@ -20,12 +20,13 @@ Ambari Agent """ -__all__ = ["Provider", "find_provider"] +__all__ = ["Provider", "find_provider", "get_provider"] from resource_management.core.exceptions import Fail from resource_management.libraries.providers import PROVIDERS as LIBRARY_PROVIDERS from ambari_commons.os_check import OSCheck + class Provider(object): def __init__(self, resource): self.resource = resource @@ -103,3 +104,42 @@ def find_provider(env, resource, class_path=None): raise Fail("Unable to find provider for %s as %s" % (resource, class_path)) mod = __import__(mod_path, {}, {}, [class_name]) return getattr(mod, class_name) + + +def get_provider(resource_type): + """ + Looking for {resource_type} provider implementation for current os and returning ready to use instance + + :param resource_type existing provider type, case sensitive + :type resource_type str + + :raise Fail + :return Provider instance + """ + os_family = OSCheck.get_os_family() + providers = [PROVIDERS, LIBRARY_PROVIDERS] + os_family_provider = None + class_path = None + + for provider in providers: + if os_family in provider: + os_family_provider = provider[os_family] + else: + # take care of os extensions + for family in provider: + if OSCheck.is_in_family(os_family, family): + os_family_provider = provider[family] + + if os_family_provider and resource_type in os_family_provider: + class_path = os_family_provider[resource_type] + break + if resource_type in provider["default"]: + class_path = provider["default"][resource_type] + break + + try: + mod_path, class_name = class_path.rsplit('.', 1) + except ValueError: + raise Fail("Unable to find provider for %s as %s" % (resource_type, class_path)) + mod = __import__(mod_path, {}, {}, [class_name]) + return getattr(mod, class_name)(None) http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index a1d3e02..256cf8f 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -20,16 +20,18 @@ Ambari Agent """ -import subprocess import time import re import logging +import sys + +import subprocess from resource_management.core.exceptions import ExecutionFailed from resource_management.core.providers import Provider from resource_management.core.logger import Logger -from resource_management.core.utils import suppress_stdout from resource_management.core import shell +from ambari_commons import shell as ac_shell PACKAGE_MANAGER_LOCK_ACQUIRED_MSG = "Cannot obtain lock for Package manager. Retrying after {0} seconds. Reason: {1}" @@ -40,13 +42,13 @@ class PackageProvider(Provider): def __init__(self, *args, **kwargs): super(PackageProvider, self).__init__(*args, **kwargs) - def install_package(self, name, version): + def install_package(self, name, use_repos=set(), skip_repos=set(), is_upgrade=False): raise NotImplementedError() - def remove_package(self, name): + def remove_package(self, name, ignore_dependencies=False): raise NotImplementedError() - def upgrade_package(self, name, version): + def upgrade_package(self, name, use_repos=set(), skip_repos=set(), is_upgrade=True): raise NotImplementedError() def action_install(self): @@ -67,9 +69,175 @@ class PackageProvider(Provider): else: return self.resource.package_name + def get_available_packages_in_repos(self, repositories): + """ + Gets all (both installed and available) packages that are available at given repositories. + :param repositories: from command configs like config['repositoryFile']['repositories'] + :return: installed and available packages from these repositories + """ + raise NotImplementedError() + + def normalize_select_tool_versions(self, versions): + """ + Function expect output from get_all_package_versions + + :type versions str|list|set + :rtype list + """ + raise NotImplementedError() + def get_repo_update_cmd(self): raise NotImplementedError() + def get_all_package_versions(self, pkg_name): + """ + :rtype list[str] + """ + raise NotImplementedError() + + def installed_pkgs_by_name(self, all_installed_packages, pkg_name): + return list([i[0] for i in all_installed_packages if i[0].startswith(pkg_name)]) + + def all_installed_packages(self, from_unknown_repo=False): + """ + Return all installed packages in the system except packages in REPO_URL_EXCLUDE + + :arg from_unknown_repo return packages from unknown repos + :type from_unknown_repo bool + + :return result_type formatted list of packages + """ + raise NotImplementedError() + + def all_available_packages(self, result_type=list, group_by_index=-1): + """ + Return all available packages in the system except packages in REPO_URL_EXCLUDE + + :arg result_type Could be list or dict, defines type of returning value + :arg group_by_index index of element in the __packages_reader result, which would be used as key + :return result_type formatted list of packages, including installed and available in repos + + :type result_type type + :type group_by_index int + :rtype list|dict + """ + raise NotImplementedError() + + def get_installed_repos(self, hint_packages, all_packages, ignore_repos): + """ + Gets all installed repos by name based on repos that provide any package + contained in hintPackages + Repos starting with value in ignoreRepos will not be returned + hintPackages must be regexps. + """ + all_repos = [] + repo_list = [] + + for hintPackage in hint_packages: + for item in all_packages: + if re.match(hintPackage, item[0]) and not item[2] in all_repos: + all_repos.append(item[2]) + + for repo in all_repos: + ignore = False + for ignoredRepo in ignore_repos: + if self.name_match(ignoredRepo, repo): + ignore = True + if not ignore: + repo_list.append(repo) + + return repo_list + + def get_installed_pkgs_by_repo(self, repos, ignore_packages, installed_packages): + """ + Get all the installed packages from the repos listed in repos + """ + packages_from_repo = [] + packages_to_remove = [] + for repo in repos: + sub_result = [] + for item in installed_packages: + if repo == item[2]: + sub_result.append(item[0]) + packages_from_repo = list(set(packages_from_repo + sub_result)) + + for package in packages_from_repo: + keep_package = True + for ignorePackage in ignore_packages: + if self.name_match(ignorePackage, package): + keep_package = False + break + if keep_package: + packages_to_remove.append(package) + return packages_to_remove + + def get_installed_pkgs_by_names(self, pkg_names, all_packages_list=None): + """ + Gets all installed packages that start with names in pkgNames + :type pkg_names list[str] + :type all_packages_list list[str] + """ + if not all_packages_list: + all_packages_list = self.all_installed_packages() + + packages = [] + for pkg_name in pkg_names: + sub_result = self.installed_pkgs_by_name(all_packages_list, pkg_name) + packages.extend(sub_result) + + return list(set(packages)) + + def get_package_details(self, installed_packages, found_packages): + """ + Gets the name, version, and repoName for the packages + :type installed_packages list[tuple[str,str,str]] + :type found_packages list[str] + """ + package_details = [] + + for package in found_packages: + pkg_detail = {} + for installed_package in installed_packages: + if package == installed_package[0]: + pkg_detail['name'] = installed_package[0] + pkg_detail['version'] = installed_package[1] + pkg_detail['repoName'] = installed_package[2] + + package_details.append(pkg_detail) + + return package_details + + def get_repos_to_remove(self, repos, ignore_list): + repos_to_remove = [] + for repo in repos: + add_to_remove_list = True + for ignore_repo in ignore_list: + if self.name_match(ignore_repo, repo): + add_to_remove_list = False + continue + if add_to_remove_list: + repos_to_remove.append(repo) + return repos_to_remove + + def get_installed_package_version(self, package_name): + raise NotImplementedError() + + def verify_dependencies(self): + """ + Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated + package installation process or invalid packages state after manual modification of packages list on the host + + :return True if no dependency issues found, False if dependency issue present + :rtype bool + """ + raise NotImplementedError() + + def name_match(self, lookup_name, actual_name): + tokens = actual_name.strip().lower() + lookup_name = lookup_name.lower() + + return " " not in lookup_name and lookup_name in tokens + def is_locked_output(self, out): return False @@ -77,7 +245,7 @@ class PackageProvider(Provider): return False def get_logoutput(self): - return self.resource.logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or self.resource.logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG) + return self.resource.logoutput is True and Logger.logger.isEnabledFor(logging.INFO) or self.resource.logoutput is None and Logger.logger.isEnabledFor(logging.DEBUG) def call_with_retries(self, cmd, **kwargs): return self._call_with_retries(cmd, is_checked=False, **kwargs) @@ -85,8 +253,13 @@ class PackageProvider(Provider): def checked_call_with_retries(self, cmd, **kwargs): return self._call_with_retries(cmd, is_checked=True, **kwargs) + def checked_call(self, cmd, **kwargs): + return shell.checked_call(cmd, **kwargs) + def _call_with_retries(self, cmd, is_checked=True, **kwargs): func = shell.checked_call if is_checked else shell.call + code, out = -1, None + # at least do one retry, to run after repository is cleaned try_count = 2 if self.resource.retry_count < 2 else self.resource.retry_count @@ -109,6 +282,32 @@ class PackageProvider(Provider): return code, out + def _call_with_timeout(self, cmd): + """ + :type cmd list[str]|str + :rtype dict + """ + try: + return ac_shell.subprocess_with_timeout(cmd) + except Exception as e: + Logger.error("Unexpected error:" + str(e)) + + return None + + def _executor_error_handler(self, command, error_log, exit_code): + """ + Error handler for ac_shell.process_executor + + :type command list|str + :type error_log list + :type exit_code int + """ + if isinstance(command, (list, tuple)): + command = " ".join(command) + + Logger.error("Command execution error: command = \"{0}\", exit code = {1}, stderr = {2}".format( + command, exit_code, "\n".join(error_log))) + def _handle_retries(self, cmd, code, out, is_first_time, is_last_time): # handle first failure in a special way (update repo metadata after it, so next try has a better chance to succeed) if is_first_time and code and not self.is_locked_output(out): @@ -140,34 +339,28 @@ class PackageProvider(Provider): Logger.info("Retrying to install package %s after %d seconds" % (name, self.resource.retry_sleep)) - def yum_check_package_available(self, name): - """ - Does the same as rpm_check_package_avaiable, but faster. - However need root permissions. - """ - import yum # Python Yum API is much faster then other check methods. (even then "import rpm") - yb = yum.YumBase() - name_regex = re.escape(name).replace("\\?", ".").replace("\\*", ".*") + '$' - regex = re.compile(name_regex) - - with suppress_stdout(): - package_list = yb.rpmdb.simplePkgList() - - for package in package_list: - if regex.match(package[0]): - return True - - return False - + +class RPMBasedPackageProvider(PackageProvider): + """ + RPM Based abstract package provider + """ + def rpm_check_package_available(self, name): import rpm # this is faster then calling 'rpm'-binary externally. ts = rpm.TransactionSet() packages = ts.dbMatch() - + name_regex = re.escape(name).replace("\\?", ".").replace("\\*", ".*") + '$' regex = re.compile(name_regex) - + for package in packages: if regex.match(package['name']): return True return False + + def get_installed_package_version(self, package_name): + result = self.checked_call("rpm -q --queryformat '%{{version}}-%{{release}}' {0} | sed -e 's/\.el[0-9]//g'".format(package_name), stderr=subprocess.PIPE) + if len(result) >= 2: + return result[1] + + return None http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index aa80557..65caeff 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -21,15 +21,17 @@ Ambari Agent import os import tempfile -import shutil import re +import subprocess +from ambari_commons.constants import AMBARI_SUDO_BINARY +from ambari_commons.shell import process_executor from resource_management.core.providers.package import PackageProvider from resource_management.core import shell from resource_management.core import sudo from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger -from resource_management.core.exceptions import Fail + INSTALL_CMD_ENV = {'DEBIAN_FRONTEND':'noninteractive'} INSTALL_CMD = { @@ -46,6 +48,24 @@ REMOVE_WITHOUT_DEPENDENCIES_CMD = ['/usr/bin/dpkg', '--remove', '--ignore-depend APT_SOURCES_LIST_DIR = "/etc/apt/sources.list.d" CHECK_CMD = "dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^%s$" +VERIFY_DEPENDENCY_CMD = ['/usr/bin/apt-get', '-qq', 'check'] + +REPO_URL_EXCLUDE = "ubuntu.com" +CONFIGURATION_DUMP_CMD = [AMBARI_SUDO_BINARY, "apt-config", "dump"] +ALL_AVAILABLE_PACKAGES_DUMP_CMD = [AMBARI_SUDO_BINARY, "apt-cache", "dump"] +ALL_INSTALLED_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "dpkg", "-l"] + +# base command output sample: +# ----------------------------- +# +# select | 2.6.3.0-63 | http://host/ubuntu16/2.x/BUILDS/2.6.3.0-63 main amd64 Packages +# select | 2.6.3.0-57 | http://host/ubuntu16/2.x/BUILDS/2.6.3.0-57 main amd64 Packages +# select | 2.6.3.0-55 | http://host/ubuntu16/2.x/BUILDS/2.6.3.0-55 main amd64 Packages + +# repository update we doing at RepositoryProvider +LIST_ALL_SELECT_TOOL_PACKAGES_CMD = "apt-cache madison {pkg_name} 2>/dev/null|grep '^{pkg_name}' | cut -d '|' -f 2" +SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\d*).*") # xx.xx.xx.xx(-xxxx) + def replace_underscores(function_to_decorate): def wrapper(*args): @@ -57,8 +77,269 @@ def replace_underscores(function_to_decorate): class AptProvider(PackageProvider): + def __parse_select_tool_version(self, v): + """ + :type v str + """ + matches = SELECT_TOOL_VERSION_PATTERN.findall(v.strip()) + return matches[0] if matches else None + + def normalize_select_tool_versions(self, versions): + """ + Function expect output from get_all_package_versions + + :type versions str|list|set + :rtype list + """ + if isinstance(versions, str): + versions = [versions] + + return [self.__parse_select_tool_version(i) for i in versions] + + def __config_reader(self, stream): + """ + apt-config dump command parser + + Function consumes io.TextIOBase compatible objects as input and return iterator with parsed items + + :type stream collections.Iterable + :return tuple(key, value) + + Usage: + for key, value in __config_reader(text_stream): + ... + + Parsing subject: + + PROPERTY ""; + PROPERTY::ITEM1:: "value"; + ..... + + """ + for line in stream: + key, value = line.strip().split(" ", 1) + key = key.strip("::") + value = value.strip(";").strip("\"").strip() + if not value: + continue + + yield key, value + + def __packages_reader(self, stream): + """ + apt-cache dump command parser + + Function consumes io.TextIOBase compatible objects as input and return iterator with parsed items + + :type stream collections.Iterable + :return tuple(package name, version, parsed repo list file) + + Usage: + for package, version, repo_file_path in __packages_reader(text_stream): + ... + + Parsing subject: + + Package: test_package + Version: 0.1.1-0 + File: /var/lib/apt/lists/some_site_dists_apt_main_binary-amd64_Packages.gz + Description Language: + File: /var/lib/apt/lists/some_site_dists_apt_main_binary-amd64_Packages.gz + MD5: 000000000000000000000000000 + """ + fields = {"Package": 0, "Version": 1, "File": 2} + field_names = fields.keys() + field_count = len(field_names) + item_set = [None] * field_count + + for line in stream: + line = line.strip() + + if not line: + continue + + values = line.split(":", 1) + if len(values) != 2: + continue + + field, value = values + value = value[1:] + + if field in field_names: + if field == "File": + value = value.rpartition("/")[2] + elif field == "Package": + item_set = [None] * field_count # reset fields which were parsed before new block + item_set[fields[field]] = value + else: + continue + + if None not in item_set: + yield item_set + item_set = [None] * field_count + + def __packages_installed_reader(self, stream): + """ + dpkg -l command parser + + Function consumes io.TextIOBase compatible objects as input and return iterator with parsed items + + :type stream collections.Iterable + :return tuple(package name, version) + + Usage: + for package, version in __packages_installed_reader(text_stream): + ... + + Parsing subject: + + ||/ Name Version Architecture Description + +++-=================================-=====================-=====================-====================== + ii package1 version1 all description1 + ii package2 version2 all description2 + """ + for line in stream: + line = line.lstrip() + + if line[:2] != "ii": + continue + + line = line[2:].lstrip() + data = line.partition(" ") + pkg_name = data[0] + version = data[2].strip().partition(" ")[0] + + if pkg_name and version: + yield pkg_name, version + + def _lookup_packages(self, command): + """ + :type command list[str]|str + """ + packages = [] + result = self._call_with_timeout(command) + + if result and 0 == result['retCode']: + for x in result['out'].split('\n'): + if x.strip(): + packages.append(x.split(' ')) + + return packages + + def all_installed_packages(self, from_unknown_repo=False): + """ + Return all installed packages in the system except packages in REPO_URL_EXCLUDE + + :arg from_unknown_repo return packages from unknown repos + :type from_unknown_repo bool + + :return result_type formatted list of packages + """ + packages = [] + available_packages = self.all_available_packages(result_type=dict, group_by_index=0) + + with process_executor(ALL_INSTALLED_PACKAGES_CMD, error_callback=self._executor_error_handler) as output: + for package, version in self.__packages_installed_reader(output): + if not from_unknown_repo and package in available_packages: + packages.append(available_packages[package]) + + if package not in available_packages: + packages.append([package, version, "installed"]) # case, when some package not belongs to any known repo + + return packages + + def all_available_packages(self, result_type=list, group_by_index=-1): + """ + Return all available packages in the system except packages in REPO_URL_EXCLUDE + + :arg result_type Could be list or dict, defines type of returning value + :arg group_by_index index of element in the __packages_reader result, which would be used as key + :return result_type formatted list of packages, including installed and available in repos + + :type result_type type + :type group_by_index int + :rtype list|dict + """ + if result_type is not list and result_type is not dict: + raise TypeError("result_type argument must be list or dict only") + packages = result_type() + + with process_executor(ALL_AVAILABLE_PACKAGES_DUMP_CMD, error_callback=self._executor_error_handler) as output: + for pkg_item in self.__packages_reader(output): + if REPO_URL_EXCLUDE not in pkg_item[2]: + if result_type is list: + packages.append(pkg_item) + elif result_type is dict: + packages[pkg_item[group_by_index]] = pkg_item + + return packages + + def get_available_packages_in_repos(self, repositories): + """ + Gets all (both installed and available) packages that are available at given repositories. + :param repositories: from command configs like config['repositoryFile']['repositories'] + :return: installed and available packages from these repositories + """ + + filtered_packages = [] + packages = self.all_available_packages() + + for repo in repositories: + repo_url_part = repo['baseUrl'].replace("http://";, "").replace("/", "_") + + for package in packages: + if repo_url_part in package[2]: + filtered_packages.append(package[0]) + + return filtered_packages + + def get_all_package_versions(self, pkg_name): + """ + :type pkg_name str + """ + command = LIST_ALL_SELECT_TOOL_PACKAGES_CMD.replace("{pkg_name}", pkg_name) + result = self._call_with_timeout(command) + + if result["retCode"] == 0: + return result["out"].split(os.linesep) + + return None + + def package_manager_configuration(self): + """ + Reading apt configuration + + :return dict with apt properties + """ + with process_executor(CONFIGURATION_DUMP_CMD, error_callback=self._executor_error_handler) as output: + configuration = list(self.__config_reader(output)) + + return dict(configuration) + + def get_installed_package_version(self, package_name): + code, out, err = self.checked_call("dpkg -s {0} | grep Version | awk '{{print $2}}'".format(package_name), stderr=subprocess.PIPE) + return out + + def verify_dependencies(self): + """ + Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated + package installation process or invalid packages state after manual modification of packages list on the host + + :return True if no dependency issues found, False if dependency issue present + :rtype bool + """ + code, out = self.checked_call(VERIFY_DEPENDENCY_CMD, sudo=True) + pattern = re.compile("has missing dependency|E:") + + if code or (out and pattern.search(out)): + err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '%s' returned %s. %s" % (VERIFY_DEPENDENCY_CMD, code, out)) + Logger.error(err_msg) + return False + + return True + @replace_underscores - def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): + def install_package(self, name, use_repos=set(), skip_repos=set(), is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] copied_sources_files = [] @@ -100,7 +381,7 @@ class AptProvider(PackageProvider): return REPO_UPDATE_CMD @replace_underscores - def upgrade_package(self, name, use_repos=[], skip_repos=[], is_upgrade=True): + def upgrade_package(self, name, use_repos=set(), skip_repos=set(), is_upgrade=True): return self.install_package(name, use_repos, skip_repos, is_upgrade) @replace_underscores http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index a2f0533..2e8422b 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -20,10 +20,14 @@ Ambari Agent """ -from resource_management.core.providers.package import PackageProvider +from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management.core.providers.package import RPMBasedPackageProvider from resource_management.core import shell from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger +from resource_management.core.utils import suppress_stdout + +import re import os INSTALL_CMD = { @@ -38,10 +42,196 @@ REMOVE_CMD = { REMOVE_WITHOUT_DEPENDENCIES_CMD = ['rpm', '-e', '--nodeps'] -REPO_UPDATE_CMD = ['/usr/bin/yum', 'clean','metadata'] +REPO_UPDATE_CMD = ['/usr/bin/yum', 'clean', 'metadata'] +ALL_INSTALLED_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "installed"] +ALL_AVAILABLE_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "yum", "list", "available"] +VERIFY_DEPENDENCY_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'dependencies'] + +# base command output sample: +# ----------------------------- +# select.noarch 2.5.6.0-40.el6 REPO-2.5 +# select.noarch 2.6.3.0-56 REPO-2.6.3.0-56 +# select.noarch 2.6.3.0-57 REPO-2.6.3.0-57 +# select.noarch 2.6.3.0-63 REPO-2.6.3.0 +# select.noarch 2.6.3.0-63 REPO-2.6.3.0-63 + +LIST_ALL_SELECT_TOOL_PACKAGES_CMD = "yum list all --showduplicates|grep -v '@' |grep '^{pkg_name}'|awk '{print $2}'" +SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\d*).*") # xx.xx.xx.xx(-xxxx) + + +class YumProvider(RPMBasedPackageProvider): + + def get_available_packages_in_repos(self, repositories): + """ + Gets all (both installed and available) packages that are available at given repositories. + :param repositories: from command configs like config['repositoryFile']['repositories'] + :return: installed and available packages from these repositories + """ + available_packages = [] + installed_packages = [] + available_packages_in_repos = [] + repo_ids = [repository['repoId'] for repository in repositories] + + for repo in repo_ids: + available_packages.extend(self._lookup_packages( + [AMBARI_SUDO_BINARY, "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo], 'Available Packages')) + installed_packages.extend(self._get_installed_packages(repo)) + + available_packages_in_repos += [package[0] for package in available_packages + installed_packages] + return available_packages_in_repos + + def get_all_package_versions(self, pkg_name): + """ + :type pkg_name str + """ + command = LIST_ALL_SELECT_TOOL_PACKAGES_CMD.replace("{pkg_name}", pkg_name) + result = self._call_with_timeout(command) + + if result["retCode"] == 0: + return result["out"].split(os.linesep) + + return None + + def __parse_select_tool_version(self, v): + """ + :type v str + """ + matches = SELECT_TOOL_VERSION_PATTERN.findall(v.strip()) + return matches[0] if matches else None + + def normalize_select_tool_versions(self, versions): + """ + Function expect output from get_all_package_versions + + :type versions str|list|set + :rtype list + """ + if isinstance(versions, str): + versions = [versions] + + return [self.__parse_select_tool_version(i) for i in versions] + + def _get_installed_packages(self, repo_filter=None): + """ + Returning list of the installed packages with possibility to filter them by name + :param repo_filter: repository name + + :type repo_filter str|None + :rtype list[list,] + """ + packages = [] + cmd_filter = "| grep \"{0}\"".format(repo_filter) if repo_filter else "" + + # tr '\n' '#' %s | sed -e 's/# / /g' | tr '#' '\n' - fix yum formatted output for default console width + cmd = AMBARI_SUDO_BINARY + " yum list installed {filter}|tr '\\n' '#' | sed -e 's/# / /g' | tr '#' '\\n'|awk '{printf \"%s;%s;%s\\n\", $1,$2,$3}'".replace("{filter}", cmd_filter) + result = self._call_with_timeout(cmd) + col_sep = ";" + + """ + command would return everything in following format: + + Loaded;plugins:;fastestmirror + Installed;Packages; + package_name;version;@Repo + .... + """ + + if result and 0 == result['retCode']: + raw_pkgs = result['out'].split("\n") + + for line in raw_pkgs: + package_item = line.split(col_sep) + + if len(package_item) < 3: + continue + elif not package_item[2].startswith("@"): + continue -class YumProvider(PackageProvider): - def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): + package_item[2] = package_item[2][1:] + packages.append(package_item) + + return packages + + def _lookup_packages(self, command, skip_till): + """ + :type command list[str] + :type skip_till str|None + """ + packages = [] + + result = self._call_with_timeout(command) + + if result and 0 == result['retCode']: + lines = result['out'].split('\n') + lines = [line.strip() for line in lines] + items = [] + if skip_till: + skip_index = 3 + for index in range(len(lines)): + if skip_till in lines[index]: + skip_index = index + 1 + break + else: + skip_index = 0 + + for line in lines[skip_index:]: + items = items + line.strip(' \t\n\r').split() + + for i in range(0, len(items), 3): + if '.' in items[i]: + items[i] = items[i][:items[i].rindex('.')] + if items[i + 2].find('@') == 0: + items[i + 2] = items[i + 2][1:] + packages.append(items[i:i + 3]) + + return packages + + def all_available_packages(self, result_type=list, group_by_index=-1): + """ + Return all available packages in the system except packages in REPO_URL_EXCLUDE + + :arg result_type Could be list or dict, defines type of returning value + :arg group_by_index index of element in the __packages_reader result, which would be used as key + :return result_type formatted list of packages, including installed and available in repos + + :type result_type type + :type group_by_index int + :rtype list|dict + """ + # ToDo: move to iterative package lookup (check apt provider for details) + return self._lookup_packages(ALL_AVAILABLE_PACKAGES_CMD, "Available Packages") + + def all_installed_packages(self, from_unknown_repo=False): + """ + Return all installed packages in the system except packages in REPO_URL_EXCLUDE + + :arg from_unknown_repo return packages from unknown repos + :type from_unknown_repo bool + + :return result_type formatted list of packages + """ + # ToDo: move to iterative package lookup (check apt provider for details) + return self._lookup_packages(ALL_INSTALLED_PACKAGES_CMD, "Installed Packages") + + def verify_dependencies(self): + """ + Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated + package installation process or invalid packages state after manual modification of packages list on the host + + :return True if no dependency issues found, False if dependency issue present + :rtype bool + """ + code, out = self.checked_call(VERIFY_DEPENDENCY_CMD, sudo=True) + pattern = re.compile("has missing requires|Error:") + + if code or (out and pattern.search(out)): + err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '%s' returned %s. %s" % (VERIFY_DEPENDENCY_CMD, code, out)) + Logger.error(err_msg) + return False + + return True + + def install_package(self, name, use_repos=set(), skip_repos=set(), is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: @@ -54,10 +244,10 @@ class YumProvider(PackageProvider): else: Logger.info("Skipping installation of existing package %s" % (name)) - def upgrade_package(self, name, use_repos=[], skip_repos=[], is_upgrade=True): + def upgrade_package(self, name, use_repos=set(), skip_repos=set(), is_upgrade=True): return self.install_package(name, use_repos, skip_repos, is_upgrade) - def remove_package(self, name, ignore_dependencies = False): + def remove_package(self, name, ignore_dependencies=False): if self._check_existence(name): if ignore_dependencies: cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [name] @@ -98,4 +288,23 @@ class YumProvider(PackageProvider): return self.yum_check_package_available(name) else: return self.rpm_check_package_available(name) - + + + def yum_check_package_available(self, name): + """ + Does the same as rpm_check_package_avaiable, but faster. + However need root permissions. + """ + import yum # Python Yum API is much faster then other check methods. (even then "import rpm") + yb = yum.YumBase() + name_regex = re.escape(name).replace("\\?", ".").replace("\\*", ".*") + '$' + regex = re.compile(name_regex) + + with suppress_stdout(): + package_list = yb.rpmdb.simplePkgList() + + for package in package_list: + if regex.match(package[0]): + return True + + return False http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index f3abdb5..1e5fb90 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -19,12 +19,13 @@ limitations under the License. Ambari Agent """ - -from resource_management.core.providers.package import PackageProvider -from resource_management.core import shell +from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management.core.providers.package import RPMBasedPackageProvider from resource_management.core.shell import string_cmd_from_args_list from resource_management.core.logger import Logger -from resource_management.core.utils import suppress_stdout + +import re +import os INSTALL_CMD = { True: ['/usr/bin/zypper', 'install', '--auto-agree-with-licenses', '--no-confirm'], @@ -40,8 +41,144 @@ REMOVE_WITHOUT_DEPENDENCIES_CMD = ['rpm', '-e', '--nodeps'] REPO_UPDATE_CMD = ['/usr/bin/zypper', 'clean'] LIST_ACTIVE_REPOS_CMD = ['/usr/bin/zypper', 'repos'] +ALL_INSTALLED_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--installed-only", "--details"] +ALL_AVAILABLE_PACKAGES_CMD = [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--uninstalled-only", "--details"] +VERIFY_DEPENDENCY_CMD = ['/usr/bin/zypper', '--quiet', '--non-interactive', 'verify', '--dry-run'] + +# base command output sample: +# ----------------------------- +# +# S | Name | Type | Version | Arch | Repository +# --+--------+---------+-------------+--------+--------------- +# i | select | package | 2.6.3.0-60 | noarch | REPO-2.6.3.0-60 +# v | select | package | 2.6.3.0-57 | noarch | REPO-2.6.3.0-57 +# v | select | package | 2.6.1.0-129 | noarch | REPO-2.6.1.0 +# v | select | package | 2.5.6.0-40 | noarch | REPO-2.5 + +LIST_ALL_SELECT_TOOL_PACKAGES_CMD = "zypper -q search -s {pkg_name}|grep '|' | grep -v 'Repository'| cut -d '|' -f 4" +SELECT_TOOL_VERSION_PATTERN = re.compile("(\d{1,2}\.\d{1,2}\.\d{1,2}\.\d{1,2}-*\d*).*") # xx.xx.xx.xx(-xxxx) + + +class ZypperProvider(RPMBasedPackageProvider): + + def get_available_packages_in_repos(self, repositories): + """ + Gets all (both installed and available) packages that are available at given repositories. + :param repositories: from command configs like config['repositoryFile']['repositories'] + :return: installed and available packages from these repositories + """ + + available_packages = [] + available_packages_in_repos = [] + repo_ids = [repository['repoId'] for repository in repositories] + + for repo in repo_ids: + available_packages.extend(self._lookup_packages([AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo])) + + available_packages_in_repos += [package[0] for package in available_packages] + + return available_packages_in_repos + + def get_all_package_versions(self, pkg_name): + """ + :type pkg_name str + """ + command = LIST_ALL_SELECT_TOOL_PACKAGES_CMD.replace("{pkg_name}", pkg_name) + result = self._call_with_timeout(command) + + if result["retCode"] == 0: + return result["out"].split(os.linesep) + + return None + + def __parse_select_tool_version(self, v): + """ + :type v str + """ + matches = SELECT_TOOL_VERSION_PATTERN.findall(v.strip()) + return matches[0] if matches else None + + def normalize_select_tool_versions(self, versions): + """ + Function expect output from get_all_package_versions + + :type versions str|list|set + :rtype list + """ + if isinstance(versions, str): + versions = [versions] + + return [self.__parse_select_tool_version(i) for i in versions] + + def _lookup_packages(self, command): + """ + :type command list[str] + """ + packages = [] + skip_index = None + + result = self._call_with_timeout(command) + + if result and 0 == result['retCode']: + lines = result['out'].strip().split('\n') + lines = [line.strip() for line in lines] + for index in range(len(lines)): + if "--+--" in lines[index]: + skip_index = index + 1 + break + + if skip_index: + for line in lines[skip_index:]: + items = line.strip(' \t\n\r').split('|') + packages.append([items[1].strip(), items[3].strip(), items[5].strip()]) + + return packages + + def all_installed_packages(self, from_unknown_repo=False): + """ + Return all installed packages in the system except packages in REPO_URL_EXCLUDE + + :arg from_unknown_repo return packages from unknown repos + :type from_unknown_repo bool + + :return result_type formatted list of packages + """ + # ToDo: move to iterative package lookup (check apt provider for details) + return self._lookup_packages(ALL_INSTALLED_PACKAGES_CMD) + + def all_available_packages(self, result_type=list, group_by_index=-1): + """ + Return all available packages in the system except packages in REPO_URL_EXCLUDE + + :arg result_type Could be list or dict, defines type of returning value + :arg group_by_index index of element in the __packages_reader result, which would be used as key + :return result_type formatted list of packages, including installed and available in repos + + :type result_type type + :type group_by_index int + :rtype list|dict + """ + # ToDo: move to iterative package lookup (check apt provider for details) + return self._lookup_packages(ALL_AVAILABLE_PACKAGES_CMD) + + def verify_dependencies(self): + """ + Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated + package installation process or invalid packages state after manual modification of packages list on the host + + :return True if no dependency issues found, False if dependency issue present + :rtype bool + """ + code, out = self.checked_call(VERIFY_DEPENDENCY_CMD, sudo=True) + pattern = re.compile("\d+ new package(s)? to install") + + if code or (out and pattern.search(out)): + err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '%s' returned %s. %s" % (VERIFY_DEPENDENCY_CMD, code, out)) + Logger.error(err_msg) + return False + + return True -class ZypperProvider(PackageProvider): def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py deleted file mode 100644 index e893a9d..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env python - -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import re -import sys -import logging -import subprocess -from threading import Thread -import threading -from ambari_commons import OSCheck, OSConst -from ambari_commons import shell -from ambari_commons.constants import AMBARI_SUDO_BINARY -from resource_management.core.logger import Logger -from resource_management.core import shell as rmf_shell -from resource_management.core.exceptions import Fail - -__all__ = ["installedPkgsByName", "allInstalledPackages", "allAvailablePackages", "nameMatch", - "getInstalledRepos", "getInstalledPkgsByRepo", "getInstalledPkgsByNames", "getPackageDetails"] - -LIST_INSTALLED_PACKAGES_UBUNTU = "COLUMNS=9999 ; for i in $(dpkg -l |grep ^ii |awk -F' ' '{print $2}'); do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done" -LIST_AVAILABLE_PACKAGES_UBUNTU = "packages=`for i in $(ls -1 /var/lib/apt/lists | grep %s ) ; do grep ^Package: /var/lib/apt/lists/$i | awk '{print $2}' ; done` ; for i in $packages; do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done" -GREP_REPO_EXCLUDE_SYSTEM = "-v \"ubuntu.com\"" -logger = logging.getLogger() - -# default timeout for async invoked processes -TIMEOUT_SECONDS = 40 - - -def _launch_subprocess(command): - isShell = not isinstance(command, (list, tuple)) - return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=isShell, close_fds=True) - - -def subprocessWithTimeout(command): - event = threading.Event() - - def watchdog_func(command): - event.wait(TIMEOUT_SECONDS) - if command.returncode is None: - logger.error("Task timed out and will be killed") - shell.kill_process_with_children(command.pid) - pass - - osStat = _launch_subprocess(command) - logger.debug("Launching watchdog thread") - - event.clear() - - thread = Thread(target=watchdog_func, args=(osStat, )) - thread.start() - - out, err = osStat.communicate() - result = {} - result['out'] = out - result['err'] = err - result['retCode'] = osStat.returncode - - event.set() - thread.join() - return result - - -def installedPkgsByName(allInstalledPackages, - pkgName, installedPkgs): - """ - Get all installed package whose name starts with the - strings contained in pkgName - """ - for item in allInstalledPackages: - if item[0].find(pkgName) == 0: - installedPkgs.append(item[0]) - - -def allInstalledPackages(allInstalledPackages): - """ - All installed packages in system - """ - if OSCheck.is_suse_family(): - return _lookUpZypperPackages( - [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--installed-only", "--details"], - allInstalledPackages) - elif OSCheck.is_redhat_family(): - return _lookUpYumPackages( - [AMBARI_SUDO_BINARY, "yum", "list", "installed"], - 'Installed Packages', - allInstalledPackages) - elif OSCheck.is_ubuntu_family(): - return _lookUpAptPackages( - LIST_INSTALLED_PACKAGES_UBUNTU, - allInstalledPackages) - - -def get_available_packages_in_repos(repositories): - """ - Gets all (both installed and available) packages that are available at given repositories. - :param repositories: from command configs like config['repositoryFile']['repositories'] - :return: installed and available packages from these repositories - """ - - available_packages = [] - installed_packages = [] - available_packages_in_repos = [] - repo_ids = [repository['repoId'] for repository in repositories] - if OSCheck.is_ubuntu_family(): - allInstalledPackages(installed_packages) - repo_urls = [repository['baseUrl'] for repository in repositories] - repo_urls = [repo_url.replace("http://","";) for repo_url in repo_urls] - repo_urls = [repo_url.replace("/","_") for repo_url in repo_urls] - for url in repo_urls: - _lookUpAptPackages( - LIST_AVAILABLE_PACKAGES_UBUNTU % url, - available_packages) - for package in installed_packages: - if url in package[2]: - available_packages_in_repos.append(package[0]) - for package in available_packages: - available_packages_in_repos.append(package[0]) - elif OSCheck.is_suse_family(): - for repo in repo_ids: - _lookUpZypperPackages([AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo], - available_packages) - available_packages_in_repos += [package[0] for package in available_packages] - elif OSCheck.is_redhat_family(): - for repo in repo_ids: - _lookUpYumPackages([AMBARI_SUDO_BINARY, "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo], - 'Available Packages', available_packages) - _lookUpYumPackages([AMBARI_SUDO_BINARY, "yum", "list", "installed", "--disablerepo=*", "--enablerepo=" + repo], - 'Installed Packages', installed_packages) - available_packages_in_repos += [package[0] for package in available_packages + installed_packages] - return available_packages_in_repos - - -def allAvailablePackages(allAvailablePackages): - if OSCheck.is_suse_family(): - return _lookUpZypperPackages( - [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--uninstalled-only", "--details"], - allAvailablePackages) - elif OSCheck.is_redhat_family(): - return _lookUpYumPackages( - [AMBARI_SUDO_BINARY, "yum", "list", "available"], - 'Available Packages', - allAvailablePackages) - elif OSCheck.is_ubuntu_family(): - return _lookUpAptPackages( - LIST_AVAILABLE_PACKAGES_UBUNTU % GREP_REPO_EXCLUDE_SYSTEM, - allAvailablePackages) - -# ToDo: add execution via sudo for ubuntu (currently Ubuntu is not supported) -def _lookUpAptPackages(command, allPackages): - try: - result = subprocessWithTimeout(command) - if 0 == result['retCode']: - for x in result['out'].split('\n'): - if x.strip(): - allPackages.append(x.split(' ')) - - except: - logger.error("Unexpected error:", sys.exc_info()[0]) - - -def _lookUpYumPackages(command, skipTill, allPackages): - try: - result = subprocessWithTimeout(command) - if 0 == result['retCode']: - lines = result['out'].split('\n') - lines = [line.strip() for line in lines] - items = [] - skipIndex = 3 - for index in range(len(lines)): - if skipTill in lines[index]: - skipIndex = index + 1 - break - - for line in lines[skipIndex:]: - items = items + line.strip(' \t\n\r').split() - - for i in range(0, len(items), 3): - if '.' in items[i]: - items[i] = items[i][:items[i].rindex('.')] - if items[i + 2].find('@') == 0: - items[i + 2] = items[i + 2][1:] - allPackages.append(items[i:i + 3]) - except: - logger.error("Unexpected error:", sys.exc_info()[0]) - - -def _lookUpZypperPackages(command, allPackages): - try: - result = subprocessWithTimeout(command) - if 0 == result['retCode']: - lines = result['out'].split('\n') - lines = [line.strip() for line in lines] - for index in range(len(lines)): - if "--+--" in lines[index]: - skipIndex = index + 1 - break - - for line in lines[skipIndex:]: - items = line.strip(' \t\n\r').split('|') - allPackages.append([items[1].strip(), items[3].strip(), items[5].strip()]) - except: - logger.error("Unexpected error:", sys.exc_info()[0]) - - -def nameMatch(lookupName, actualName): - tokens = actualName.strip().split() - for token in tokens: - if lookupName.lower() in token.lower(): - return True - return False - - -def getInstalledRepos(hintPackages, allPackages, ignoreRepos, repoList): - """ - Gets all installed repos by name based on repos that provide any package - contained in hintPackages - Repos starting with value in ignoreRepos will not be returned - hintPackages must be regexps. - """ - allRepos = [] - for hintPackage in hintPackages: - for item in allPackages: - if re.match(hintPackage, item[0]) and not item[2] in allRepos: - allRepos.append(item[2]) - - for repo in allRepos: - ignore = False - for ignoredRepo in ignoreRepos: - if nameMatch(ignoredRepo, repo): - ignore = True - if not ignore: - repoList.append(repo) - - -def getInstalledPkgsByRepo(repos, ignorePackages, installedPackages): - """ - Get all the installed packages from the repos listed in repos - """ - packagesFromRepo = [] - packagesToRemove = [] - for repo in repos: - subResult = [] - for item in installedPackages: - if repo == item[2]: - subResult.append(item[0]) - packagesFromRepo = list(set(packagesFromRepo + subResult)) - - for package in packagesFromRepo: - keepPackage = True - for ignorePackage in ignorePackages: - if nameMatch(ignorePackage, package): - keepPackage = False - break - if keepPackage: - packagesToRemove.append(package) - return packagesToRemove - - -def getInstalledPkgsByNames(pkgNames, installedPackages): - """ - Gets all installed packages that start with names in pkgNames - """ - packages = [] - for pkgName in pkgNames: - subResult = [] - installedPkgsByName(installedPackages, pkgName, subResult) - packages = list(set(packages + subResult)) - return packages - - -def getPackageDetails(installedPackages, foundPackages): - """ - Gets the name, version, and repoName for the packages - """ - packageDetails = [] - for package in foundPackages: - pkgDetail = {} - for installedPackage in installedPackages: - if package == installedPackage[0]: - pkgDetail['name'] = installedPackage[0] - pkgDetail['version'] = installedPackage[1] - pkgDetail['repoName'] = installedPackage[2] - packageDetails.append(pkgDetail) - return packageDetails - -def getReposToRemove(repos, ignoreList): - reposToRemove = [] - for repo in repos: - addToRemoveList = True - for ignoreRepo in ignoreList: - if nameMatch(ignoreRepo, repo): - addToRemoveList = False - continue - if addToRemoveList: - reposToRemove.append(repo) - return reposToRemove - -def getInstalledPackageVersion(package_name): - if OSCheck.is_ubuntu_family(): - code, out = rmf_shell.checked_call("dpkg -s {0} | grep Version | awk '{{print $2}}'".format(package_name)) - else: - code, out = rmf_shell.checked_call("rpm -q --queryformat '%{{version}}-%{{release}}' {0} | sed -e 's/\.el[0-9]//g'".format(package_name)) - - return out - - -def verifyDependencies(): - """ - Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated - package installation process or invalid packages state after manual modification of packages list on the host - :return True if no dependency issues found, False if dependency issue present - :rtype bool - """ - check_str = None - cmd = None - - if OSCheck.is_redhat_family(): - cmd = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'dependencies'] - check_str = "has missing requires|Error:" - elif OSCheck.is_suse_family(): - cmd = ['/usr/bin/zypper', '--quiet', '--non-interactive', 'verify', '--dry-run'] - check_str = "\d+ new package(s)? to install" - elif OSCheck.is_ubuntu_family(): - cmd = ['/usr/bin/apt-get', '-qq', 'check'] - check_str = "has missing dependency|E:" - - if check_str is None or cmd is None: - raise Fail("Unsupported OSFamily on the Agent Host") - - code, out = rmf_shell.checked_call(cmd, sudo=True) - - output_regex = re.compile(check_str) - - if code or (out and output_regex.search(out)): - err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '%s' returned %s. %s" % (cmd, code, out)) - Logger.error(err_msg) - return False - - return True \ No newline at end of file
