AMBARI-21405. Create custom action to force-remove packages
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6c5d78a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6c5d78a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6c5d78a Branch: refs/heads/branch-2.5 Commit: b6c5d78ac633e238310419e45314bd69ac4edc26 Parents: 85d7c19 Author: Attila Doroszlai <[email protected]> Authored: Tue Jul 4 20:20:02 2017 +0200 Committer: Attila Doroszlai <[email protected]> Committed: Fri Jul 7 22:29:38 2017 +0200 ---------------------------------------------------------------------- .../resource_management/TestPackageResource.py | 41 ++++++++++++++ .../core/providers/package/__init__.py | 2 +- .../core/providers/package/apt.py | 10 ++-- .../core/providers/package/choco.py | 4 +- .../core/providers/package/yumrpm.py | 9 +++- .../core/providers/package/zypper.py | 9 +++- .../core/resources/packaging.py | 6 +++ .../system_action_definitions.xml | 10 ++++ .../scripts/force_remove_packages.py | 56 ++++++++++++++++++++ 9 files changed, 137 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-agent/src/test/python/resource_management/TestPackageResource.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestPackageResource.py b/ambari-agent/src/test/python/resource_management/TestPackageResource.py index 66227c6..bc1bfeb 100644 --- a/ambari-agent/src/test/python/resource_management/TestPackageResource.py +++ b/ambari-agent/src/test/python/resource_management/TestPackageResource.py @@ -217,6 +217,21 @@ class TestPackageResource(TestCase): @patch.object(shell, "call", new = MagicMock(return_value=(0, None))) @patch.object(shell, "checked_call") + @patch.object(System, "os_family", new = 'redhat') + def test_action_remove_nodeps_rhel(self, shell_mock): + sys.modules['rpm'] = MagicMock() + sys.modules['rpm'].TransactionSet.return_value = MagicMock() + sys.modules['rpm'].TransactionSet.return_value.dbMatch.return_value = [{'name':'some_package'}] + with Environment('/') as env: + Package("some_package", + action = "remove", + logoutput = False, + ignore_dependencies = True + ) + shell_mock.assert_called_with(['/usr/bin/rpm', '-e', '--nodeps', 'some_package'], logoutput=False, sudo=True) + + @patch.object(shell, "call", new = MagicMock(return_value=(0, None))) + @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'suse') def test_action_remove_suse(self, shell_mock): shell_mock.return_value = (0, '') @@ -230,6 +245,32 @@ class TestPackageResource(TestCase): ) shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm', 'some_package'], logoutput=False, sudo=True) + @patch.object(shell, "call", new = MagicMock(return_value=(0, None))) + @patch.object(shell, "checked_call") + @patch.object(System, "os_family", new = 'suse') + def test_action_remove_nodeps_suse(self, shell_mock): + shell_mock.return_value = (0, '') + with Environment('/') as env: + Package("some_package", + action = "remove", + logoutput = False, + ignore_dependencies = True + ) + shell_mock.assert_called_with(['/usr/bin/rpm', '-e', '--nodeps', 'some_package'], logoutput=False, sudo=True) + + @patch.object(shell, "call", new = MagicMock(return_value=(0, None))) + @patch.object(shell, "checked_call") + @patch.object(System, "os_family", new = 'ubuntu') + def test_action_remove_nodeps_ubuntu(self, shell_mock): + shell_mock.return_value = (0, '') + with Environment('/') as env: + Package("some-package", + action = "remove", + logoutput = False, + ignore_dependencies = True + ) + shell_mock.assert_called_with(['/usr/bin/dpkg', '--remove', '--ignore-depends', 'some-package', 'some-package'], logoutput=False, sudo=True) + @patch.object(shell, "call", new = MagicMock(return_value=(1, None))) @patch.object(shell, "checked_call") @patch.object(System, "os_family", new = 'redhat') http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py index 21de183..a1d3e02 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/__init__.py @@ -59,7 +59,7 @@ class PackageProvider(Provider): def action_remove(self): package_name = self.get_package_name_with_version() - self.remove_package(package_name) + self.remove_package(package_name, self.resource.ignore_dependencies) def get_package_name_with_version(self): if self.resource.version: http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py index d095173..aa80557 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py @@ -41,6 +41,7 @@ REMOVE_CMD = { False: ['/usr/bin/apt-get', '-y', '-q', 'remove'], } REPO_UPDATE_CMD = ['/usr/bin/apt-get', 'update','-qq'] +REMOVE_WITHOUT_DEPENDENCIES_CMD = ['/usr/bin/dpkg', '--remove', '--ignore-depends'] APT_SOURCES_LIST_DIR = "/etc/apt/sources.list.d" @@ -103,9 +104,12 @@ class AptProvider(PackageProvider): return self.install_package(name, use_repos, skip_repos, is_upgrade) @replace_underscores - def remove_package(self, name): + def remove_package(self, name, ignore_dependencies = False): if self._check_existence(name): - cmd = REMOVE_CMD[self.get_logoutput()] + [name] + if ignore_dependencies: + cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [name, name] # have to specify name twice: one for --ignore-depends, one for --remove + else: + cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: @@ -132,4 +136,4 @@ class AptProvider(PackageProvider): we should not rely on that. """ code, out = shell.call(CHECK_CMD % name) - return not bool(code) \ No newline at end of file + return not bool(code) http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-common/src/main/python/resource_management/core/providers/package/choco.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/choco.py b/ambari-common/src/main/python/resource_management/core/providers/package/choco.py index db55296..1bb6abf 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/choco.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/choco.py @@ -75,7 +75,7 @@ class ChocoProvider(PackageProvider): if res['exitCode'] != 0: raise Exception("Error while upgrading choco package " + name + ". " + res['error'] + res['output']) - def remove_package(self, name): + def remove_package(self, name, ignore_dependencies = False): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] cmdString = " ".join(cmd) @@ -93,4 +93,4 @@ class ChocoProvider(PackageProvider): res = runner.run(cmd) if name in res['output']: return True - return False \ No newline at end of file + return False http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py index ea10a86..064b504 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py @@ -36,6 +36,8 @@ REMOVE_CMD = { False: ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'erase'], } +REMOVE_WITHOUT_DEPENDENCIES_CMD = ['/usr/bin/rpm', '-e', '--nodeps'] + REPO_UPDATE_CMD = ['/usr/bin/yum', 'clean','metadata'] class YumProvider(PackageProvider): @@ -55,9 +57,12 @@ class YumProvider(PackageProvider): def upgrade_package(self, name, use_repos=[], skip_repos=[], is_upgrade=True): return self.install_package(name, use_repos, skip_repos, is_upgrade) - def remove_package(self, name): + def remove_package(self, name, ignore_dependencies = False): if self._check_existence(name): - cmd = REMOVE_CMD[self.get_logoutput()] + [name] + if ignore_dependencies: + cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [name] + else: + cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py index 265c162..c1aab60 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py +++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py @@ -35,6 +35,8 @@ REMOVE_CMD = { False: ['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm'], } +REMOVE_WITHOUT_DEPENDENCIES_CMD = ['/usr/bin/rpm', '-e', '--nodeps'] + REPO_UPDATE_CMD = ['/usr/bin/zypper', 'clean'] LIST_ACTIVE_REPOS_CMD = ['/usr/bin/zypper', 'repos'] @@ -63,9 +65,12 @@ class ZypperProvider(PackageProvider): def upgrade_package(self, name, use_repos=[], skip_repos=[], is_upgrade=True): return self.install_package(name, use_repos, skip_repos, is_upgrade) - def remove_package(self, name): + def remove_package(self, name, ignore_dependencies = False): if self._check_existence(name): - cmd = REMOVE_CMD[self.get_logoutput()] + [name] + if ignore_dependencies: + cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [name] + else: + cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-common/src/main/python/resource_management/core/resources/packaging.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/resources/packaging.py b/ambari-common/src/main/python/resource_management/core/resources/packaging.py index e3adc30..5febdae 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/packaging.py +++ b/ambari-common/src/main/python/resource_management/core/resources/packaging.py @@ -52,3 +52,9 @@ class Package(Resource): version = ResourceArgument() actions = ["install", "upgrade", "remove"] build_vars = ForcedListArgument(default=[]) + + """ + False - also remove any packages that depend on the one being removed + True - possibly break dependencies by keeping them installed + """ + ignore_dependencies = BooleanArgument(default=False) http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml index 0f50256..c3f97e7 100644 --- a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml +++ b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml @@ -94,4 +94,14 @@ <description>Perform remove old stack version action</description> <targetType>ANY</targetType> </actionDefinition> + <actionDefinition> + <actionName>force_remove_packages</actionName> + <actionType>SYSTEM</actionType> + <inputs>package_list</inputs> + <targetService/> + <targetComponent/> + <defaultTimeout>600</defaultTimeout> + <description>Remove packages specified by package_list, instructing the package manager to ignore dependencies.</description> + <targetType>ALL</targetType> + </actionDefinition> </actionDefinitions> http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c5d78a/ambari-server/src/main/resources/custom_actions/scripts/force_remove_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/force_remove_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/force_remove_packages.py new file mode 100644 index 0000000..237e135 --- /dev/null +++ b/ambari-server/src/main/resources/custom_actions/scripts/force_remove_packages.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management import Script +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger +from resource_management.core.resources.packaging import Package + +class ForceRemovePackages(Script): + """ + This script is used during cross-stack upgrade to remove packages + required by the old stack that are in conflict with packages from the + new stack (eg. stack tools). It can be called via REST API as a custom + action. + """ + + def actionexecute(self, env): + config = Script.get_config() + packages_to_remove = config['roleParams']['package_list'].split(',') + structured_output = {'success': [], 'failure': []} + + for package_name in packages_to_remove: + try: + Package(package_name, action='remove', ignore_dependencies = True) + Logger.info('Removed {0}'.format(package_name)) + structured_output['success'].append(package_name) + except Exception, e: + Logger.exception('Failed to remove {0}: {1}'.format(package_name, str(e))) + structured_output['failure'].append(package_name) + + self.put_structured_out(structured_output) + + if structured_output['failure']: + raise Fail('Failed to remove packages: ' + ', '.join(structured_output['failure'])) + + +if __name__ == '__main__': + ForceRemovePackages().execute()
