AMBARI-22139 - CURRENT cluster Shows Upgrade If Component Didn't Report Version (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8b83a0a5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8b83a0a5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8b83a0a5 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 8b83a0a538358e54c5aa7f3c4eccc4a458296f9a Parents: a9d2698 Author: Jonathan Hurley <[email protected]> Authored: Thu Oct 5 16:06:06 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Fri Oct 6 10:38:29 2017 -0400 ---------------------------------------------------------------------- .../python/resource_management/TestScript.py | 26 ++++++++- .../libraries/script/script.py | 22 ++++++-- .../0.4.0/package/scripts/ranger_admin.py | 13 ++--- .../configs/ranger_admin_default.json | 55 ++++++++++++++++++++ .../src/test/python/stacks/utils/RMFTestCase.py | 9 ++-- 5 files changed, 110 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-agent/src/test/python/resource_management/TestScript.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestScript.py b/ambari-agent/src/test/python/resource_management/TestScript.py index 75726d6..79d0598 100644 --- a/ambari-agent/src/test/python/resource_management/TestScript.py +++ b/ambari-agent/src/test/python/resource_management/TestScript.py @@ -21,9 +21,9 @@ import StringIO import sys, pprint from resource_management.libraries.script import Script from resource_management.core.environment import Environment +from resource_management.core.logger import Logger from mock.mock import patch, MagicMock from stacks.utils.RMFTestCase import * -import logging class TestScript(RMFTestCase): @@ -110,7 +110,7 @@ class TestScript(RMFTestCase): @patch("__builtin__.open") def test_status_commands_clear_structured_out(self, open_mock): """ - Tests that status commands will clear and stored structured output from prior status commands. + Tests that status commands will clear any stored structured output from prior status commands. :param open_mock: :return: """ @@ -141,6 +141,28 @@ class TestScript(RMFTestCase): self.assertTrue(open_mock.called) self.assertEquals({}, Script.structuredOut) + + @patch.object(Logger, "error", new = MagicMock()) + @patch.object(Script, "put_structured_out") + @patch("resource_management.libraries.functions.version_select_util.get_component_version_from_symlink", new = MagicMock(return_value=None)) + @patch("resource_management.libraries.functions.stack_select.get_package_name", new = MagicMock(return_value="foo-package")) + @patch("resource_management.libraries.functions.stack_select.unsafe_get_stack_versions", new = MagicMock(return_value=("",0,["2.6.0.0-1234"]))) + def test_save_version_structured_out_stack_select(self, pso_mock): + """ + Tests that when writing out the version of the component to the structure output, + if all else fails, we'll invoke the stack-select tool to see if there are any versions + reported. + :param pso_mock: + :return: + """ + script = Script() + script.stroutfile = '' + script.save_component_version_to_structured_out("start") + + self.assertEqual(pso_mock.call_count, 1) + self.assertEquals(pso_mock.call_args[0][0], {'version':'2.6.0.0-1234'}) + + def tearDown(self): # enable stdout sys.stdout = sys.__stdout__ http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index e612638..d5b4469 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -47,7 +47,7 @@ from resource_management.core.environment import Environment from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning from resource_management.core.resources.packaging import Package -from resource_management.libraries.functions.version_select_util import get_component_version_from_symlink +from resource_management.libraries.functions import version_select_util from resource_management.libraries.functions.version import compare_versions from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions import stack_tools @@ -212,6 +212,12 @@ class Script(object): Saves the version of the component for this command to the structured out file. If the command is an install command and the repository is trusted, then it will use the version of the repository. Otherwise, it will consult the stack-select tool to read the symlink version. + + Under rare circumstances, a component may have a bug which prevents it from reporting a + version back after being installed. This is most likely due to the stack-select tool not being + invoked by the package's installer. In these rare cases, we try to see if the component + should have reported a version and we try to fallback to the "<stack-select> versions" command. + :param command_name: command name :return: None """ @@ -240,7 +246,17 @@ class Script(object): if stack_select_package_name and stack_name: # only query for the component version from stack-select if we can't trust the repository yet if component_version is None: - component_version = get_component_version_from_symlink(stack_name, stack_select_package_name) + component_version = version_select_util.get_component_version_from_symlink(stack_name, stack_select_package_name) + + # last ditch effort - should cover the edge case where the package failed to setup its + # link and we have to try to see if <stack-select> can help + if component_version is None: + output, code, versions = stack_select.unsafe_get_stack_versions() + if len(versions) == 1: + component_version = versions[0] + Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging. " \ + "However, the stack-select tool was able to report a single version installed ({1}). " \ + "This is the version that will be reported.".format(stack_select_package_name, component_version)) if component_version: self.put_structured_out({"version": component_version}) @@ -252,7 +268,7 @@ class Script(object): self.put_structured_out({"repository_version_id": repo_version_id}) else: if not self.is_hook(): - Logger.error("Component '{0}' did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name)) + Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name)) def should_expose_component_version(self, command_name): http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py index f779c18..848b137 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py @@ -20,7 +20,6 @@ limitations under the License. from resource_management.core.exceptions import Fail from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions.constants import Direction from resource_management.libraries.script import Script from resource_management.core.resources.system import Execute, File from resource_management.core.exceptions import ComponentIsNotRunning @@ -28,10 +27,12 @@ from resource_management.libraries.functions.format import format from resource_management.core.logger import Logger from resource_management.core import shell from ranger_service import ranger_service -from setup_ranger_xml import setup_ranger_audit_solr, setup_ranger_admin_passwd_change, update_password_configs from resource_management.libraries.functions import solr_cloud_util -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING from resource_management.libraries.functions.constants import Direction + +import setup_ranger_xml + import os, errno class RangerAdmin(Script): @@ -93,9 +94,9 @@ class RangerAdmin(Script): if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled: solr_cloud_util.setup_solr_client(params.config, custom_log4j = params.custom_log4j) - setup_ranger_audit_solr() + setup_ranger_xml.setup_ranger_audit_solr() - update_password_configs() + setup_ranger_xml.update_password_configs() ranger_service('ranger_admin') @@ -142,7 +143,7 @@ class RangerAdmin(Script): setup_java_patch() if params.stack_supports_ranger_admin_password_change: - setup_ranger_admin_passwd_change() + setup_ranger_xml.setup_ranger_admin_passwd_change() def set_ru_rangeradmin_in_progress(self, upgrade_marker_file): config_dir = os.path.dirname(upgrade_marker_file) http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json b/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json index b360c07..2e66c62 100644 --- a/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json +++ b/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json @@ -64,6 +64,61 @@ "db_host": "localhost", "xa_ldap_groupRoleAttribute": "\"cn\"" }, + "ranger-ugsync-site": { + "ranger.usersync.ldap.binddn": "", + "ranger.usersync.policymgr.username": "rangerusersync", + "ranger.usersync.policymanager.mockrun": "false", + "ranger.usersync.group.searchbase": "", + "ranger.usersync.ldap.bindalias": "testldapalias", + "ranger.usersync.truststore.file": "/usr/hdp/current/ranger-usersync/conf/mytruststore.jks", + "ranger.usersync.port": "5151", + "ranger.usersync.pagedresultssize": "500", + "ranger.usersync.group.memberattributename": "", + "ranger.usersync.kerberos.principal": "", + "ranger.usersync.source.impl.class": "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder", + "ranger.usersync.ldap.referral": "ignore", + "ranger.usersync.group.searchfilter": "", + "ranger.usersync.ldap.user.objectclass": "person", + "ranger.usersync.logdir": "{{usersync_log_dir}}", + "ranger.usersync.ldap.user.searchfilter": "", + "ranger.usersync.ldap.groupname.caseconversion": "none", + "ranger.usersync.ldap.ldapbindpassword": "", + "ranger.usersync.unix.minUserId": "500", + "ranger.usersync.policymanager.maxrecordsperapicall": "1000", + "ranger.usersync.group.nameattribute": "", + "ranger.usersync.policymgr.alias": "ranger.usersync.policymgr.password", + "ranger.usersync.keystore.file": "/usr/hdp/current/ranger-usersync/conf/unixauthservice.jks", + "ranger.usersync.user.searchenabled": "false", + "ranger.usersync.group.usermapsyncenabled": "true", + "ranger.usersync.ldap.bindkeystore": "", + "ranger.usersync.ldap.user.groupnameattribute": "memberof, ismemberof", + "ranger.usersync.kerberos.keytab": "", + "ranger.usersync.passwordvalidator.path": "./native/credValidator.uexe", + "ranger.usersync.group.objectclass": "", + "ranger.usersync.ldap.user.searchscope": "sub", + "ranger.usersync.unix.password.file": "/etc/passwd", + "ranger.usersync.ldap.user.nameattribute": "", + "ranger.usersync.pagedresultsenabled": "true", + "ranger.usersync.policymanager.baseURL": "{{ranger_external_url}}", + "ranger.usersync.group.search.first.enabled": "false", + "ranger.usersync.group.searchenabled": "false", + "ranger.usersync.sink.impl.class": "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder", + "ranger.usersync.ssl": "true", + "ranger.usersync.ldap.url": "", + "ranger.usersync.ldap.searchBase": "dc=hadoop,dc=apache,dc=org", + "ranger.usersync.policymgr.keystore": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks", + "ranger.usersync.ldap.user.searchbase": "", + "ranger.usersync.ldap.username.caseconversion": "none", + "ranger.usersync.credstore.filename": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks", + "ranger.usersync.keystore.password": "UnIx529p", + "ranger.usersync.unix.group.file": "/etc/group", + "ranger.usersync.filesource.file": "/tmp/usergroup.txt", + "ranger.usersync.group.searchscope": "", + "ranger.usersync.truststore.password": "changeit", + "ranger.usersync.enabled": "true", + "ranger.usersync.sleeptimeinmillisbetweensynccycle": "60000", + "ranger.usersync.filesource.text.delimiter": "," + }, "ranger-site": { "http.enabled": "true", "http.service.port": "6080", http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index d98e0b1..bff8642 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -154,11 +154,12 @@ class RMFTestCase(TestCase): with patch('resource_management.libraries.functions.stack_select.is_package_supported', return_value=True): with patch('resource_management.libraries.functions.stack_select.get_supported_packages', return_value=MagicMock()): with patch.object(os, "environ", new=os_env) as mocks_dict['environ']: - if not try_install: - with patch.object(Script, 'install_packages') as install_mock_value: + with patch('resource_management.libraries.functions.stack_select.unsafe_get_stack_versions', return_value = (("",0,[]))): + if not try_install: + with patch.object(Script, 'install_packages') as install_mock_value: + method(RMFTestCase.env, *command_args) + else: method(RMFTestCase.env, *command_args) - else: - method(RMFTestCase.env, *command_args) sys.path.remove(scriptsdir)
