This is an automated email from the ASF dual-hosted git repository. jonathanhurley pushed a commit to branch branch-feature-AMBARI-14714 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push: new f54cef1 [AMBARI-24410] - Remove conf-select Tool From Ambari Framework (#1981) f54cef1 is described below commit f54cef1e337986264bc6c38a6ff042b02f148b9f Author: Jonathan Hurley <jonathanhur...@apache.org> AuthorDate: Wed Aug 8 11:11:48 2018 -0400 [AMBARI-24410] - Remove conf-select Tool From Ambari Framework (#1981) --- .../libraries/functions/conf_select.py | 387 +-------------------- .../libraries/functions/stack_tools.py | 1 - .../resource_management/libraries/script/script.py | 2 - .../stack-hooks/after-INSTALL/scripts/hook.py | 6 +- .../stack-hooks/after-INSTALL/scripts/params.py | 2 - .../after-INSTALL/scripts/shared_initialization.py | 18 - .../hooks/after-INSTALL/test_after_install.py | 195 ----------- .../python/stacks/2.2/common/test_conf_select.py | 200 ----------- 8 files changed, 6 insertions(+), 805 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index 7b3f66d..f285c32 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@ -18,144 +18,24 @@ limitations under the License. """ -__all__ = ["select", "create", "get_hadoop_conf_dir", "get_hadoop_dir", "get_package_dirs"] +__all__ = ["get_hadoop_conf_dir"] # Python Imports import os -from ambari_commons import subprocess32 -import ambari_simplejson as json # Local Imports -from resource_management.core import shell -from resource_management.libraries.functions.format import format from resource_management.libraries.script.script import Script from resource_management.core.logger import Logger -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import Execute -from resource_management.core.resources.system import Link from resource_management.libraries.functions import component_version from resource_management.libraries.functions.default import default -from resource_management.libraries.functions import stack_tools -from resource_management.core.exceptions import Fail from resource_management.core import sudo -from resource_management.core.shell import as_sudo from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.decorator import deprecated -def _get_cmd(command, package, version): - conf_selector_path = stack_tools.get_stack_tool_path(stack_tools.CONF_SELECTOR_NAME) - return ('ambari-python-wrap', conf_selector_path, command, '--package', package, '--stack-version', version, '--conf-version', '0') - -def _valid(stack_name, package, ver): - return (ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, ver)) - -def get_package_dirs(): - """ - Get package dir mappings - :return: - """ - stack_name = default("/stackSettings/stack_name", None) - if stack_name is None: - raise Fail("The stack name is not present in the command. Packages for conf-select tool cannot be loaded.") - - stack_packages_config = default("/configurations/cluster-env/stack_packages", None) - if stack_packages_config is None: - raise Fail("The stack packages are not defined on the command. Unable to load packages for the conf-select tool") - - data = json.loads(stack_packages_config) - - if stack_name not in data: - raise Fail( - "Cannot find conf-select packages for the {0} stack".format(stack_name)) - - conf_select_key = "conf-select" - data = data[stack_name] - if conf_select_key not in data: - raise Fail( - "There are no conf-select packages defined for this command for the {0} stack".format(stack_name)) - - package_dirs = data[conf_select_key] - - stack_root = Script.get_stack_root() - for package_name, directories in package_dirs.iteritems(): - for dir in directories: - current_dir = dir['current_dir'] - current_dir = current_dir.format(stack_root) - dir['current_dir'] = current_dir - - return package_dirs - -def create(stack_name, package, version, dry_run = False): - """ - Creates a config version for the specified package - :param stack_name: the name of the stack - :param package: the name of the package, as-used by <conf-selector-tool> - :param version: the version number to create - :param dry_run: False to create the versioned config directory, True to only return what would be created - :return List of directories created - """ - if not _valid(stack_name, package, version): - Logger.info("Unable to create versioned configuration directories since the parameters supplied do not support it") - return [] - - # clarify the logging of what we're doing ... - if dry_run: - Logger.info( - "Checking to see which directories will be created for {0} on version {1}".format(package, version)) - else: - Logger.info("Creating /etc/{0}/{1}/0 if it does not exist".format(package, version)) - - command = "dry-run-create" if dry_run else "create-conf-dir" - - code, stdout, stderr = shell.call(_get_cmd(command, package, version), logoutput=False, quiet=False, sudo=True, stderr = subprocess32.PIPE) - - # <conf-selector-tool> can set more than one directory - # per package, so return that list, especially for dry_run - # > <conf-selector-tool> dry-run-create --package hive-hcatalog --stack-version 2.4.0.0-169 0 - # /etc/hive-webhcat/2.4.0.0-169/0 - # /etc/hive-hcatalog/2.4.0.0-169/0 - created_directories = [] - if 0 == code and stdout is not None: # just be sure we have a stdout - for line in stdout.splitlines(): - created_directories.append(line.rstrip('\n')) - - # if directories were created, then do some post-processing - if not code and stdout and not dry_run: - # take care of permissions if directories were created - for directory in created_directories: - Directory(directory, mode=0755, cd_access='a', create_parents=True) - - # seed the new directories with configurations from the old (current) directories - _seed_new_configuration_directories(package, created_directories) - - return created_directories - - -def select(stack_name, package, version, ignore_errors=False): - """ - Selects a config version for the specified package. - - :param stack_name: the name of the stack - :param package: the name of the package, as-used by <conf-selector-tool> - :param version: the version number to create - :param ignore_errors: optional argument to ignore any error and simply log a warning - """ - try: - # do nothing if the stack does not support versioned configurations - if not _valid(stack_name, package, version): - return - - create(stack_name, package, version) - shell.checked_call(_get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True) - except Exception, exception: - if ignore_errors is True: - Logger.warning("Could not select the directory for package {0}. Error: {1}".format(package, - str(exception))) - else: - raise - - - +@deprecated(comment = "The conf-select tools are no longer used. In order to get the hadoop conf " + "directory, the mpack_manager_helper module should be used to get the conf " + "directory of HADOOP_CLIENTS for the component's service group.") def get_hadoop_conf_dir(): """ Return the hadoop shared conf directory which should be used for the command's component. The @@ -180,260 +60,3 @@ def get_hadoop_conf_dir(): return hadoop_conf_dir - -def convert_conf_directories_to_symlinks(package, version, dirs): - """ - Reverses the symlinks created by the package installer and invokes the conf-select tool to - create versioned configuration directories for the given package. If the package does not exist, - then no work is performed. - - - Creates /etc/<component>/<version>/0 via <conf-selector-tool> - - Creates a /etc/<component>/conf.backup directory, if needed - - Copies all configs from /etc/<component>/conf to conf.backup, if needed - - Removes /etc/<component>/conf, if needed - - <stack-root>/current/<component>-client/conf -> /etc/<component>/<version>/0 via <conf-selector-tool> - - Links /etc/<component>/conf -> <stack-root>/current/[component]-client/conf - - :param package: the package to create symlinks for (zookeeper, falcon, etc) - :param version: the version number to use with <conf-selector-tool> (2.3.0.0-1234) - :param dirs: the directories associated with the package (from get_package_dirs()) - """ - # if the conf_dir doesn't exist, then that indicates that the package's service is not installed - # on this host and nothing should be done with conf symlinks - stack_name = Script.get_stack_name() - for directory_struct in dirs: - if not os.path.exists(directory_struct['conf_dir']): - Logger.info("Skipping the conf-select tool on {0} since {1} does not exist.".format( - package, directory_struct['conf_dir'])) - - return - - # determine which directories would be created, if any are needed - dry_run_directory = create(stack_name, package, version, dry_run = True) - - # if the dry run reported an error, then we must assume that the package does not exist in - # the conf-select tool - if len(dry_run_directory) == 0: - Logger.info("The conf-select tool reported an error for the package {0}. The configuration linking will be skipped.".format(package)) - return - - - need_dirs = [] - for d in dry_run_directory: - if not os.path.exists(d): - need_dirs.append(d) - - # log that we'll actually be creating some directories soon - if len(need_dirs) > 0: - Logger.info("Package {0} will have the following new configuration directories created: {1}".format( - package, ", ".join(dry_run_directory))) - - # Create the versioned /etc/[component]/[version]/0 folder (using create-conf-dir) and then - # set it for the installed component: - # - Creates /etc/<component>/<version>/0 - # - Links <stack-root>/<version>/<component>/conf -> /etc/<component>/<version>/0 - select(stack_name, package, version, ignore_errors = True) - - # check every existing link to see if it's a link and if it's pointed to the right spot - for directory_struct in dirs: - try: - # check if conf is a link already - old_conf = directory_struct['conf_dir'] - current_dir = directory_struct['current_dir'] - if os.path.islink(old_conf): - # it's already a link; make sure it's a link to where we want it - if os.readlink(old_conf) != current_dir: - # the link isn't to the right spot; re-link it - Logger.info("Re-linking symlink {0} to {1}".format(old_conf, current_dir)) - Link(old_conf, action = "delete") - Link(old_conf, to = current_dir) - else: - Logger.info("{0} is already linked to {1}".format(old_conf, current_dir)) - elif os.path.isdir(old_conf): - # the /etc/<component>/conf directory is not a link, so turn it into one - Logger.info("{0} is a directory - it must be converted into a symlink".format(old_conf)) - - backup_dir = _get_backup_conf_directory(old_conf) - Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir)) - Execute(("cp", "-R", "-p", old_conf, backup_dir), - not_if = format("test -e {backup_dir}"), sudo = True) - - # delete the old /etc/<component>/conf directory now that it's been backed up - Directory(old_conf, action = "delete") - - # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf - Link(old_conf, to = current_dir) - else: - # missing entirely - # /etc/<component>/conf -> <stack-root>/current/<component>/conf - if package in ["atlas", ]: - # HACK for Atlas - ''' - In the case of Atlas, the Hive RPM installs /usr/$stack/$version/atlas with some partial packages that - contain Hive hooks, while the Atlas RPM is responsible for installing the full content. - - If the user does not have Atlas currently installed on their stack, then /usr/$stack/current/atlas-client - will be a broken symlink, and we should not create the - symlink /etc/atlas/conf -> /usr/$stack/current/atlas-client/conf . - If we mistakenly create this symlink, then when the user performs an EU/RU and then adds Atlas service - then the Atlas RPM will not be able to copy its artifacts into /etc/atlas/conf directory and therefore - prevent Ambari from by copying those unmanaged contents into /etc/atlas/$version/0 - ''' - component_list = default("/localComponents", []) - if "ATLAS_SERVER" in component_list or "ATLAS_CLIENT" in component_list: - Logger.info("Atlas is installed on this host.") - parent_dir = os.path.dirname(current_dir) - if os.path.exists(parent_dir): - Link(old_conf, to = current_dir) - else: - Logger.info( - "Will not create symlink from {0} to {1} because the destination's parent dir does not exist.".format( - old_conf, current_dir)) - else: - Logger.info( - "Will not create symlink from {0} to {1} because Atlas is not installed on this host.".format( - old_conf, current_dir)) - else: - # Normal path for other packages - Link(old_conf, to = current_dir) - - except Exception, e: - Logger.warning("Could not change symlink for package {0} to point to current directory. Error: {1}".format(package, e)) - - -def get_restricted_packages(): - """ - Gets the list of conf-select 'package' names that need to be invoked on the command. - When the server passes down the list of packages to install, check the service names - and use the information in stack_packages json to determine the list of packages that should - be executed. That is valid only for PATCH or MAINT upgrades. STANDARD upgrades should be - conf-select'ing everything it can find. - """ - package_names = [] - - # shortcut the common case if we are not patching - cluster_version_summary = default("/roleParameters/cluster_version_summary/services", None) - - if cluster_version_summary is None: - Logger.info("Cluster Summary is not available, there are no restrictions for conf-select") - return package_names - - service_names = [] - - # pick out the services that are targeted - for servicename, servicedetail in cluster_version_summary.iteritems(): - if servicedetail['upgrade']: - service_names.append(servicename) - - if 0 == len(service_names): - Logger.info("No services found, there are no restrictions for conf-select") - return package_names - - stack_name = default("/stackSettings/stack_name", None) - if stack_name is None: - Logger.info("The stack name is not present in the command. Restricted names skipped.") - return package_names - - stack_packages_config = default("/configurations/cluster-env/stack_packages", None) - if stack_packages_config is None: - Logger.info("The stack packages are not defined on the command. Restricted names skipped.") - return package_names - - data = json.loads(stack_packages_config) - - if stack_name not in data: - Logger.info("Cannot find conf-select packages for the {0} stack".format(stack_name)) - return package_names - - conf_select_key = "conf-select-patching" - if conf_select_key not in data[stack_name]: - Logger.info("There are no conf-select-patching elements defined for this command for the {0} stack".format(stack_name)) - return package_names - - service_dict = data[stack_name][conf_select_key] - - for servicename in service_names: - if servicename in service_dict and 'packages' in service_dict[servicename]: - package_names.extend(service_dict[servicename]['packages']) - - return package_names - - -def _seed_new_configuration_directories(package, created_directories): - """ - Copies any files from the "current" configuration directory to the directories which were - newly created with <conf-selector-tool>. This function helps ensure that files which are not tracked - by Ambari will be available after performing a stack upgrade. Although old configurations - will be copied as well, they will be overwritten when the components are writing out their - configs after upgrade during their restart. - - This function will catch all errors, logging them, but not raising an exception. This is to - prevent problems here from stopping and otherwise healthy upgrade. - - :param package: the <conf-selector-tool> package name - :param created_directories: a list of directories that <conf-selector-tool> said it created - :return: None - """ - package_dirs = get_package_dirs() - if package not in package_dirs: - Logger.warning("Unable to seed newly created configuration directories for {0} because it is an unknown component".format(package)) - return - - # seed the directories with any existing configurations - # this allows files which are not tracked by Ambari to be available after an upgrade - Logger.info("Seeding versioned configuration directories for {0}".format(package)) - expected_directories = package_dirs[package] - - try: - # if the expected directories don't match those created, we can't seed them - if len(created_directories) != len(expected_directories): - Logger.warning("The known configuration directories for {0} do not match those created by conf-select: {1}".format( - package, str(created_directories))) - - return - - # short circuit for a simple 1:1 mapping - if len(expected_directories) == 1: - # <stack-root>/current/component/conf - # the current directory is the source of the seeded configurations; - source_seed_directory = expected_directories[0]["current_dir"] - target_seed_directory = created_directories[0] - _copy_configurations(source_seed_directory, target_seed_directory) - else: - for created_directory in created_directories: - for expected_directory_structure in expected_directories: - prefix = expected_directory_structure.get("prefix", None) - if prefix is not None and created_directory.startswith(prefix): - source_seed_directory = expected_directory_structure["current_dir"] - target_seed_directory = created_directory - _copy_configurations(source_seed_directory, target_seed_directory) - - except Exception, e: - Logger.warning("Unable to seed new configuration directories for {0}. {1}".format(package, str(e))) - - -def _copy_configurations(source_directory, target_directory): - """ - Copies from the source directory to the target directory. If the source directory is a symlink - then it will be followed (deferenced) but any other symlinks found to copy will not be. This - will ensure that if the configuration directory itself is a symlink, then it's contents will be - copied, preserving and children found which are also symlinks. - - :param source_directory: the source directory to copy from - :param target_directory: the target directory to copy to - :return: None - """ - # append trailing slash so the cp command works correctly WRT recursion and symlinks - source_directory = os.path.join(source_directory, "*") - Execute(as_sudo(["cp", "-R", "-p", "-v", source_directory, target_directory], auto_escape = False), - logoutput = True) - -def _get_backup_conf_directory(old_conf): - """ - Calculates the conf.backup absolute directory given the /etc/<component>/conf location. - :param old_conf: the old conf directory (ie /etc/<component>/conf) - :return: the conf.backup absolute directory - """ - old_parent = os.path.abspath(os.path.join(old_conf, os.pardir)) - backup_dir = os.path.join(old_parent, "conf.backup") - return backup_dir \ No newline at end of file diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py index c16dd22..4ad83e0 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py @@ -24,7 +24,6 @@ __all__ = ["get_stack_tool", "get_stack_tool_name", "get_stack_tool_path", # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. import ambari_simplejson as json -from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.core.utils import pad from resource_management.libraries.functions import stack_settings diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 2211baf..c208731 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -93,8 +93,6 @@ def get_path_from_configuration(name, configuration): return configuration -def get_config_lock_file(): - return os.path.join(Script.get_tmp_dir(), "link_configs_lock_file") class Script(object): instance = None diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/hook.py index a4733c0..97b0dc5 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/hook.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/hook.py @@ -18,8 +18,6 @@ limitations under the License. """ from resource_management.libraries.script.hook import Hook -from shared_initialization import link_configs -from shared_initialization import setup_config from shared_initialization import setup_stack_symlinks class AfterInstallHook(Hook): @@ -29,9 +27,7 @@ class AfterInstallHook(Hook): env.set_params(params) setup_stack_symlinks(self.stroutfile) - # setup_config() - # - # link_configs(self.stroutfile) + if __name__ == "__main__": AfterInstallHook().execute() diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py index f8eb445..42ec7d5 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py @@ -22,7 +22,6 @@ import os from ambari_commons.constants import AMBARI_SUDO_BINARY from ambari_commons.constants import LOGFEEDER_CONF_DIR from resource_management.libraries.script import Script -from resource_management.libraries.script.script import get_config_lock_file from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.format_jvm_option import format_jvm_option_value @@ -113,7 +112,6 @@ if has_namenode or dfs_type == 'HCFS': mount_table_xml_inclusion_file_full_path = os.path.join(hadoop_conf_dir, xml_inclusion_file_name) mount_table_content = mount_table['content'] -link_configs_lock_file = get_config_lock_file() stack_select_lock_file = os.path.join(tmp_dir, "stack_select_lock_file") upgrade_suspended = execution_command.is_upgrade_suspended() diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py index 073cdb6..99ba28d 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py @@ -123,21 +123,3 @@ def load_version(struct_out_file): return json_info['version'] except (IOError, KeyError, TypeError): return None - - -def link_configs(struct_out_file): - """ - Use the conf_select module to link configuration directories correctly. - """ - import params - - json_version = load_version(struct_out_file) - - if not json_version: - Logger.info("Could not load 'version' from {0}".format(struct_out_file)) - return - - # On parallel command execution this should be executed by a single process at a time. - with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): - for package_name, directories in conf_select.get_package_dirs().iteritems(): - conf_select.convert_conf_directories_to_symlinks(package_name, json_version, directories) \ No newline at end of file diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py index c03a10f..a5298f7 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py @@ -24,7 +24,6 @@ import json from mock.mock import MagicMock, patch from stacks.utils.RMFTestCase import * from resource_management.core.logger import Logger -from resource_management.libraries.functions import conf_select from resource_management.libraries.script import Script @patch("os.path.exists", new = MagicMock(return_value=True)) @@ -66,138 +65,6 @@ class TestHookAfterInstall(RMFTestCase): create_parents = True) self.assertNoMoreResources() - @patch("os.path.isdir", new = MagicMock(return_value = True)) - @patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1234")) - @patch("resource_management.libraries.functions.conf_select.create") - @patch("resource_management.libraries.functions.conf_select.select") - @patch("os.symlink") - @patch("shutil.rmtree") - def test_hook_default_conf_select(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): - - def mocked_conf_select(arg1, arg2, arg3, dry_run = False): - return "/etc/{0}/{1}/0".format(arg2, arg3) - - conf_select_create_mock.side_effect = mocked_conf_select - - config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json" - with open(config_file, "r") as f: - json_content = json.load(f) - - version = '2.3.0.0-1234' - json_content['commandParams']['version'] = version - json_content['clusterLevelParams']['stack_version'] = "2.3" - - self.executeScript("after-INSTALL/scripts/hook.py", - classname="AfterInstallHook", - command="hook", - stack_version = self.STACK_VERSION, - target=RMFTestCase.TARGET_STACK_HOOKS, - config_dict = json_content, - config_overrides = self.CONFIG_OVERRIDES) - - - self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.3.0.0-1234'), - sudo = True) - - self.assertResourceCalled('XmlConfig', 'core-site.xml', - owner = 'hdfs', - group = 'hadoop', - conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", - configurations = self.getConfig()['configurations']['core-site'], - configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", - xml_include_file=None) - - self.assertResourceCalled('Directory', - '/usr/lib/ambari-logsearch-logfeeder/conf', - mode = 0755, - cd_access = 'a', - create_parents = True) - - package_dirs = conf_select.get_package_dirs(); - for package, dir_defs in package_dirs.iteritems(): - for dir_def in dir_defs: - conf_dir = dir_def['conf_dir'] - conf_backup_dir = conf_dir + ".backup" - current_dir = dir_def['current_dir'] - self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), - not_if = 'test -e ' + conf_backup_dir, - sudo = True,) - - self.assertResourceCalled('Directory', conf_dir, action = ['delete'],) - self.assertResourceCalled('Link', conf_dir, to = current_dir,) - - self.assertNoMoreResources() - - @patch("os.path.isdir", new = MagicMock(return_value = True)) - @patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1234")) - @patch("resource_management.libraries.functions.conf_select.create") - @patch("resource_management.libraries.functions.conf_select.select") - @patch("os.symlink") - @patch("shutil.rmtree") - def test_hook_default_conf_select_with_error(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): - - def mocked_conf_select(arg1, arg2, arg3, dry_run = False, ignore_errors = False): - if arg2 == "pig" and not dry_run: - if not ignore_errors: - raise Exception("whoops") - else: - return None - return "/etc/{0}/{1}/0".format(arg2, arg3) - - conf_select_create_mock.side_effect = mocked_conf_select - conf_select_select_mock.side_effect = mocked_conf_select - - config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json" - with open(config_file, "r") as f: - json_content = json.load(f) - - version = '2.3.0.0-1234' - json_content['commandParams']['version'] = version - json_content['clusterLevelParams']['stack_version'] = "2.3" - - self.executeScript("after-INSTALL/scripts/hook.py", - classname="AfterInstallHook", - command="hook", - stack_version = self.STACK_VERSION, - target=RMFTestCase.TARGET_STACK_HOOKS, - config_dict = json_content, - config_overrides = self.CONFIG_OVERRIDES) - - - self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.3.0.0-1234'), - sudo = True) - - self.assertResourceCalled('XmlConfig', 'core-site.xml', - owner = 'hdfs', - group = 'hadoop', - conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", - configurations = self.getConfig()['configurations']['core-site'], - configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", - xml_include_file=None) - - self.assertResourceCalled('Directory', - '/usr/lib/ambari-logsearch-logfeeder/conf', - mode = 0755, - cd_access = 'a', - create_parents = True) - - package_dirs = conf_select.get_package_dirs(); - for package, dir_defs in package_dirs.iteritems(): - for dir_def in dir_defs: - conf_dir = dir_def['conf_dir'] - conf_backup_dir = conf_dir + ".backup" - current_dir = dir_def['current_dir'] - self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), - not_if = 'test -e ' + conf_backup_dir, - sudo = True,) - - self.assertResourceCalled('Directory', conf_dir, action = ['delete'],) - self.assertResourceCalled('Link', conf_dir, to = current_dir,) - - self.assertNoMoreResources() - @patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1234")) @patch("resource_management.libraries.functions.conf_select.create") @@ -240,68 +107,6 @@ class TestHookAfterInstall(RMFTestCase): self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hive-server2', '2.3.0.0-1234'), sudo = True) - @patch("os.path.isdir", new = MagicMock(return_value = True)) - @patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1234")) - @patch("resource_management.libraries.functions.conf_select.create") - @patch("resource_management.libraries.functions.conf_select.select") - @patch("os.symlink") - @patch("shutil.rmtree") - def test_hook_default_conf_select_suspended(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock): - - def mocked_conf_select(arg1, arg2, arg3, dry_run = False): - return "/etc/{0}/{1}/0".format(arg2, arg3) - - conf_select_create_mock.side_effect = mocked_conf_select - - config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json" - with open(config_file, "r") as f: - json_content = json.load(f) - - version = '2.3.0.0-1234' - json_content['commandParams']['version'] = version - json_content['clusterLevelParams']['stack_version'] = "2.3" - json_content['roleParams']['upgrade_suspended'] = "true" - - self.executeScript("after-INSTALL/scripts/hook.py", - classname="AfterInstallHook", - command="hook", - stack_version = self.STACK_VERSION, - target=RMFTestCase.TARGET_STACK_HOOKS, - config_dict = json_content, - config_overrides = self.CONFIG_OVERRIDES) - - # same assertions as test_hook_default_conf_select, but skip hdp-select set all - - self.assertResourceCalled('XmlConfig', 'core-site.xml', - owner = 'hdfs', - group = 'hadoop', - conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", - configurations = self.getConfig()['configurations']['core-site'], - configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", - xml_include_file=None) - - self.assertResourceCalled('Directory', - '/usr/lib/ambari-logsearch-logfeeder/conf', - mode = 0755, - cd_access = 'a', - create_parents = True) - - package_dirs = conf_select.get_package_dirs(); - for package, dir_defs in package_dirs.iteritems(): - for dir_def in dir_defs: - conf_dir = dir_def['conf_dir'] - conf_backup_dir = conf_dir + ".backup" - current_dir = dir_def['current_dir'] - self.assertResourceCalled('Execute', ('cp', '-R', '-p', conf_dir, conf_backup_dir), - not_if = 'test -e ' + conf_backup_dir, - sudo = True,) - - self.assertResourceCalled('Directory', conf_dir, action = ['delete'],) - self.assertResourceCalled('Link', conf_dir, to = current_dir,) - - self.assertNoMoreResources() - @patch("resource_management.core.Logger.warning") @patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1234")) diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py b/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py deleted file mode 100644 index a893d75..0000000 --- a/ambari-server/src/test/python/stacks/2.2/common/test_conf_select.py +++ /dev/null @@ -1,200 +0,0 @@ -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - -import pprint -import os -from mock.mock import patch, MagicMock -from stacks.utils.RMFTestCase import * -from resource_management.core.logger import Logger -from resource_management.libraries.functions import conf_select -from resource_management.libraries.script import Script - -class TestConfSelect(RMFTestCase): - - def setUp(self): - Logger.initialize_logger() - - # required for the test to run since the Execute calls need this - from resource_management.core.environment import Environment - self.env = Environment(test_mode=True) - self.env.__enter__() - - Script.config = dict() - Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } ) - Script.config["configurations"]["cluster-env"]["stack_packages"] = RMFTestCase.get_stack_packages() - Script.config["clusterLevelParams"] = { "stack_name" : "HDP" } - - - def tearDown(self): - self.env.__exit__(None,None,None) - - - @patch("resource_management.libraries.functions.conf_select._valid", new = MagicMock(return_value=True)) - def test_select_throws_error(self): - """ - Tests that conf-select throws errors correctly - :return: - """ - try: - conf_select.select("foo", "bar", "version", ignore_errors = False) - self.fail("Expected an error from conf-select") - except: - pass - - conf_select.select("foo", "bar", "version", ignore_errors = True) - - - @patch("resource_management.core.shell.call") - @patch("resource_management.libraries.functions.conf_select._valid", new = MagicMock(return_value=True)) - @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_path", new = MagicMock(return_value="/usr/bin/conf-select")) - def test_create_seeds_configuration_directories(self, shell_call_mock): - """ - Tests that conf-select seeds new directories - :return: - """ - - def mock_call(command, **kwargs): - """ - Instead of shell.call, call a command whose output equals the command. - :param command: Command that will be echoed. - :return: Returns a tuple of (process output code, stdout, stderr) - """ - return (0, "/etc/foo/conf", None) - - shell_call_mock.side_effect = mock_call - conf_select.create("HDP", "oozie", "version") - - self.assertEqual(pprint.pformat(self.env.resource_list), - "[Directory['/etc/foo/conf'],\n " - "Execute['ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E cp -R -p -v /usr/hdp/current/oozie-client/conf/* /etc/foo/conf']]") - - - def test_symlink_conversion_bad_linkto(self): - """ - Tests that a bad enum throws an exception. - :return: - """ - packages = conf_select.get_package_dirs() - - try: - conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", - packages["hadoop"], link_to = "INVALID") - raise Exception("Expected failure when supplying a bad enum for link_to") - except: - pass - - - @patch("resource_management.core.shell.call") - @patch.object(os.path, "isdir") - @patch.object(os.path, "exists") - @patch.object(os.path, "islink") - @patch("resource_management.libraries.functions.conf_select._valid", new = MagicMock(return_value = True)) - @patch("resource_management.libraries.functions.conf_select.create", new = MagicMock(return_value = ["/etc/hadoop/2.3.0.0-1234/0"])) - @patch("resource_management.libraries.functions.conf_select.select", new = MagicMock()) - def test_symlink_conversion_to_current(self, islink_mock, path_mock, isdir_mock, shell_call_mock): - """ - Tests that conf-select creates the correct symlink directories. - :return: - """ - - def mock_call(command, **kwargs): - """ - Instead of shell.call, call a command whose output equals the command. - :param command: Command that will be echoed. - :return: Returns a tuple of (process output code, stdout, stderr) - """ - return (0, "/etc/hadoop/conf", None) - - def path_mock_call(path): - if path == "/etc/hadoop/conf": - return True - - if path == "/etc/hadoop/2.3.0.0-1234/0": - return True - - return False - - def islink_mock_call(path): - if path == "/etc/hadoop/conf": - return False - - return False - - def isdir_mock_call(path): - if path == "/etc/hadoop/conf": - return True - - return False - - - packages = conf_select.get_package_dirs() - - path_mock.side_effect = path_mock_call - islink_mock.side_effect = islink_mock_call - shell_call_mock.side_effect = mock_call - conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", packages["hadoop"]) - - self.assertEqual(pprint.pformat(self.env.resource_list[0]), "Execute[('cp', '-R', '-p', u'/etc/hadoop/conf', u'/etc/hadoop/conf.backup')]") - self.assertEqual(pprint.pformat(self.env.resource_list[1]), "Directory['/etc/hadoop/conf']") - self.assertEqual(pprint.pformat(self.env.resource_list[2]), "Link['/etc/hadoop/conf']") - - - @patch.object(os.path, "exists", new = MagicMock(return_value = True)) - @patch.object(os.path, "islink", new = MagicMock(return_value = True)) - @patch.object(os, "readlink", new = MagicMock(return_value = "/etc/component/invalid")) - @patch("resource_management.libraries.functions.conf_select._valid", new = MagicMock(return_value = True)) - @patch("resource_management.libraries.functions.conf_select.create", new = MagicMock(return_value = ["/etc/hadoop/2.3.0.0-1234/0"])) - @patch("resource_management.libraries.functions.conf_select.select", new = MagicMock()) - def test_symlink_conversion_relinks_wrong_link(self): - """ - Tests that conf-select symlinking can detect a wrong directory - :return: - """ - packages = conf_select.get_package_dirs() - - conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", - packages["hadoop"]) - - self.assertEqual(pprint.pformat(self.env.resource_list), - "[Link['/etc/hadoop/conf'], Link['/etc/hadoop/conf']]") - - - @patch.object(os.path, "exists", new = MagicMock(return_value = False)) - @patch("resource_management.libraries.functions.conf_select._valid", new = MagicMock(return_value = True)) - def test_symlink_noop(self): - """ - Tests that conf-select symlinking does nothing if the directory doesn't exist - :return: - """ - packages = conf_select.get_package_dirs() - - conf_select.convert_conf_directories_to_symlinks("hadoop", "2.3.0.0-1234", packages["hadoop"]) - - self.assertEqual(pprint.pformat(self.env.resource_list), "[]") - - - def test_restrictions(self): - - Script.config.update({'roleParameters': {'cluster_version_summary': {'services': {'HIVE': {'upgrade': True}}}}}) - - restricted = conf_select.get_restricted_packages() - self.assertTrue("hive" in restricted) - self.assertTrue("hive-hcatalog" in restricted) - self.assertTrue("hive2" in restricted) - self.assertTrue("tez_hive2" in restricted) - self.assertTrue("hadoop" not in restricted)