Repository: ambari Updated Branches: refs/heads/branch-2.5 4078c4867 -> 282c4e213
AMBARI-21530 - Service Checks During Upgrades Should Use Desired Stack (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/282c4e21 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/282c4e21 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/282c4e21 Branch: refs/heads/branch-2.5 Commit: 282c4e213056d351331adac498d4655b8ebc9251 Parents: 4078c48 Author: Jonathan Hurley <[email protected]> Authored: Wed Jul 19 22:04:07 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Thu Jul 20 08:09:38 2017 -0400 ---------------------------------------------------------------------- .../AmbariCustomCommandExecutionHelper.java | 5 -- .../internal/UpgradeResourceProvider.java | 50 +++++++++++----- .../2.1.0.2.0/package/scripts/historyserver.py | 2 +- .../2.1.0.2.0/package/scripts/params_linux.py | 11 ++-- .../2.1.0.2.0/package/scripts/service_check.py | 6 +- .../src/test/python/TestStackFeature.py | 61 ++++++++++++++++++++ 6 files changed, 103 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 2eca76c..723a10d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -27,7 +27,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_CO import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION; @@ -40,7 +39,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JD import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS; @@ -772,9 +770,6 @@ public class AmbariCustomCommandExecutionHelper { } commandParams.put(COMMAND_TIMEOUT, commandTimeout); - commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); - commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); - execCmd.setCommandParams(commandParams); if (actionParameters != null) { // If defined http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 0b47ff1..de3225f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1337,6 +1337,37 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } /** + * Adds the hooks and service folders based on the effective stack ID and the + * name of the service from the wrapper. + * + * @param wrapper + * the stage wrapper to use when detemrining the service name. + * @param effectiveStackId + * the stack ID to use when getting the hooks and service folders. + * @param commandParams + * the params to update with the new values + * @throws AmbariException + */ + private void applyRepositoryAssociatedParameters(StageWrapper wrapper, StackId effectiveStackId, + Map<String, String> commandParams) throws AmbariException { + if (CollectionUtils.isNotEmpty(wrapper.getTasks()) + && wrapper.getTasks().get(0).getService() != null) { + + AmbariMetaInfo ambariMetaInfo = s_metaProvider.get(); + + StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(), + effectiveStackId.getStackVersion()); + + String serviceName = wrapper.getTasks().get(0).getService(); + ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(), + effectiveStackId.getStackVersion(), serviceName); + + commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); + commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); + } + } + + /** * Creates an action stage using the {@link #EXECUTE_TASK_ROLE} custom action * to execute some Python command. * @@ -1389,21 +1420,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // the ru_execute_tasks invokes scripts - it needs information about where // the scripts live and for that it should always use the target repository // stack - if (CollectionUtils.isNotEmpty(wrapper.getTasks()) - && wrapper.getTasks().get(0).getService() != null) { - - AmbariMetaInfo ambariMetaInfo = s_metaProvider.get(); - - StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(), - effectiveStackId.getStackVersion()); - - String serviceName = wrapper.getTasks().get(0).getService(); - ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(), - effectiveStackId.getStackVersion(), serviceName); - - params.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder()); - params.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); - } + applyRepositoryAssociatedParameters(wrapper, effectiveStackId, params); ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), EXECUTE_TASK_ROLE, Collections.singletonList(filter), params); @@ -1567,6 +1584,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // Apply additional parameters to the command that come from the stage. applyAdditionalParameters(wrapper, commandParams); + // add things like hooks and service folders based on effective repo + applyRepositoryAssociatedParameters(wrapper, effectiveStackId, commandParams); + ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), "SERVICE_CHECK", filters, commandParams); http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py index 7405a4e..36417a8 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py @@ -92,7 +92,7 @@ class HistoryServerDefault(HistoryServer): env.set_params(params) self.configure(env) # FOR SECURITY - if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major): + if check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.version_for_stack_feature_checks): # MC Hammer said, "Can't touch this" resource_created = copy_to_hdfs( "mapreduce", http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 5f400a3..a4f7ee9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -66,21 +66,20 @@ tarball_map = default("/configurations/cluster-env/tarball_map", None) config_path = os.path.join(stack_root, "current/hadoop-client/conf") config_dir = os.path.realpath(config_path) +# get the correct version to use for checking stack features +version_for_stack_feature_checks = get_stack_feature_version(config) + # This is expected to be of the form #.#.#.# stack_version_unformatted = config['hostLevelParams']['stack_version'] -stack_version_formatted_major = format_stack_version(stack_version_unformatted) stack_version_formatted = functions.get_stack_version('hadoop-yarn-resourcemanager') -stack_supports_ru = stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted_major) -stack_supports_timeline_state_store = stack_version_formatted_major and check_stack_feature(StackFeature.TIMELINE_STATE_STORE, stack_version_formatted_major) +stack_supports_ru = check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks) +stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_STATE_STORE, version_for_stack_feature_checks) # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade. # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) -# get the correct version to use for checking stack features -version_for_stack_feature_checks = get_stack_feature_version(config) - stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py index a36e8cc..1193b60 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py @@ -95,11 +95,7 @@ class ServiceCheckDefault(ServiceCheck): mode=params.smoke_hdfs_user_mode, ) - if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major): - path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar") - else: - path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar" - + path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar") yarn_distrubuted_shell_check_params = ["yarn org.apache.hadoop.yarn.applications.distributedshell.Client", "-shell_command", "ls", "-num_containers", "{number_of_nm}", "-jar", "{path_to_distributed_shell_jar}", "-timeout", "300000", http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/test/python/TestStackFeature.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py index 230734c..6e8bcec 100644 --- a/ambari-server/src/test/python/TestStackFeature.py +++ b/ambari-server/src/test/python/TestStackFeature.py @@ -21,10 +21,13 @@ limitations under the License. from resource_management.core.logger import Logger from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.script import Script from resource_management.core.exceptions import Fail from unittest import TestCase +import json + Logger.initialize_logger() class TestStackFeature(TestCase): @@ -115,6 +118,34 @@ class TestStackFeature(TestCase): stack_feature_version = get_stack_feature_version(command_json) self.assertEqual("2.5.9.9-9999", stack_feature_version) + + def test_get_stack_feature(self): + """ + Tests the stack feature version calculated during a STOP command in a downgrade. + :return: + """ + command_json = TestStackFeature._get_cluster_upgrade_restart_json() + Script.config = command_json + + Script.config["configurations"] = {} + Script.config["configurations"]["cluster-env"] = {} + Script.config["configurations"]["cluster-env"]["stack_features"] = {} + Script.config["configurations"]["cluster-env"]["stack_features"] = json.dumps(TestStackFeature._get_stack_feature_json()) + + stack_feature_version = get_stack_feature_version(command_json) + self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version)) + self.assertTrue(check_stack_feature("stack-feature-2", stack_feature_version)) + self.assertFalse(check_stack_feature("stack-feature-3", stack_feature_version)) + + command_json = TestStackFeature._get_cluster_install_command_json() + Script.config.update(command_json) + + stack_feature_version = get_stack_feature_version(command_json) + self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version)) + self.assertTrue(check_stack_feature("stack-feature-2", stack_feature_version)) + self.assertFalse(check_stack_feature("stack-feature-3", stack_feature_version)) + + @staticmethod def _get_cluster_install_command_json(): """ @@ -221,4 +252,34 @@ class TestStackFeature(TestCase): "version":"2.5.9.9-9999", "downgrade_from_version":"2.5.9.9-9999" } + } + + @staticmethod + def _get_stack_feature_json(): + """ + A STOP command during a downgrade. + :return: + """ + return { + "HDP": { + "stack_features":[ + { + "name":"stack-feature-1", + "description":"Stack Feature 1", + "min_version":"2.2.0.0" + }, + { + "name":"stack-feature-2", + "description":"Stack Feature 2", + "min_version":"2.2.0.0", + "max_version":"2.6.0.0" + }, + { + "name":"stack-feature-3", + "description":"Stack Feature 3", + "min_version":"2.2.0.0", + "max_version":"2.3.0.0" + } + ] + } } \ No newline at end of file
