This is an automated email from the ASF dual-hosted git repository. jonathanhurley pushed a commit to branch branch-feature-AMBARI-14714 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push: new fb09ba2 [AMBARI-24282] - Report MPack Version on Start/Status (#1750) fb09ba2 is described below commit fb09ba2af256deebb7917de0c3e7839d225ede11 Author: Jonathan Hurley <jonathanhur...@apache.org> AuthorDate: Thu Jul 12 08:24:45 2018 -0400 [AMBARI-24282] - Report MPack Version on Start/Status (#1750) --- .../libraries/functions/decorator.py | 17 +++ .../libraries/functions/mpack_manager_helper.py | 21 +++ .../libraries/functions/stack_select.py | 20 ++- .../libraries/functions/stack_tools.py | 8 ++ .../libraries/functions/version_select_util.py | 5 +- .../resource_management/libraries/script/script.py | 105 +++++++-------- .../apache/ambari/server/agent/CommandReport.java | 6 +- .../ambari/server/agent/HeartbeatProcessor.java | 102 ++++++++------ .../ambari/server/agent/StructuredOutputType.java | 73 +++++++++++ .../alerts/ComponentVersionAlertRunnable.java | 19 ++- .../HostComponentVersionAdvertisedEvent.java | 71 +++++----- .../hostcomponents/VersionReportedListener.java | 103 +++++++++++++++ .../upgrade/MpackInstallStateListener.java | 16 ++- .../orm/entities/HostComponentStateEntity.java | 146 ++++++++++----------- .../serveraction/upgrades/AddComponentAction.java | 2 +- .../ambari/server/state/ServiceComponentHost.java | 21 ++- .../apache/ambari/server/state/UpgradeHelper.java | 2 +- .../apache/ambari/server/state/UpgradeState.java | 7 - .../svccomphost/ServiceComponentHostImpl.java | 25 +++- .../src/main/resources/Ambari-DDL-Derby-CREATE.sql | 1 + .../src/main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 1 + .../main/resources/Ambari-DDL-Postgres-CREATE.sql | 1 + .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 1 + .../main/resources/Ambari-DDL-SQLServer-CREATE.sql | 1 + .../1.10.3-30/package/scripts/kerberos_client.py | 32 +++-- .../alerts/ComponentVersionAlertRunnableTest.java | 15 +++ .../UpgradeSummaryResourceProviderTest.java | 4 +- .../state/DefaultServiceCalculatedStateTest.java | 4 +- .../state/FlumeServiceCalculatedStateTest.java | 2 +- .../state/HBaseServiceCalculatedStateTest.java | 6 +- .../state/HDFSServiceCalculatedStateTest.java | 6 +- .../state/HiveServiceCalculatedStateTest.java | 10 +- .../state/OozieServiceCalculatedStateTest.java | 4 +- .../state/YarnServiceCalculatedStateTest.java | 6 +- .../publishers/VersionEventPublisherTest.java | 8 +- .../upgrades/AddComponentActionTest.java | 2 +- .../upgrades/ComponentVersionCheckActionTest.java | 10 +- ...ceComponentHostConcurrentWriteDeadlockTest.java | 4 +- 39 files changed, 603 insertions(+), 285 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/decorator.py b/ambari-common/src/main/python/resource_management/libraries/functions/decorator.py index 9446d56..ff0e472 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/decorator.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/decorator.py @@ -127,3 +127,20 @@ def experimental(feature=None, comment=None, disable=False): return wrapper return decorator + +def deprecated(comment=None, disable=False): + """ + Annotates a function as being deprecated, optionally logging a warning comment. + :param comment: the comment to log + :param disable True to skip invocation of the method entirely, defaults to False. + :return: + """ + def decorator(function): + def wrapper(*args, **kwargs): + if comment: + Logger.warning(comment) + + if not disable: + return function(*args, **kwargs) + return wrapper + return decorator diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py index c79402e..dcce601 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py @@ -28,6 +28,8 @@ RUN_DIR_KEY_NAME = 'run_dir' PATH_KEY_NAME = 'mpack_path' COMPONENTS_PLURAL_KEY_NAME = 'components' COMPONENT_INSTANCES_PLURAL_KEY_NAME = 'component-instances' +MPACK_VERSION_KEY_NAME = 'mpack_version' +MODULE_VERSION_KEY_NAME = 'module_version' def get_component_conf_path(mpack_name, instance_name, module_name, components_instance_type, @@ -83,6 +85,25 @@ def get_component_target_path(mpack_name, instance_name, module_name, components COMPONENT_INSTANCES_PLURAL_KEY_NAME][component_instance_name][PATH_KEY_NAME] +def get_versions(mpack_name, instance_name, module_name, components_instance_type, + subgroup_name='default', component_instance_name='default'): + """ + :returns a tuple representing the mpack version and the module version + :raises ValueError if the parameters doesn't match the mpack or instances structure + """ + + instances_json = list_instances(mpack_name, instance_name, subgroup_name, module_name, + {components_instance_type: [component_instance_name]}) + + mpack_version = instances_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][ + COMPONENT_INSTANCES_PLURAL_KEY_NAME][component_instance_name][MPACK_VERSION_KEY_NAME] + + module_version = instances_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][ + COMPONENT_INSTANCES_PLURAL_KEY_NAME][component_instance_name][MODULE_VERSION_KEY_NAME] + + return mpack_version, module_version + + def get_component_home_path(mpack_name, instance_name, module_name, components_instance_type, subgroup_name='default', component_instance_name='default'): """ diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index acd0bd7..e204a2d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -35,6 +35,7 @@ from resource_management.libraries.functions.format import format from resource_management.libraries.script.script import Script from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions import stack_settings +from resource_management.libraries.functions.decorator import deprecated from resource_management.core import shell from resource_management.core import sudo from resource_management.core.shell import call @@ -89,6 +90,7 @@ _PACKAGE_SCOPES = (PACKAGE_SCOPE_INSTALL, PACKAGE_SCOPE_STANDARD, PACKAGE_SCOPE_ # the orchestration types which equal to a partial (non-STANDARD) upgrade _PARTIAL_ORCHESTRATION_SCOPES = ("PATCH", "MAINT") +@deprecated(comment = "The stack-select tools are no longer used") def get_package_name(default_package = None): """ Gets the stack-select package name for the service name and @@ -118,7 +120,7 @@ def get_package_name(default_package = None): else: raise - +@deprecated(comment = "The stack-select tools are no longer used") def is_package_supported(package, supported_packages = None): """ Gets whether the specified package is supported by the <stack_select> tool. @@ -138,6 +140,7 @@ def is_package_supported(package, supported_packages = None): return False +@deprecated(comment = "The stack-select tools are no longer used") def get_supported_packages(): """ Parses the output from <stack-select> packages and returns an array of the various packages. @@ -157,6 +160,7 @@ def get_supported_packages(): return [line.strip() for line in stdout.splitlines()] +@deprecated(comment = "The stack-select tools are no longer used") def get_packages(scope, service_name = None, component_name = None): """ Gets the packages which should be used with the stack's stack-select tool for the @@ -253,6 +257,7 @@ def get_packages(scope, service_name = None, component_name = None): return packages +@deprecated(comment = "The stack-select tools are no longer used") def select_all(version_to_select): """ Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a @@ -275,6 +280,7 @@ def select_all(version_to_select): Execute(command, only_if = only_if_command) +@deprecated(comment = "The stack-select tools are no longer used") def select_packages(version): """ Uses the command's service and role to determine the stack-select packages which need to be invoked. @@ -307,6 +313,7 @@ def select_packages(version): select(stack_select_package_name, version) +@deprecated(comment = "The stack-select tools are no longer used") def select(component, version): """ Executes <stack-selector-tool> on the specific component and version. Some global @@ -336,7 +343,7 @@ def select(component, version): reload(module) Logger.info("After {0}, reloaded module {1}".format(command, moduleName)) - +@deprecated(comment = "The stack-select tools are no longer used") def get_role_component_current_stack_version(): """ Gets the current HDP version of the component that this role command is for. @@ -370,7 +377,7 @@ def get_role_component_current_stack_version(): return current_stack_version - +@deprecated(comment = "The stack-select tools are no longer used") def get_hadoop_dir(target): """ Return the hadoop shared directory which should be used for the command's component. The @@ -408,6 +415,7 @@ def get_hadoop_dir(target): return hadoop_dir +@deprecated(comment = "The stack-select tools are no longer used") def get_hadoop_dir_for_stack_version(target, stack_version): """ Return the hadoop shared directory for the provided stack version. This is necessary @@ -429,6 +437,7 @@ def get_hadoop_dir_for_stack_version(target, stack_version): return hadoop_dir +@deprecated(comment = "The stack-select tools are no longer used") def _get_upgrade_stack(): """ Gets the stack name and stack version if an upgrade is currently in progress. @@ -445,6 +454,7 @@ def _get_upgrade_stack(): return None +@deprecated(comment = "The stack-select tools are no longer used") def unsafe_get_stack_versions(): """ Gets list of stack versions installed on the host. @@ -460,6 +470,8 @@ def unsafe_get_stack_versions(): versions.append(line.rstrip('\n')) return (code, out, versions) + +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_versions(stack_root): """ Gets list of stack versions installed on the host. @@ -478,6 +490,8 @@ def get_stack_versions(stack_root): versions = get_versions_from_stack_root(stack_root) return versions + +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_version_before_install(component_name): """ Works in the similar way to '<stack-selector-tool> status component', diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py index 1bf4c6b..c16dd22 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py @@ -28,11 +28,13 @@ from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.core.utils import pad from resource_management.libraries.functions import stack_settings +from resource_management.libraries.functions.decorator import deprecated STACK_SELECTOR_NAME = "stack_selector" CONF_SELECTOR_NAME = "conf_selector" +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_tool(name): """ Give a tool selector name get the stack-specific tool name, tool path, tool package @@ -76,6 +78,8 @@ def get_stack_tool(name): # Return fixed length (tool_name, tool_path, tool_package) tuple return tuple(pad(tool_config[:3], 3)) + +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_tool_name(name): """ Give a tool selector name get the stack-specific tool name @@ -86,6 +90,7 @@ def get_stack_tool_name(name): return tool_name +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_tool_path(name): """ Give a tool selector name get the stack-specific tool path @@ -96,6 +101,7 @@ def get_stack_tool_path(name): return tool_path +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_tool_package(name): """ Give a tool selector name get the stack-specific tool package @@ -106,6 +112,7 @@ def get_stack_tool_package(name): return tool_package +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_root(stack_name, stack_root_json): """ Get the stack-specific install root directory from the raw, JSON-escaped properties. @@ -127,6 +134,7 @@ def get_stack_root(stack_name, stack_root_json): return stack_root[stack_name] +@deprecated(comment = "The stack-select tools are no longer used") def get_stack_name(stack_formatted): """ Get the stack name (eg. HDP) from formatted string that may contain stack version (eg. HDP-2.6.1.0-123) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py index 538e427..bc19b68 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py @@ -26,8 +26,9 @@ import tempfile from resource_management.core.logger import Logger from resource_management.core import shell from resource_management.libraries.functions import stack_tools +from resource_management.libraries.functions.decorator import deprecated - +@deprecated(comment = "The stack-select tools are no longer used") def get_component_version_from_symlink(stack_name, component_name): """ Gets the version of the specified component by invoking the stack-select tool to query for the @@ -72,6 +73,7 @@ def get_component_version_from_symlink(stack_name, component_name): return version +@deprecated(comment = "The stack-select tools are no longer used") def get_component_version_with_stack_selector(stack_selector_path, component_name): """ For specific cases where we deal with HDP add on services from a management pack, the version @@ -113,6 +115,7 @@ def get_component_version_with_stack_selector(stack_selector_path, component_nam (component_name, get_stack_comp_version_cmd, str(code), str(out))) return version +@deprecated(comment = "The stack-select tools are no longer used") def get_versions_from_stack_root(stack_root): """ Given a stack install root, returns a list of stack versions currently installed. diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index c8d0197..ddb5d1d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -196,9 +196,13 @@ class Script(object): Logger.info("Reporting installation state for {0}".format(command_repository)) - self.put_structured_out({"mpackId": command_repository.mpack_id}) - self.put_structured_out({"mpackName":command_repository.mpack_name}) - self.put_structured_out({"mpackVersion":command_repository.version_string}) + mpack_dictionary = { + "mpackId": command_repository.mpack_id, + "mpackName":command_repository.mpack_name, + "mpackVersion":command_repository.version_string + } + + self.put_structured_out({"mpack_installation": mpack_dictionary}) def save_component_version_to_structured_out(self, command_name): @@ -207,52 +211,57 @@ class Script(object): command is an install command and the repository is trusted, then it will use the version of the repository. Otherwise, it will consult the stack-select tool to read the symlink version. - Under rare circumstances, a component may have a bug which prevents it from reporting a - version back after being installed. This is most likely due to the stack-select tool not being - invoked by the package's installer. In these rare cases, we try to see if the component - should have reported a version and we try to fallback to the "<stack-select> versions" command. - :param command_name: command name :return: None """ - from resource_management.libraries.functions import stack_select + from resource_management.libraries.functions import mpack_manager_helper - stack_name = Script.get_stack_name() - stack_select_package_name = stack_select.get_package_name() + if self.is_hook(): + return; - if stack_select_package_name and stack_name: - component_version = version_select_util.get_component_version_from_symlink(stack_name, stack_select_package_name) + execution_command = self.get_execution_command() + mpack_name = execution_command.get_mpack_name() + mpack_instance_name = execution_command.get_servicegroup_name() + module_name = execution_command.get_module_name() + component_type = execution_command.get_component_type() - if component_version: - self.put_structured_out({"version": component_version}) - # if repository_version_id is passed, pass it back with the version - from resource_management.libraries.functions.default import default - repo_version_id = default("/repositoryFile/repoVersionId", None) - if repo_version_id: - self.put_structured_out({"repository_version_id": repo_version_id}) - else: - if not self.is_hook(): - Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name)) + try: + mpack_version, component_version = mpack_manager_helper.get_versions(mpack_name, + mpack_instance_name, module_name, component_type ) + + mpack_version_dictionary = { + "mpackVersion": mpack_version, + "version": component_version + } + + self.put_structured_out({"version_reporting": mpack_version_dictionary}) + except ValueError: + Logger.exception( + "The '{0}' component from {1} did not advertise a version. This may indicate a problem with the mpack JSON.".format( + component_type, mpack_name)) + except Exception as e: + Logger.exception( + "The '{0}' component from {1} did not advertise a version. This may indicate a problem with the mpack JSON.".format( + component_type, mpack_name)) def should_expose_component_version(self, command_name): """ Analyzes config and given command to determine if stack version should be written - to structured out. Currently only HDP stack versions >= 2.2 are supported. + to structured out. :param command_name: command name :return: True or False """ from resource_management.libraries.functions.default import default - stack_version_unformatted = self.execution_command.get_mpack_version() - stack_version_formatted = format_stack_version(stack_version_unformatted) - if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): - if command_name.lower() == "status": - request_version = default("/commandParams/request_version", None) - if request_version is not None: - return True - else: - # Populate version only on base commands - return command_name.lower() == "start" or command_name.lower() == "install" or command_name.lower() == "restart" + if command_name.lower() == "status": + request_version = default("/commandParams/request_version", None) + if request_version is not None: + return True + else: + # Populate version only on base commands + version_reporting_commands = ["start", "install", "restart"] + return command_name.lower() in version_reporting_commands + return False def execute(self): @@ -908,8 +917,7 @@ class Script(object): """ self.fail_with_error("stop method isn't implemented") - # TODO, remove after all services have switched to pre_upgrade_restart - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env): """ To be overridden by subclasses """ @@ -955,12 +963,7 @@ class Script(object): if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower(): if is_stack_upgrade: - # Remain backward compatible with the rest of the services that haven't switched to using - # the pre_upgrade_restart method. Once done. remove the else-block. - if "pre_upgrade_restart" in dir(self): - self.pre_upgrade_restart(env, upgrade_type=upgrade_type) - else: - self.pre_rolling_restart(env) + self.pre_upgrade_restart(env, upgrade_type=upgrade_type) self.install(env) else: @@ -977,10 +980,7 @@ class Script(object): if is_stack_upgrade: # Remain backward compatible with the rest of the services that haven't switched to using # the pre_upgrade_restart method. Once done. remove the else-block. - if "pre_upgrade_restart" in dir(self): - self.pre_upgrade_restart(env, upgrade_type=upgrade_type) - else: - self.pre_rolling_restart(env) + self.pre_upgrade_restart(env, upgrade_type=upgrade_type) service_name = config['serviceName'] if config is not None and 'serviceName' in config else None try: @@ -1011,19 +1011,10 @@ class Script(object): self.post_start(env) if is_stack_upgrade: - # Remain backward compatible with the rest of the services that haven't switched to using - # the post_upgrade_restart method. Once done. remove the else-block. - if "post_upgrade_restart" in dir(self): - self.post_upgrade_restart(env, upgrade_type=upgrade_type) - else: - self.post_rolling_restart(env) - - if self.should_expose_component_version(self.command_name): - self.save_component_version_to_structured_out(self.command_name) + self.post_upgrade_restart(env, upgrade_type=upgrade_type) - # TODO, remove after all services have switched to post_upgrade_restart - def post_rolling_restart(self, env): + def post_upgrade_restart(self, env): """ To be overridden by subclasses """ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java index 9fbcb81..fe45100 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java @@ -21,9 +21,7 @@ import java.util.Map; import org.codehaus.jackson.annotate.JsonProperty; - - -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; @@ -244,7 +242,7 @@ public class CommandReport { */ @Override public String toString() { - return Objects.toStringHelper(this).add("role", role) + return MoreObjects.toStringHelper(this).add("role", role) .add("actionId", actionId) .add("status", status) .add("exitCode", exitCode) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java index 6fc08f3..e5bf9dc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java @@ -31,6 +31,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -40,7 +42,6 @@ import org.apache.ambari.server.ServiceNotFoundException; import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; -import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.events.AlertEvent; @@ -67,8 +68,6 @@ import org.apache.ambari.server.state.UpgradeState; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; import org.apache.ambari.server.state.host.HostStatusUpdatesReceivedEvent; import org.apache.ambari.server.state.scheduler.RequestExecution; -import org.apache.ambari.server.state.stack.upgrade.Direction; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpFailedEvent; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpSucceededEvent; @@ -82,6 +81,8 @@ import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.AbstractService; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; import com.google.gson.annotations.SerializedName; import com.google.inject.Inject; @@ -370,6 +371,15 @@ public class HeartbeatProcessor extends AbstractService{ } } + @Nullable + JsonObject structuredOutputJson = null; + String structuredOutputString = report.getStructuredOut(); + if (StringUtils.isNotBlank(structuredOutputString) + && !StringUtils.equals(structuredOutputString, "{}")) { + JsonElement element = gson.fromJson(structuredOutputString, JsonElement.class); + structuredOutputJson = element.getAsJsonObject(); + } + // If the report indicates the keytab file was successfully transferred to a host or removed // from a host, record this for future reference if (Service.Type.KERBEROS.name().equalsIgnoreCase(report.getServiceName()) && @@ -382,7 +392,10 @@ public class HeartbeatProcessor extends AbstractService{ if (SET_KEYTAB.equalsIgnoreCase(customCommand)) { WriteKeytabsStructuredOut writeKeytabsStructuredOut; try { - writeKeytabsStructuredOut = gson.fromJson(report.getStructuredOut(), WriteKeytabsStructuredOut.class); + JsonElement setKeytabsStructuredOutRoot = structuredOutputJson.get( + StructuredOutputType.SET_KEYTABS.getRoot()); + + writeKeytabsStructuredOut = gson.fromJson(setKeytabsStructuredOutRoot, WriteKeytabsStructuredOut.class); } catch (JsonSyntaxException ex) { //Json structure was incorrect do nothing, pass this data further for processing writeKeytabsStructuredOut = null; @@ -404,7 +417,12 @@ public class HeartbeatProcessor extends AbstractService{ } } } else if (CHECK_KEYTABS.equalsIgnoreCase(customCommand)) { - ListKeytabsStructuredOut structuredOut = gson.fromJson(report.getStructuredOut(), ListKeytabsStructuredOut.class); + JsonElement checkKeytabsStructuredOutRoot = structuredOutputJson.get( + StructuredOutputType.CHECK_KEYTABS.getRoot()); + + ListKeytabsStructuredOut structuredOut = gson.fromJson(checkKeytabsStructuredOutRoot, + ListKeytabsStructuredOut.class); + for (MissingKeytab each : structuredOut.missingKeytabs) { LOG.info("Missing principal: {} for keytab: {} on host: {}", each.principal, each.keytabFilePath, hostName); KerberosKeytabPrincipalEntity kkpe = kerberosKeytabPrincipalDAO.findByHostKeytabAndPrincipal(host.getHostId(), each.keytabFilePath, each.principal); @@ -428,6 +446,7 @@ public class HeartbeatProcessor extends AbstractService{ if (service == null || service.isEmpty()) { throw new AmbariException("Invalid command report, service: " + service); } + if (actionMetadata.getActions(service.toLowerCase()).contains(report.getRole())) { LOG.debug("{} is an action - skip component lookup", report.getRole()); } else { @@ -438,25 +457,26 @@ public class HeartbeatProcessor extends AbstractService{ String schName = scHost.getServiceComponentName(); if (report.getStatus().equals(HostRoleStatus.COMPLETED.toString())) { - // Reading component version if it is present - if (StringUtils.isNotBlank(report.getStructuredOut()) - && !StringUtils.equals("{}", report.getStructuredOut())) { - ComponentVersionStructuredOut structuredOutput = null; - try { - structuredOutput = gson.fromJson(report.getStructuredOut(), ComponentVersionStructuredOut.class); - } catch (JsonSyntaxException ex) { - //Json structure for component version was incorrect - //do nothing, pass this data further for processing - } + ComponentVersionStructuredOut componentVersionStructuredOut = null; + if (null != structuredOutputJson) { + JsonElement versionStructuredOutRoot = structuredOutputJson.get( + StructuredOutputType.VERSION_REPORTING.getRoot()); - String newVersion = structuredOutput == null ? null : structuredOutput.version; - Long mpackId = structuredOutput == null ? null : structuredOutput.mpackId; + if (null != versionStructuredOutRoot) { + try { + componentVersionStructuredOut = gson.fromJson(versionStructuredOutRoot, + ComponentVersionStructuredOut.class); - HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent( - cl, scHost, newVersion); + HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent( + cl, scHost, componentVersionStructuredOut); - versionEventPublisher.publish(event); + versionEventPublisher.publish(event); + } catch (JsonSyntaxException ex) { + // Json structure for component version was incorrect + // do nothing, pass this data further for processing + } + } } if (!scHost.getState().equals(org.apache.ambari.server.state.State.UPGRADING) && @@ -504,16 +524,19 @@ public class HeartbeatProcessor extends AbstractService{ hostName, now)); } } else if (report.getStatus().equals("FAILED")) { - - if (StringUtils.isNotBlank(report.getStructuredOut())) { + JsonElement upgradeStructedOutput = structuredOutputJson.get( + StructuredOutputType.UPGRADE_SUMMARY.getRoot()); + if (null != upgradeStructedOutput) { try { - ComponentVersionStructuredOut structuredOutput = gson.fromJson(report.getStructuredOut(), ComponentVersionStructuredOut.class); + UpgradeSummaryStructuredOuut upgradeStructuredOutput = gson.fromJson( + upgradeStructedOutput, UpgradeSummaryStructuredOuut.class); - if (null != structuredOutput.upgradeDirection) { + if (null != upgradeStructuredOutput.direction) { scHost.setUpgradeState(UpgradeState.FAILED); } } catch (JsonSyntaxException ex) { - LOG.warn("Structured output was found, but not parseable: {}", report.getStructuredOut()); + LOG.warn("Structured output was found, but not parseable: {}", + structuredOutputString); } } @@ -602,13 +625,6 @@ public class HeartbeatProcessor extends AbstractService{ List<Map<String, String>> list = (List<Map<String, String>>) extra.get("processes"); scHost.setProcesses(list); } - if (extra.containsKey("version")) { - String version = extra.get("version").toString(); - - HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cl, scHost, version); - versionEventPublisher.publish(event); - } - } catch (Exception e) { LOG.error("Could not access extra JSON for " + scHost.getServiceComponentName() + " from " + @@ -725,18 +741,20 @@ public class HeartbeatProcessor extends AbstractService{ /** * This class is used for mapping json of structured output for component START action. */ - private static class ComponentVersionStructuredOut { - @SerializedName("version") - private String version; + public static class ComponentVersionStructuredOut { + @SerializedName("mpackVersion") + public String mpackVersion; - @SerializedName("upgrade_type") - private UpgradeType upgradeType = null; + @SerializedName("version") + public String version; + } + /** + * This class is used for mapping json of structured output for information + * about an upgrade in progress. + */ + public static class UpgradeSummaryStructuredOuut { @SerializedName("direction") - private Direction upgradeDirection = null; - - @SerializedName(KeyNames.MPACK_ID) - private Long mpackId; - + public String direction; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/StructuredOutputType.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/StructuredOutputType.java new file mode 100644 index 0000000..c3588e4 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/StructuredOutputType.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.agent; + +/** + * Represents the type of output from commands which report a + * {@link CommandReport#getStructuredOut()}. This is used for deserializing + * output. + */ +public enum StructuredOutputType { + + /** + * The structured output from listing keytabs on a host. + */ + CHECK_KEYTABS("check_keytabs"), + + /** + * The structured output from writing out keytabs on a host. + */ + SET_KEYTABS("set_keytabs"), + + /** + * The structured output from an mpack installation. + */ + MPACK_INSTALLATION("mpack_installation"), + + /** + * The structured output from a start command which usually contains version + * and mpack information. + */ + VERSION_REPORTING("version_reporting"), + + /** + * Information about an upgrade in progress + */ + UPGRADE_SUMMARY("upgrade_summary"); + + /** + * The root JSON element. + */ + private final String m_root; + + /** + * Constructor. + * + * @param root the root JSON element which the structured data is stored under. + */ + private StructuredOutputType(String root) { + m_root = root; + } + + /** + * Gets the root of the JSON for a specific structured output type. + */ + public String getRoot() { + return m_root; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnable.java b/ambari-server/src/main/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnable.java index adef295..a5c70d5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnable.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnable.java @@ -135,8 +135,12 @@ public class ComponentVersionAlertRunnable extends AlertRunnable { ModuleComponent moduleComponent = mpack.getModuleComponent(hostComponent.getServiceType(), hostComponent.getServiceComponentType()); + // check for both mpack version and component version String version = hostComponent.getVersion(); - if (!StringUtils.equals(version, moduleComponent.getVersion())) { + String mpackVersion = hostComponent.getMpackVersion(); + + if (!StringUtils.equals(version, moduleComponent.getVersion()) + || !StringUtils.equals(mpackVersion, mpack.getVersion())) { Set<ServiceComponentHost> mismatchedComponents = versionMismatches.get(host); if (null == mismatchedComponents) { mismatchedComponents = new HashSet<>(); @@ -160,8 +164,17 @@ public class ComponentVersionAlertRunnable extends AlertRunnable { buffer.append(" ").append(host.getHostName()); buffer.append(System.lineSeparator()); for (ServiceComponentHost hostComponent : versionMismatches.get(host)) { - buffer.append(" ").append(hostComponent.getServiceComponentName()).append(": ").append( - hostComponent.getVersion()).append(System.lineSeparator()); + buffer.append(" ") + .append(hostComponent.getServiceComponentName()) + .append(": ") + .append(hostComponent.getDesiredStackId().getStackName()) + .append(" ") + .append(hostComponent.getMpackVersion()) + .append(", ") + .append(hostComponent.getServiceType()) + .append(" ") + .append(hostComponent.getVersion()) + .append(System.lineSeparator()); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/HostComponentVersionAdvertisedEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/HostComponentVersionAdvertisedEvent.java index 664bce6..067611d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/HostComponentVersionAdvertisedEvent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/HostComponentVersionAdvertisedEvent.java @@ -17,35 +17,22 @@ */ package org.apache.ambari.server.events; -import org.apache.ambari.annotations.Experimental; -import org.apache.ambari.annotations.ExperimentalFeature; +import org.apache.ambari.server.agent.HeartbeatProcessor.ComponentVersionStructuredOut; +import org.apache.ambari.server.agent.StructuredOutputType; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ServiceComponentHost; +import com.google.common.base.MoreObjects; + /** * The {@link HostComponentVersionAdvertisedEvent} * occurs when a Host Component advertises it's current version value. */ -@Deprecated -@Experimental(feature = ExperimentalFeature.VERSION_REPORTING) public class HostComponentVersionAdvertisedEvent extends ClusterEvent { - protected Cluster cluster; - protected ServiceComponentHost sch; - protected String version; - protected Long repoVersionId; - - /** - * Constructor. - * - * @param cluster: cluster. - * @param sch: the service component host - */ - public HostComponentVersionAdvertisedEvent(Cluster cluster, ServiceComponentHost sch, - String version, Long repoVersionId) { - this(cluster, sch, version); - this.repoVersionId = repoVersionId; - } + private final Cluster cluster; + private final ServiceComponentHost sch; + private final ComponentVersionStructuredOut componentVersionStructuredOut; /** * Constructor. @@ -53,28 +40,39 @@ public class HostComponentVersionAdvertisedEvent extends ClusterEvent { * @param cluster: cluster. * @param sch: the service component host */ - public HostComponentVersionAdvertisedEvent(Cluster cluster, ServiceComponentHost sch, - String version) { + public HostComponentVersionAdvertisedEvent(Cluster cluster, ServiceComponentHost sch, ComponentVersionStructuredOut componentVersionStructuredOut) { super(AmbariEventType.HOST_COMPONENT_VERSION_ADVERTISED, cluster.getClusterId()); this.cluster = cluster; this.sch = sch; - this.version = version; + this.componentVersionStructuredOut = componentVersionStructuredOut; } + /** + * Gets the component/host combination associated with this event. + * + * @return + */ public ServiceComponentHost getServiceComponentHost() { return sch; } + /** + * Gets the cluster associated with this event. + * + * @return + */ public Cluster getCluster() { return cluster; } - public String getVersion() { - return version; - } - - public Long getRepositoryVersionId() { - return repoVersionId; + /** + * Gets the structured output parsed from + * {@link StructuredOutputType#VERSION_REPORTING}. + * + * @return + */ + public ComponentVersionStructuredOut getStructuredOutput() { + return componentVersionStructuredOut; } /** @@ -82,14 +80,11 @@ public class HostComponentVersionAdvertisedEvent extends ClusterEvent { */ @Override public String toString() { - StringBuilder buffer = new StringBuilder("HostComponentVersionAdvertisedEvent{"); - buffer.append("cluserId=").append(m_clusterId); - buffer.append(", serviceName=").append(sch.getServiceName()); - buffer.append(", componentName=").append(sch.getServiceComponentName()); - buffer.append(", hostName=").append(sch.getHostName()); - buffer.append(", version=").append(version); - buffer.append(", repo_version_id=").append(repoVersionId); - buffer.append("}"); - return buffer.toString(); + return MoreObjects.toStringHelper(this) + .add("hostName", sch.getHostName()) + .add("service", sch.getServiceName()) + .add("component", sch.getServiceComponentName()) + .add("mpackVersion", componentVersionStructuredOut.mpackVersion) + .add("version", componentVersionStructuredOut.version).toString(); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/hostcomponents/VersionReportedListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/hostcomponents/VersionReportedListener.java new file mode 100644 index 0000000..7cf9db2 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/hostcomponents/VersionReportedListener.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.events.listeners.hostcomponents; + +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.EagerSingleton; +import org.apache.ambari.server.agent.HeartbeatProcessor.ComponentVersionStructuredOut; +import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; +import org.apache.ambari.server.events.publishers.VersionEventPublisher; +import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.eventbus.Subscribe; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +/** + * The {@link VersionReportedListener} is used to respond to Ambari events which + * deal with host components reporting events on startup and registration. + */ +@Singleton +@EagerSingleton +@Experimental(feature = ExperimentalFeature.UNIT_TEST_REQUIRED) +public class VersionReportedListener { + + /** + * Logger. + */ + private static final Logger LOG = LoggerFactory.getLogger(VersionReportedListener.class); + + /** + * Constructor. + * + * @param ambariEventPublisher + * @param lockFactory + */ + @Inject + public VersionReportedListener(VersionEventPublisher versionEventPublisher) { + versionEventPublisher.register(this); + } + + /** + * Handles the {@link HostComponentVersionAdvertisedEvent} which is fired when + * a component responds with {@link ComponentVersionStructuredOut}. This + * usually happens on start commands and on agent registration. + */ + @Subscribe + public void onVersionReportedEvent(HostComponentVersionAdvertisedEvent event) { + if (LOG.isDebugEnabled()) { + LOG.debug(event.toString()); + } + + try { + ServiceComponentHost sch = event.getServiceComponentHost(); + ComponentVersionStructuredOut stdOut = event.getStructuredOutput(); + + String mpackVersion = stdOut.mpackVersion; + String version = stdOut.version; + + if (StringUtils.isBlank(version)) { + version = "UNKNOWN"; + } + + if (StringUtils.isBlank(mpackVersion)) { + mpackVersion = "UNKNOWN"; + } + + String currentVersion = sch.getVersion(); + String currentMpackVersion = sch.getMpackVersion(); + + if (!StringUtils.equals(mpackVersion, currentMpackVersion) + || !StringUtils.equals(version, currentVersion)) { + try { + sch.setVersions(mpackVersion, version); + } catch (AmbariException ambariException) { + LOG.error("Unable to update the reported version for {} on {}", + sch.getServiceComponentName(), sch.getHostName(), ambariException); + } + } + } catch (Exception exception) { + LOG.error("Unable to handle version event {}", event, exception); + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java index 4e4de67..42bb125 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java @@ -28,6 +28,7 @@ import org.apache.ambari.server.EagerSingleton; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.StructuredOutputType; import org.apache.ambari.server.controller.internal.UpgradePlanInstallResourceProvider; import org.apache.ambari.server.events.CommandReportReceivedEvent; import org.apache.ambari.server.events.HostsAddedEvent; @@ -49,6 +50,8 @@ import org.slf4j.LoggerFactory; import com.google.common.eventbus.Subscribe; import com.google.common.util.concurrent.Striped; import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; import com.google.gson.annotations.SerializedName; import com.google.inject.Inject; @@ -234,9 +237,16 @@ public class MpackInstallStateListener { Long mpackId = null; try { - // try to parse the structured output of the install command - structuredOutput = m_gson.fromJson(commandReport.getStructuredOut(), - InstallCommandStructuredOutput.class); + if(null != commandReport.getStructuredOut()) { + JsonElement element = m_gson.fromJson (commandReport.getStructuredOut(), JsonElement.class); + JsonObject jsonObj = element.getAsJsonObject(); + JsonElement installReportingElement = jsonObj.get(StructuredOutputType.MPACK_INSTALLATION.getRoot()); + if(null != installReportingElement) { + // try to parse the structured output of the install command + structuredOutput = m_gson.fromJson(installReportingElement, + InstallCommandStructuredOutput.class); + } + } } catch (JsonSyntaxException jsonException) { LOG.error( "Unable to parse the installation structured output for command {} for {} on host {}", diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java index c696242..e829955 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java @@ -33,12 +33,12 @@ import javax.persistence.NamedQuery; import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.TableGenerator; - import javax.persistence.UniqueConstraint; import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.UpgradeState; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; @Entity @@ -113,7 +113,28 @@ public class HostComponentStateEntity { private String componentType; /** - * Version reported by host component during last status update. + * The mpack version reported by the host component during the last status + * update. Components are associated with more than a single version; they + * have an mpack version and then their specific component version. + * </p> + * + * <pre> + * /usr/vendor/mpacks/my-mpack/1.0.0-b450/some_component -> /usr/vendor/modules/some_component/3.4.0.0-b42 + * </pre> + * + * In this example, the version of the mpack can change, but still technically + * point to the same component version. This is why both are tracked. + * + * @see #version + */ + @Column(name = "mpack_version", nullable = false, insertable = true, updatable = true) + private String mpackVersion = State.UNKNOWN.toString(); + + /** + * The component version reported by the host component during the last status + * update. + * + * @see #mpackVersion */ @Column(name = "version", nullable = false, insertable = true, updatable = true) private String version = State.UNKNOWN.toString(); @@ -168,7 +189,7 @@ public class HostComponentStateEntity { } public Long getServiceId() { - return this.serviceId; + return serviceId; } public void setServiceId(Long serviceId) { @@ -196,7 +217,7 @@ public class HostComponentStateEntity { } public void setComponentId(Long componentId) { - this.id = componentId; + id = componentId; } public Long getComponentId() { @@ -243,6 +264,26 @@ public class HostComponentStateEntity { this.version = version; } + /** + * Gets the version of the mpack which was reported for this host component. + * + * @return the mpack version reporting for this component, or + * {@link State#UNKNOWN}. + */ + public String getMpackVersion() { + return mpackVersion; + } + + /** + * Sets the version of the mpack which was reported for this host component. + * + * @param mpackVersion + * the version to set, or {@link State#UNKNOWN}. + */ + public void setMpackVersion(String mpackVersion) { + this.mpackVersion = mpackVersion; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -254,78 +295,24 @@ public class HostComponentStateEntity { } HostComponentStateEntity that = (HostComponentStateEntity) o; - - if (id != null ? !id.equals(that.id) : that.id != null) { - return false; - } - - if (clusterId != null ? !clusterId.equals(that.clusterId) : that.clusterId != null) { - return false; - } - - if (serviceGroupId != null ? !serviceGroupId.equals(that.serviceGroupId) : that.serviceGroupId != null) { - return false; - } - - if (serviceId != null ? !serviceId.equals(that.serviceId) : that.serviceId != null) { - return false; - } - - if (componentName != null ? !componentName.equals(that.componentName) - : that.componentName != null) { - return false; - } - - if (componentType != null ? !componentType.equals(that.componentType) - : that.componentType != null) { - return false; - } - - if (currentState != null ? !currentState.equals(that.currentState) - : that.currentState != null) { - return false; - } - - if (lastLiveState != null ? !lastLiveState.equals(that.lastLiveState) - : that.lastLiveState != null) { - return false; - } - - if (upgradeState != null ? !upgradeState.equals(that.upgradeState) - : that.upgradeState != null) { - return false; - } - - if (hostEntity != null ? !hostEntity.equals(that.hostEntity) : that.hostEntity != null) { - return false; - } - - if (hostComponentDesiredStateEntity != null ? !hostComponentDesiredStateEntity.equals(that.hostComponentDesiredStateEntity) : that.hostComponentDesiredStateEntity != null) { - return false; - } - - if (version != null ? !version.equals(that.version) : that.version != null) { - return false; - } - - return true; + return Objects.equal(id, that.id) && Objects.equal(clusterId, that.clusterId) + && Objects.equal(serviceGroupId, that.serviceGroupId) + && Objects.equal(serviceId, that.serviceId) + && Objects.equal(componentName, that.componentName) + && Objects.equal(componentType, that.componentType) + && Objects.equal(currentState, that.currentState) + && Objects.equal(lastLiveState, that.lastLiveState) + && Objects.equal(upgradeState, that.upgradeState) + && Objects.equal(hostEntity, that.hostEntity) + && Objects.equal(hostComponentDesiredStateEntity, that.hostComponentDesiredStateEntity) + && Objects.equal(mpackVersion, that.version) && Objects.equal(version, that.version); } @Override public int hashCode() { - int result = id != null ? id.intValue() : 0; - result = 31 * result + (clusterId != null ? clusterId.intValue() : 0); - result = 31 * result + (serviceGroupId != null ? serviceGroupId.intValue() : 0); - result = 31 * result + (serviceId != null ? serviceId.intValue() : 0); - result = 31 * result + (hostEntity != null ? hostEntity.hashCode() : 0); - result = 31 * result + (hostComponentDesiredStateEntity != null ? hostComponentDesiredStateEntity.hashCode() : 0); - result = 31 * result + (componentName != null ? componentName.hashCode() : 0); - result = 31 * result + (componentType != null ? componentType.hashCode() : 0); - result = 31 * result + (currentState != null ? currentState.hashCode() : 0); - result = 31 * result + (lastLiveState != null ? lastLiveState.hashCode() : 0); - result = 31 * result + (upgradeState != null ? upgradeState.hashCode() : 0); - result = 31 * result + (version != null ? version.hashCode() : 0); - return result; + return Objects.hashCode(id, clusterId, serviceGroupId, serviceId, hostEntity, + hostComponentDesiredStateEntity, componentName, componentType, currentState, lastLiveState, + upgradeState, mpackVersion, version); } public ServiceComponentDesiredStateEntity getServiceComponentDesiredStateEntity() { @@ -357,9 +344,16 @@ public class HostComponentStateEntity { */ @Override public String toString() { - return Objects.toStringHelper(this).add("clusterId", clusterId).add("serviceGroupId", serviceGroupId).add( - "serviceId", serviceId).add("componentId", id).add("componentName", componentName).add - ("componentType", componentType).add("hostId", hostId).add("state", currentState).toString(); + return MoreObjects.toStringHelper(this) + .add("clusterId", clusterId) + .add("serviceGroupId", serviceGroupId) + .add("serviceId", serviceId) + .add("componentId", id) + .add("componentName", componentName) + .add("componentType", componentType) + .add("hostId", hostId) + .add("state", currentState) + .add("mpackVersion", mpackVersion) + .add("version", version).toString(); } - } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java index 16623aa..3bf0215 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java @@ -131,7 +131,7 @@ public class AddComponentAction extends AbstractUpgradeServerAction { // for now, this is the easiest way to fire a topology event which // refreshes the information about the cluster (needed for restart // commands) - sch.setVersion("UNKNOWN"); + sch.setVersions("UNKNOWN", "UNKNOWN"); buffer.append(" ") .append(host.getHostName()) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java index f43ae8b..aff61e4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java @@ -141,14 +141,27 @@ public interface ServiceComponentHost { String getVersion(); /** - * Sets the version of the component from the stack. + * Gets the version reported for this component's associated mpack. * - * @param version component version (e.g. 2.2.0.0-2041) + * @return mpack version reported for this component. */ - void setVersion(String version) throws AmbariException; + String getMpackVersion(); /** - * @param upgradeState the upgrade state + * Sets the versions reported for a component which include the mpack it + * belongs to and the specific version of that component within the mpack. + * + * @param mpackVersion + * the version of the component's mpack (e.g. 1.0.0.0-b1234). + * @param version + * component version (e.g. 2.2.0.0-2041) + */ + void setVersions(String mpackVersion, String version) throws AmbariException; + + + /** + * @param upgradeState + * the upgrade state */ void setUpgradeState(UpgradeState upgradeState); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 0b8d949..178c7b3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -953,7 +953,7 @@ public class UpgradeHelper { // !!! if we aren't version advertised, but there IS a version, set it to UNKNOWN if (!versionAdvertised && !StringUtils.equals("UNKNOWN", serviceComponentHost.getVersion())) { - serviceComponentHost.setVersion("UNKNOWN"); + serviceComponentHost.setVersions("UNKNOWN", "UNKNOWN"); } } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeState.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeState.java index 8f787c2..ad2a543 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeState.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeState.java @@ -17,8 +17,6 @@ */ package org.apache.ambari.server.state; -import java.util.EnumSet; - /** * Indicates the upgrade state */ @@ -43,9 +41,4 @@ public enum UpgradeState { * Component reported unexpected/wrong version */ VERSION_MISMATCH; - - /** - * States when new/correct version has not been yet advertised - */ - public static final EnumSet<UpgradeState> VERSION_NON_ADVERTISED_STATES = EnumSet.of(IN_PROGRESS, FAILED, VERSION_MISMATCH); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java index 1e17f89..3b29037 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java @@ -947,9 +947,25 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { if (stateEntity != null) { return stateEntity.getVersion(); } else { - LOG.warn("Trying to fetch a member from an entity object that may " - + "have been previously deleted, serviceName = " + getServiceName() + ", " - + "componentName = " + getServiceComponentName() + ", " + "hostName = " + getHostName()); + LOG.warn( + "Trying to fetch a member from an entity object that may " + + "have been previously deleted, serviceName = {}, componentName = {}, hostName = ", + getServiceName(), getServiceComponentName(), getHost()); + } + + return null; + } + + @Override + public String getMpackVersion() { + HostComponentStateEntity stateEntity = getStateEntity(); + if (stateEntity != null) { + return stateEntity.getMpackVersion(); + } else { + LOG.warn( + "Trying to fetch a member from an entity object that may " + + "have been previously deleted, serviceName = {}, componentName = {}, hostName = ", + getServiceName(), getServiceComponentName(), getHost()); } return null; @@ -957,10 +973,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { @Override @Transactional - public void setVersion(String version) throws AmbariException { + public void setVersions(String mpackVersion, String version) throws AmbariException { HostComponentStateEntity stateEntity = getStateEntity(); if (stateEntity != null) { stateEntity.setVersion(version); + stateEntity.setMpackVersion(mpackVersion); stateEntity = hostComponentStateDAO.merge(stateEntity); ServiceComponentHostRequest serviceComponentHostRequest = new ServiceComponentHostRequest( diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 17a1db9..cedd301 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -304,6 +304,7 @@ CREATE TABLE hostcomponentstate ( cluster_id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, component_type VARCHAR(255) NOT NULL, + mpack_version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', current_state VARCHAR(255) NOT NULL, last_live_state VARCHAR(255) NOT NULL DEFAULT 'UNKNOWN', diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 3498da1..2e3c9f1 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -323,6 +323,7 @@ CREATE TABLE hostcomponentstate ( cluster_id BIGINT NOT NULL, component_name VARCHAR(100) NOT NULL, component_type VARCHAR(100) NOT NULL, + mpack_version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', current_state VARCHAR(255) NOT NULL, last_live_state VARCHAR(255) NOT NULL DEFAULT 'UNKNOWN', diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 0c24788..5df6ae3 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -302,6 +302,7 @@ CREATE TABLE hostcomponentstate ( cluster_id NUMBER(19) NOT NULL, component_name VARCHAR2(255) NOT NULL, component_type VARCHAR2(255) NOT NULL, + mpack_version VARCHAR(32) DEFAULT 'UNKNOWN' NOT NULL, version VARCHAR2(32) DEFAULT 'UNKNOWN' NOT NULL, current_state VARCHAR2(255) NOT NULL, last_live_state VARCHAR2(255) DEFAULT 'UNKNOWN' NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index c8d19d3..78ccf07 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -305,6 +305,7 @@ CREATE TABLE hostcomponentstate ( cluster_id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, component_type VARCHAR(255) NOT NULL, + mpack_version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', current_state VARCHAR(255) NOT NULL, last_live_state VARCHAR(255) NOT NULL DEFAULT 'UNKNOWN', diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index 49a963f..469ba54 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -301,6 +301,7 @@ CREATE TABLE hostcomponentstate ( cluster_id NUMERIC(19) NOT NULL, component_name VARCHAR(255) NOT NULL, component_type VARCHAR(255) NOT NULL, + mpack_version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', current_state VARCHAR(255) NOT NULL, last_live_state VARCHAR(255) NOT NULL DEFAULT 'UNKNOWN', diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 6fb6507..511d727 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -316,6 +316,7 @@ CREATE TABLE hostcomponentstate ( cluster_id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, component_type VARCHAR(255) NOT NULL, + mpack_version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', version VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', current_state VARCHAR(255) NOT NULL, last_live_state VARCHAR(255) NOT NULL DEFAULT 'UNKNOWN', diff --git a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-30/package/scripts/kerberos_client.py b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-30/package/scripts/kerberos_client.py index 202d48a..7c2366c 100644 --- a/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-30/package/scripts/kerberos_client.py +++ b/ambari-server/src/main/resources/common-services/KERBEROS/1.10.3-30/package/scripts/kerberos_client.py @@ -52,12 +52,17 @@ class KerberosClient(Script): if principal is not None: curr_content = Script.structuredOut - if "keytabs" not in curr_content: - curr_content['keytabs'] = {} + if "set_keytabs" not in curr_content: + curr_content['set_keytabs'] = {} - curr_content['keytabs'][principal.replace("_HOST", params.hostname)] = keytab_file_path + set_keytabs_dictionary = curr_content['set_keytabs'] - self.put_structured_out(curr_content) + if "keytabs" not in set_keytabs_dictionary: + set_keytabs_dictionary['keytabs'] = {} + + set_keytabs_dictionary['keytabs'][principal.replace("_HOST", params.hostname)] = keytab_file_path + + self.put_structured_out({"set_keytabs", set_keytabs_dictionary}) write_keytab_file(params, output_hook) @@ -68,11 +73,16 @@ class KerberosClient(Script): if principal is not None: curr_content = Script.structuredOut - if "removedKeytabs" not in curr_content: - curr_content['removedKeytabs'] = {} - curr_content['removedKeytabs'][principal.replace("_HOST", params.hostname)] = keytab_file_path + if "set_keytabs" not in curr_content: + curr_content['set_keytabs'] = {} + + set_keytabs_dictionary = curr_content['set_keytabs'] + + if "removedKeytabs" not in set_keytabs_dictionary: + set_keytabs_dictionary['removedKeytabs'] = {} - self.put_structured_out(curr_content) + set_keytabs_dictionary['removedKeytabs'][principal.replace("_HOST", params.hostname)] = keytab_file_path + self.put_structured_out({"set_keytabs", set_keytabs_dictionary}) delete_keytab_file(params, output_hook) @@ -80,9 +90,9 @@ class KerberosClient(Script): import params def output_hook(missing_keytabs): - curr_content = Script.structuredOut - curr_content['missing_keytabs'] = missing_keytabs - self.put_structured_out(curr_content) + missing_keytabs_dictionary = {} + missing_keytabs_dictionary['missing_keytabs'] = missing_keytabs + self.put_structured_out({"check_keytabs", missing_keytabs_dictionary}) find_missing_keytabs(params, output_hook) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java b/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java index 0e304f0..687780a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java @@ -46,6 +46,7 @@ import org.apache.ambari.server.state.ModuleComponent; import org.apache.ambari.server.state.Mpack; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceGroup; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.testutils.PartialNiceMockBinder; import org.easymock.EasyMock; @@ -72,7 +73,9 @@ public class ComponentVersionAlertRunnableTest extends EasyMockSupport { private final static String HOSTNAME_2 = "c6402.ambari.apache.org"; private final static String EXPECTED_VERSION = "2.6.0.0-1234"; + private final static String EXPECTED_MPACK_VERSION = "1.0.0.0-b1"; private final static String WRONG_VERSION = "9.9.9.9-9999"; + private final static String WRONG_MPACK_VERSION = "9.9.9.9-b9"; private final static String DEFINITION_NAME = "ambari_server_component_version"; private final static String DEFINITION_SERVICE = "AMBARI"; @@ -138,18 +141,27 @@ public class ComponentVersionAlertRunnableTest extends EasyMockSupport { ServiceComponentHost sch2_1 = createNiceMock(ServiceComponentHost.class); ServiceComponentHost sch2_2 = createNiceMock(ServiceComponentHost.class); + StackId stackId = new StackId("MY-STACK", "1.0"); + expect(sch1_1.getDesiredStackId()).andReturn(stackId).anyTimes(); expect(sch1_1.getServiceType()).andReturn("FOO").atLeastOnce(); expect(sch1_1.getServiceComponentName()).andReturn("FOO_COMPONENT").atLeastOnce(); expect(sch1_1.getVersion()).andReturn(EXPECTED_VERSION).atLeastOnce(); + expect(sch1_1.getMpackVersion()).andReturn(EXPECTED_MPACK_VERSION).atLeastOnce(); + expect(sch1_2.getDesiredStackId()).andReturn(stackId).anyTimes(); expect(sch1_2.getServiceType()).andReturn("BAR").atLeastOnce(); expect(sch1_2.getServiceComponentName()).andReturn("BAR_COMPONENT").atLeastOnce(); expect(sch1_2.getVersion()).andReturn(EXPECTED_VERSION).atLeastOnce(); + expect(sch1_2.getMpackVersion()).andReturn(EXPECTED_MPACK_VERSION).atLeastOnce(); + expect(sch2_1.getDesiredStackId()).andReturn(stackId).anyTimes(); expect(sch2_1.getServiceType()).andReturn("FOO").atLeastOnce(); expect(sch2_1.getServiceComponentName()).andReturn("FOO_COMPONENT").atLeastOnce(); expect(sch2_1.getVersion()).andReturn(EXPECTED_VERSION).atLeastOnce(); + expect(sch2_1.getMpackVersion()).andReturn(EXPECTED_MPACK_VERSION).atLeastOnce(); + expect(sch2_2.getDesiredStackId()).andReturn(stackId).anyTimes(); expect(sch2_2.getServiceType()).andReturn("BAZ").atLeastOnce(); expect(sch2_2.getServiceComponentName()).andReturn("BAZ_COMPONENT").atLeastOnce(); expect(sch2_2.getVersion()).andReturn(EXPECTED_VERSION).atLeastOnce(); + expect(sch2_2.getMpackVersion()).andReturn(EXPECTED_MPACK_VERSION).atLeastOnce(); m_hostComponentMap.get(HOSTNAME_1).add(sch1_1); m_hostComponentMap.get(HOSTNAME_1).add(sch1_2); @@ -185,9 +197,11 @@ public class ComponentVersionAlertRunnableTest extends EasyMockSupport { // mock the mpack Mpack mpack = createNiceMock(Mpack.class); + expect(mpack.getVersion()).andReturn(EXPECTED_MPACK_VERSION).atLeastOnce(); expect(mpack.getModuleComponent(EasyMock.anyString(), EasyMock.anyString())).andReturn( moduleComponent).atLeastOnce(); + expect(m_metaInfo.getMpack(1L)).andReturn(mpack).atLeastOnce(); m_metaInfo.init(); @@ -296,6 +310,7 @@ public class ComponentVersionAlertRunnableTest extends EasyMockSupport { // reset expectation so that it returns a wrong version ServiceComponentHost sch = m_hostComponentMap.get(HOSTNAME_1).get(0); EasyMock.reset(sch); + expect(sch.getDesiredStackId()).andReturn(new StackId("MY-STACK", "1.0")).atLeastOnce(); expect(sch.getServiceType()).andReturn("FOO").atLeastOnce(); expect(sch.getServiceComponentName()).andReturn("FOO_COMPONENT").atLeastOnce(); expect(sch.getVersion()).andReturn(WRONG_VERSION).atLeastOnce(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java index 4e66811..8065870 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java @@ -173,11 +173,11 @@ public class UpgradeSummaryResourceProviderTest { ServiceComponent component = service.addServiceComponent("ZOOKEEPER_SERVER", "ZOOKEEPER_SERVER"); ServiceComponentHost sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.2.0.0"); + sch.setVersions("1.0.0.0-b1234", "2.2.0.0"); component = service.addServiceComponent("ZOOKEEPER_CLIENT", "ZOOKEEPER_CLIENT"); sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.2.0.0"); + sch.setVersions("1.0.0.0-b1234", "2.2.0.0"); } /** diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/DefaultServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/DefaultServiceCalculatedStateTest.java index 14db57a..a928ef3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/DefaultServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/DefaultServiceCalculatedStateTest.java @@ -58,11 +58,11 @@ public final class DefaultServiceCalculatedStateTest extends GeneralServiceCalcu clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); sch = clientComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.INSTALLED); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/FlumeServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/FlumeServiceCalculatedStateTest.java index 35ec2d2..d29cc44 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/FlumeServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/FlumeServiceCalculatedStateTest.java @@ -56,7 +56,7 @@ public class FlumeServiceCalculatedStateTest extends GeneralServiceCalculatedSta clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HBaseServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HBaseServiceCalculatedStateTest.java index 5b97220..cf56509 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HBaseServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HBaseServiceCalculatedStateTest.java @@ -57,15 +57,15 @@ public class HBaseServiceCalculatedStateTest extends GeneralServiceCalculatedSta clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = clientComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.INSTALLED); sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); sch = secondMasterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HDFSServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HDFSServiceCalculatedStateTest.java index 6c317cd..d366e2b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HDFSServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HDFSServiceCalculatedStateTest.java @@ -58,15 +58,15 @@ public class HDFSServiceCalculatedStateTest extends GeneralServiceCalculatedStat clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); ServiceComponentHost sch1 = masterComponent1.addServiceComponentHost(hostName); - sch1.setVersion("2.1.1.0"); + sch1.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch1.setState(State.STARTED); sch = clientComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.INSTALLED); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HiveServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HiveServiceCalculatedStateTest.java index fb598d6..96916d7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HiveServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/HiveServiceCalculatedStateTest.java @@ -60,23 +60,23 @@ public class HiveServiceCalculatedStateTest extends GeneralServiceCalculatedStat clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = clientComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.INSTALLED); sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); sch = secondMasterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); sch = thirdMasterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); sch = fourMasterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/OozieServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/OozieServiceCalculatedStateTest.java index 6e3a1c8..7745d05 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/OozieServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/OozieServiceCalculatedStateTest.java @@ -56,11 +56,11 @@ public class OozieServiceCalculatedStateTest extends GeneralServiceCalculatedSta clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = clientComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.INSTALLED); sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/YarnServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/YarnServiceCalculatedStateTest.java index cf23f31..1f8ade2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/YarnServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/YarnServiceCalculatedStateTest.java @@ -58,15 +58,15 @@ public class YarnServiceCalculatedStateTest extends GeneralServiceCalculatedStat clusters.mapHostToCluster(hostName, clusterName); ServiceComponentHost sch = secondMasterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); sch = clientComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.INSTALLED); sch = masterComponent.addServiceComponentHost(hostName); - sch.setVersion("2.1.1.0"); + sch.setVersions("1.0.0.0-b1234", "2.1.1.0"); sch.setState(State.STARTED); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/publishers/VersionEventPublisherTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/publishers/VersionEventPublisherTest.java index bdc424c..87c1829 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/publishers/VersionEventPublisherTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/publishers/VersionEventPublisherTest.java @@ -24,6 +24,7 @@ import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; +import org.apache.ambari.server.agent.HeartbeatProcessor.ComponentVersionStructuredOut; import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ServiceComponentHost; @@ -61,7 +62,12 @@ public class VersionEventPublisherTest { Listener listener = injector.getInstance(Listener.class); - HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cluster, sch, "1.2.3.4-5678"); + ComponentVersionStructuredOut componentVersionStructuredOut = new ComponentVersionStructuredOut(); + componentVersionStructuredOut.mpackVersion = "1.0.0.0-b1234"; + componentVersionStructuredOut.version = "1.2.3.4-5678"; + + HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cluster, + sch, componentVersionStructuredOut); publisher.publish(event); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/AddComponentActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/AddComponentActionTest.java index 34c05c1..7ce6116 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/AddComponentActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/AddComponentActionTest.java @@ -149,7 +149,7 @@ public class AddComponentActionTest extends EasyMockSupport { mockServiceComponentHost.setDesiredState(State.INSTALLED); expectLastCall().once(); - mockServiceComponentHost.setVersion("UNKNOWN"); + mockServiceComponentHost.setVersions("UNKNOWN", "UNKNOWN"); expectLastCall().once(); PowerMock.replay(m_action); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java index f84c79f..a6be79b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java @@ -335,7 +335,7 @@ public class ComponentVersionCheckActionTest { // set the SCH versions to the new stack so that the finalize action is // happy - don't update DATANODE - we want to make the action complain - cluster.getServiceComponentHosts("HDFS", "NAMENODE").get(0).setVersion(targetVersion); + cluster.getServiceComponentHosts("HDFS", "NAMENODE").get(0).setVersions("1.0.0.0-b1234", targetVersion); // now finalize and ensure we can transition from UPGRADING to UPGRADED // automatically before CURRENT @@ -358,7 +358,7 @@ public class ComponentVersionCheckActionTest { assertEquals(-1, report.getExitCode()); // OK, now set the datanode so it completes - cluster.getServiceComponentHosts("HDFS", "DATANODE").get(0).setVersion(targetVersion); + cluster.getServiceComponentHosts("HDFS", "DATANODE").get(0).setVersions("1.0.0.0-b1234", targetVersion); report = action.execute(null); assertNotNull(report); @@ -394,15 +394,15 @@ public class ComponentVersionCheckActionTest { addServiceComponent(service, "DATANODE"); ServiceComponentHost sch = createNewServiceComponentHost(cluster, "HDFS", "NAMENODE", "h1"); - sch.setVersion(HDP_2_1_1_0); + sch.setVersions("1.0.0.0-b1234", HDP_2_1_1_0); sch = createNewServiceComponentHost(cluster, "HDFS", "DATANODE", "h1"); - sch.setVersion(HDP_2_1_1_0); + sch.setVersions("1.0.0.0-b1234", HDP_2_1_1_0); service = installService(cluster, serviceGroup, "ZOOKEEPER", targetMpack); addServiceComponent(service, "ZOOKEEPER_SERVER"); sch = createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_SERVER", "h1"); - sch.setVersion(HDP_2_1_1_1); + sch.setVersions("1.0.0.0-b1234", HDP_2_1_1_1); // Finalize the upgrade Map<String, String> commandParams = new HashMap<>(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java index fe046a4..fe5d61a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java @@ -204,7 +204,7 @@ public class ServiceComponentHostConcurrentWriteDeadlockTest { for (ServiceComponentHost serviceComponentHost : serviceComponentHosts) { serviceComponentHost.setState(state); - serviceComponentHost.setVersion(version); + serviceComponentHost.setVersions("1.0.0.0-b1234", version); } Thread.sleep(10); @@ -233,7 +233,7 @@ public class ServiceComponentHostConcurrentWriteDeadlockTest { sc.addServiceComponentHost(sch); sch.setDesiredState(State.INSTALLED); sch.setState(State.INSTALLED); - sch.setVersion(REPO_VERSION); + sch.setVersions("1.0.0.0-b1234", REPO_VERSION); return sch; }