Repository: ambari Updated Branches: refs/heads/trunk 8429a6a1d -> 7f725853f
AMBARI-13824. Stop-and-Start Upgrade: Fix for Oozie Express upgrade issues from HDP 2.1 to 2.3 (Swapan Shridhar via alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7f725853 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7f725853 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7f725853 Branch: refs/heads/trunk Commit: 7f725853f6e4282ef0710371c57d18d784998527 Parents: 8429a6a Author: Alejandro Fernandez <[email protected]> Authored: Thu Nov 12 11:33:09 2015 -0800 Committer: Alejandro Fernandez <[email protected]> Committed: Thu Nov 12 11:34:51 2015 -0800 ---------------------------------------------------------------------- .../libraries/functions/conf_select.py | 31 +++++++++---- .../AmbariCustomCommandExecutionHelper.java | 37 +++++++++++++-- .../internal/UpgradeResourceProvider.java | 47 ++++++++++---------- .../state/stack/upgrade/ConfigureFunction.java | 36 +++++++++++++++ .../server/state/stack/upgrade/Grouping.java | 3 ++ .../state/stack/upgrade/StageWrapper.java | 7 +-- .../ambari/server/state/stack/upgrade/Task.java | 8 +++- .../4.0.0.2.0/package/scripts/oozie_server.py | 20 +++++++-- .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 7 ++- 9 files changed, 153 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index 620f021..9bf259b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@ -224,14 +224,12 @@ def create(stack_name, package, version, dry_run = False): def select(stack_name, package, version, try_create=True): """ - Selects a config version for the specified package. Currently only works if the version is - for HDP-2.3 or higher + Selects a config version for the specified package. :stack_name: the name of the stack :package: the name of the package, as-used by conf-select :version: the version number to create :try_create: optional argument to attempt to create the directory before setting it """ - if not _valid(stack_name, package, version): return @@ -240,6 +238,28 @@ def select(stack_name, package, version, try_create=True): shell.checked_call(get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True) + # Create the symbolic link using 'PACKAGE_DIRS' for the given package + # Starting with 2.3, we have sym links instead of flat directories. + # Eg: /etc/<service-name>/conf -> /etc/<service-name>/2.3.x.y-<version>/0 + # But, in case of Express upgrade from HDP 2.1-> HDP 2.3 where we have + # deleted the /etc/<service-name>/conf directory, the above mentioned + # symlink needs to be created here. + if package in PACKAGE_DIRS: + conf_dirs = PACKAGE_DIRS[package] + Logger.info("For package : {0}, DIRS = {1}".format(package, conf_dirs)) + for dirInfo in conf_dirs: + if "conf_dir" in dirInfo and "current_dir" in dirInfo: + conf_dir = dirInfo["conf_dir"] + current_dir = dirInfo["current_dir"] + Logger.info("For package : {0}, Source dir: {1}, Dest dir: {2}".format(package, conf_dir, current_dir)) + if os.path.exists(current_dir): + real_path_of_current_dir = os.path.realpath(current_dir) + normalized_conf_dir = (os.path.normpath(conf_dir)).strip() + normalized_current_dir = (os.path.normpath(real_path_of_current_dir)).strip() + Logger.info("Normalized Conf Dir : {0}, Normalized Current Dir : {1}".format(normalized_conf_dir, normalized_current_dir)) + if not os.path.isdir(normalized_conf_dir) and os.path.isdir(normalized_current_dir) and normalized_current_dir != normalized_conf_dir: + Logger.info("Creating Symlink : {0} -> {1}".format(normalized_conf_dir, normalized_current_dir)) + os.symlink(normalized_current_dir, normalized_conf_dir) def get_hadoop_conf_dir(force_latest_on_upgrade=False): """ @@ -293,23 +313,18 @@ def create_config_links(stack_id, stack_version): stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ - if stack_id is None: Logger.info("Cannot create config links when stack_id is not defined") return - args = stack_id.upper().split('-') if len(args) != 2: Logger.info("Unrecognized stack id {0}".format(stack_id)) return - if args[0] != "HDP": Logger.info("Unrecognized stack name {0}".format(args[0])) - if version.compare_versions(version.format_hdp_stack_version(args[1]), "2.3.0.0") < 0: Logger.info("Cannot link configs unless HDP-2.3 or higher") return - for k, v in PACKAGE_DIRS.iteritems(): dirs = create(args[0], k, stack_version, dry_run = True) if 0 == len(dirs): http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 9ba5a22..1f8c687 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -180,10 +180,41 @@ public class AmbariCustomCommandExecutionHelper { stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName); - return !(!componentInfo.isCustomCommand(commandName) && - !actionMetadata.isDefaultHostComponentCommand(commandName)); + return actionMetadata.isDefaultHostComponentCommand(commandName); } + private Boolean isValidDefaultCommand(String clusterName, + String serviceName, String componentName, String commandName) + throws AmbariException { + + Cluster cluster = clusters.getCluster(clusterName); + StackId stackId = cluster.getDesiredStackVersion(); + + if (componentName == null) { + return false; + } + ComponentInfo componentInfo = ambariMetaInfo.getComponent( + stackId.getStackName(), stackId.getStackVersion(), + serviceName, componentName); + + return (actionMetadata.isDefaultHostComponentCommand(commandName)); + } + + private Boolean isValidDefaultCommand(ActionExecutionContext + actionExecutionContext, RequestResourceFilter resourceFilter) + throws AmbariException { + String clusterName = actionExecutionContext.getClusterName(); + String serviceName = resourceFilter.getServiceName(); + String componentName = resourceFilter.getComponentName(); + String commandName = actionExecutionContext.getActionName(); + + if (componentName == null) { + return false; + } + + return isValidCustomCommand(clusterName, serviceName, componentName, commandName); + } + private Boolean isValidCustomCommand(ActionExecutionContext actionExecutionContext, RequestResourceFilter resourceFilter) throws AmbariException { @@ -908,7 +939,7 @@ public class AmbariCustomCommandExecutionHelper { findHostAndAddServiceCheckAction(actionExecutionContext, resourceFilter, stage); } else if (actionName.equals(DECOMMISSION_COMMAND_NAME)) { addDecommissionAction(actionExecutionContext, resourceFilter, stage); - } else if (isValidCustomCommand(actionExecutionContext, resourceFilter)) { + } else if (isValidDefaultCommand(actionExecutionContext, resourceFilter)) { String commandDetail = getReadableCustomCommandDetail(actionExecutionContext, resourceFilter); http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 8dff73a..748dbbe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -17,25 +17,10 @@ */ package org.apache.ambari.server.controller.internal; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; - -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.inject.Inject; +import com.google.inject.Provider; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -108,10 +93,24 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.google.inject.Inject; -import com.google.inject.Provider; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; /** * Manages the ability to start and get status of upgrades. @@ -1095,6 +1094,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider throws AmbariException { switch (wrapper.getType()) { + case CONFIGURE: case START: case STOP: case RESTART: @@ -1242,6 +1242,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider String function = null; switch (wrapper.getType()) { + case CONFIGURE: case START: case STOP: case RESTART: http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java new file mode 100644 index 0000000..fc6a45b --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java @@ -0,0 +1,36 @@ +package org.apache.ambari.server.state.stack.upgrade; + + import javax.xml.bind.annotation.XmlAccessType; + import javax.xml.bind.annotation.XmlAccessorType; + import javax.xml.bind.annotation.XmlRootElement; + import javax.xml.bind.annotation.XmlTransient; + import javax.xml.bind.annotation.XmlType; + +/** + * Used to represent Configuring of a component. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name="configure_function") +public class ConfigureFunction extends Task { + + @XmlTransient + private Task.Type type = Type.CONFIGURE_FUNCTION; + + public static final String actionVerb = "Configuring"; + + @Override + public Task.Type getType() { + return type; + } + + @Override + public StageWrapper.Type getStageWrapperType() { + return StageWrapper.Type.CONFIGURE; + } + + @Override + public String getActionVerb() { + return actionVerb; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java index 6122e2d..04e9aaa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java @@ -244,6 +244,9 @@ public class Grouping { case EXECUTE: type = StageWrapper.Type.RU_TASKS; break; + case CONFIGURE_FUNCTION: + type = StageWrapper.Type.CONFIGURE; + break; case RESTART: type = StageWrapper.Type.RESTART; break; http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java index 2ea3671..86a8f55 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java @@ -17,6 +17,8 @@ */ package org.apache.ambari.server.state.stack.upgrade; +import com.google.gson.Gson; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -25,8 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.google.gson.Gson; - /** * */ @@ -149,6 +149,7 @@ public class StageWrapper { RU_TASKS, SERVICE_CHECK, STOP, - START + START, + CONFIGURE } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java index f443e53..60091a6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java @@ -24,7 +24,7 @@ import javax.xml.bind.annotation.XmlSeeAlso; /** * Base class to identify the items that could possibly occur during an upgrade */ -@XmlSeeAlso(value={ExecuteTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class, StartTask.class, StopTask.class, ServerActionTask.class}) +@XmlSeeAlso(value={ExecuteTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class, StartTask.class, StopTask.class, ServerActionTask.class, ConfigureFunction.class}) public abstract class Task { /** @@ -66,6 +66,10 @@ public abstract class Task { */ CONFIGURE, /** + * Task that sets up the configuration for subsequent task + */ + CONFIGURE_FUNCTION, + /** * Task that displays a message and must be confirmed before continuing */ MANUAL, @@ -101,7 +105,7 @@ public abstract class Task { * @return {@code true} if the task is a command type (as opposed to an action) */ public boolean isCommand() { - return this == RESTART || this == START || this == STOP || this == SERVICE_CHECK; + return this == RESTART || this == START || this == CONFIGURE_FUNCTION || this == STOP || this == SERVICE_CHECK; } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index f35df93..35975df 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -24,6 +24,7 @@ from resource_management.libraries.functions import compare_versions from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import format_hdp_stack_version +from resource_management.libraries.functions.constants import Direction from resource_management.libraries.functions.security_commons import build_expectations from resource_management.libraries.functions.security_commons import cached_kinit_executor from resource_management.libraries.functions.security_commons import get_params_from_filesystem @@ -47,8 +48,20 @@ class OozieServer(Script): def install(self, env): self.install_packages(env) - def configure(self, env): + def configure(self, env, upgrade_type=None): import params + + if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \ + params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + conf_select.select(params.stack_name, "oozie", params.version) + # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of + # oozie, we need to create the symlinks both for server and client. + # This is required as both need to be pointing to new installed oozie version. + + # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie + hdp_select.select("oozie-client", params.version) + # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie + hdp_select.select("oozie-server", params.version) env.set_params(params) oozie(is_server=True) @@ -166,8 +179,9 @@ class OozieServerDefault(OozieServer): OozieUpgrade.backup_configuration() - conf_select.select(params.stack_name, "oozie", params.version) - hdp_select.select("oozie-server", params.version) + if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: + conf_select.select(params.stack_name, "oozie", params.version) + hdp_select.select("oozie-server", params.version) OozieUpgrade.restore_configuration() OozieUpgrade.prepare_libext_directory() http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml index f12d0cd..49c8385 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -697,6 +697,11 @@ <service name="OOZIE"> <component name="OOZIE_SERVER"> <pre-upgrade> + <!-- We need to set up the "/etc/oozie/conf" symlink before upgrading, as the HDP 2.1 directory + pertaining to oozie has been deleted as part of HDP 2.1 removal in Express Upgrade + from HDP 2.1->2.3. + --> + <task xsi:type="configure_function"/> <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib"> <script>scripts/oozie_server_upgrade.py</script> <function>upgrade_oozie_database_and_sharelib</function> @@ -775,4 +780,4 @@ </component> </service> </processing> -</upgrade> \ No newline at end of file +</upgrade>
