HOTFIX-729. Performance hacks for PayPal to do EU on large clusters for HDP 2.4 -> 2.5 and 2.2 -> 2.5 (alejandro)
Change-Id: I648baf21f0d272eaa201aa2f381571035f2f4551 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1427726a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1427726a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1427726a Branch: refs/heads/AMBARI-2.4.2.16 Commit: 1427726a4c3a97d9089de141296e98ce01ee6d87 Parents: df428f6 Author: Alejandro Fernandez <[email protected]> Authored: Thu Mar 16 16:56:30 2017 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Thu Mar 16 16:56:30 2017 -0700 ---------------------------------------------------------------------- .../libraries/functions/stack_select.py | 14 + .../server/checks/AtlasPresenceCheck.java | 4 +- .../internal/UpgradeResourceProvider.java | 3 +- .../ambari/server/state/UpgradeHelper.java | 5 + .../state/stack/upgrade/ClusterGrouping.java | 192 ++- .../server/state/stack/upgrade/Grouping.java | 2 - .../state/stack/upgrade/StageWrapper.java | 23 +- .../stack/upgrade/StageWrapperBuilder.java | 22 +- ambari-server/src/main/resources/alerts.json | 12 + .../package/scripts/datanode_upgrade.py | 4 +- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 7 +- .../package/scripts/nodemanager_upgrade.py | 4 + .../host_scripts/alert_version_select.py | 108 ++ .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 320 ++++ .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 65 +- .../HDP/2.2/upgrades/nonrolling-upgrade-2.5.xml | 1613 ++++++++++++++++++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 216 ++- .../ambari/server/state/UpgradeHelperTest.java | 37 + .../HDP/2.1.1/upgrades/upgrade_test_batch.xml | 73 + 19 files changed, 2540 insertions(+), 184 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index 513ceac..bce69c1 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -283,6 +283,20 @@ def _get_upgrade_stack(): return None +def unsafe_get_stack_versions(): + """ + Gets list of stack versions installed on the host. + Be default a call to <stack-selector-tool> versions is made to get the list of installed stack versions. + DO NOT use a fall-back since this function is called by alerts in order to find potential errors. + :return: Returns a tuple of (exit code, output, list of installed stack versions). + """ + stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) + code, out = call((STACK_SELECT_PREFIX, stack_selector_path, 'versions')) + versions = [] + if 0 == code: + for line in out.splitlines(): + versions.append(line.rstrip('\n')) + return (code, out, versions) def get_stack_versions(stack_root): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/checks/AtlasPresenceCheck.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/AtlasPresenceCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/AtlasPresenceCheck.java index 8556436..04b73fa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/AtlasPresenceCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/AtlasPresenceCheck.java @@ -29,8 +29,8 @@ import org.slf4j.LoggerFactory; import com.google.inject.Singleton; /** - * Checks if Atlas service is present. Upgrade to stack HDP 2.5 can't pursuit - * with existed on the cluster Atlas service. + * Checks if Atlas service is present. Upgrade to stack HDP 2.5 from previous stack + * must first delete Atlas from the cluster. */ @Singleton @UpgradeCheck(group = UpgradeCheckGroup.DEFAULT) http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index d37e32b..6c6c9e8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1421,7 +1421,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // need to set meaningful text on the command for (Map<String, HostRoleCommand> map : stage.getHostRoleCommands().values()) { for (HostRoleCommand hrc : map.values()) { - hrc.setCommandDetail(entity.getText()); + String commandDetail = wrapper.getOriginalText() + ((hrc.getHostName() != null && !hrc.getHostName().isEmpty()) ? " on " + hrc.getHostName() : ""); + hrc.setCommandDetail(commandDetail); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 3a2dc89..ea275da 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -500,6 +500,11 @@ public class UpgradeHelper { null, null)); } + if (null != stageWrapper.getOriginalText()) { + stageWrapper.setOriginalText(tokenReplace(ctx, stageWrapper.getOriginalText(), + null, null)); + } + for (TaskWrapper taskWrapper : stageWrapper.getTasks()) { for (Task task : taskWrapper.getTasks()) { if (null != task.summary) { http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java index 80bb26c..f745266 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java @@ -42,6 +42,7 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; +import org.apache.ambari.server.utils.SetUtils; import org.apache.commons.lang.StringUtils; import com.google.gson.JsonArray; @@ -104,6 +105,12 @@ public class ClusterGrouping extends Grouping { @XmlElement(name="scope") public UpgradeScope scope = UpgradeScope.ANY; + /** + * Parallelization of this stage. By default, an Execute Stage in a Cluster Group runs on all hosts in parallel, + * but the Execute Stage is allowed to override how many hosts per batch. + */ + @XmlElement(name="parallel-scheduler") + public ParallelScheduler parallelScheduler; } public class ClusterBuilder extends StageWrapperBuilder { @@ -156,7 +163,8 @@ public class ClusterGrouping extends Grouping { break; case EXECUTE: - wrapper = getExecuteStageWrapper(upgradeContext, execution); + List<StageWrapper> wrappers = getExecuteStageWrapper(upgradeContext, execution); + results.addAll(wrappers); break; default: @@ -171,6 +179,113 @@ public class ClusterGrouping extends Grouping { return results; } + + /** + * Return a Stage Wrapper for a task meant to execute code, typically on Ambari Server. + * @param ctx Upgrade Context + * @param execution Execution Stage + * @return Returns a list of Stage Wrappers, or null if a valid one could not be created. + */ + private List<StageWrapper> getExecuteStageWrapper(UpgradeContext ctx, ExecuteStage execution) { + List<StageWrapper> wrappers = new ArrayList<>(); + String service = execution.service; + String component = execution.component; + ExecuteTask et = (ExecuteTask) execution.task; + + if (StringUtils.isNotBlank(service) && StringUtils.isNotBlank(component)) { + + // !!! if the context is not scoped for the execute-stage, bail + if (!ctx.isScoped(execution.scope)) { + return null; + } + + // !!! if the context is targeted and does not include the service, bail + if (!ctx.isServiceSupported(service)) { + return null; + } + + HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component); + + if (hosts != null) { + + Set<String> realHosts = new LinkedHashSet<String>(hosts.hosts); + if (ExecuteHostType.MASTER == et.hosts && null != hosts.master) { + realHosts = Collections.singleton(hosts.master); + } + + // Pick a random host. + if (ExecuteHostType.ANY == et.hosts && !hosts.hosts.isEmpty()) { + realHosts = Collections.singleton(hosts.hosts.iterator().next()); + } + + // Pick the first host sorted alphabetically (case insensitive) + if (ExecuteHostType.FIRST == et.hosts && !hosts.hosts.isEmpty()) { + List<String> sortedHosts = new ArrayList<>(hosts.hosts); + Collections.sort(sortedHosts, String.CASE_INSENSITIVE_ORDER); + realHosts = Collections.singleton(sortedHosts.get(0)); + } + + // !!! cannot execute against empty hosts (safety net) + if (realHosts.isEmpty()) { + return null; + } + + StageWrapper wrapper = new StageWrapper( + StageWrapper.Type.RU_TASKS, + execution.title, + new TaskWrapper(service, component, realHosts, et)); + wrappers.add(wrapper); + return wrappers; + } + } else if (null == service && null == component) { + + // By default, 0 implies running all hosts in the cluster in a single stage for this task. + // If greater than 0, batch server side tasks meant to run on the cluster if possible. + int numHostsPerBatch = 0; + if (execution.parallelScheduler != null && execution.parallelScheduler.maxDegreeOfParallelism > 0) { + numHostsPerBatch = execution.parallelScheduler.maxDegreeOfParallelism; + } + + // no service and no component will distributed the task to all healthy + // hosts not in maintenance mode + Cluster cluster = ctx.getCluster(); + Set<String> hostNames = new HashSet<String>(); + for (Host host : ctx.getCluster().getHosts()) { + MaintenanceState maintenanceState = host.getMaintenanceState(cluster.getClusterId()); + if (maintenanceState == MaintenanceState.OFF) { + hostNames.add(host.getHostName()); + } + } + + // Batch the hosts if applicable. + // This is important for operations like "hdp-select set all" in order to batch. + List<Set<String>> batchedHostSets = null; + if (numHostsPerBatch > 0) { + batchedHostSets = SetUtils.split(hostNames, numHostsPerBatch); + } else { + batchedHostSets = SetUtils.split(hostNames, 1); + } + + int numBatchesNeeded = batchedHostSets.size(); + int batchNum = 0; + for (Set<String> hostSubset : batchedHostSets) { + batchNum++; + + // Preserve the original execution title + StageWrapper sw = new StageWrapper( + StageWrapper.Type.RU_TASKS, + execution.title, + new TaskWrapper(service, component, hostSubset, et)); + + String stageText = this.getStageText(execution.title, null, hostSubset, batchNum, numBatchesNeeded); + sw.setText(stageText); + wrappers.add(sw); + } + + return wrappers; + } + return null; + } } /** @@ -213,81 +328,6 @@ public class ClusterGrouping extends Grouping { } /** - * Return a Stage Wrapper for a task meant to execute code, typically on Ambari Server. - * @param ctx Upgrade Context - * @param execution Execution Stage - * @return Returns a Stage Wrapper, or null if a valid one could not be created. - */ - private StageWrapper getExecuteStageWrapper(UpgradeContext ctx, ExecuteStage execution) { - String service = execution.service; - String component = execution.component; - ExecuteTask et = (ExecuteTask) execution.task; - - if (StringUtils.isNotBlank(service) && StringUtils.isNotBlank(component)) { - - // !!! if the context is not scoped for the execute-stage, bail - if (!ctx.isScoped(execution.scope)) { - return null; - } - - // !!! if the context is targeted and does not include the service, bail - if (!ctx.isServiceSupported(service)) { - return null; - } - - - HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component); - - if (hosts != null) { - - Set<String> realHosts = new LinkedHashSet<String>(hosts.hosts); - if (ExecuteHostType.MASTER == et.hosts && null != hosts.master) { - realHosts = Collections.singleton(hosts.master); - } - - // Pick a random host. - if (ExecuteHostType.ANY == et.hosts && !hosts.hosts.isEmpty()) { - realHosts = Collections.singleton(hosts.hosts.iterator().next()); - } - - // Pick the first host sorted alphabetically (case insensitive) - if (ExecuteHostType.FIRST == et.hosts && !hosts.hosts.isEmpty()) { - List<String> sortedHosts = new ArrayList<>(hosts.hosts); - Collections.sort(sortedHosts, String.CASE_INSENSITIVE_ORDER); - realHosts = Collections.singleton(sortedHosts.get(0)); - } - - // !!! cannot execute against empty hosts (safety net) - if (realHosts.isEmpty()) { - return null; - } - - return new StageWrapper( - StageWrapper.Type.RU_TASKS, - execution.title, - new TaskWrapper(service, component, realHosts, et)); - } - } else if (null == service && null == component) { - // no service and no component will distributed the task to all healthy - // hosts not in maintenance mode - Cluster cluster = ctx.getCluster(); - Set<String> hostNames = new HashSet<String>(); - for (Host host : ctx.getCluster().getHosts()) { - MaintenanceState maintenanceState = host.getMaintenanceState(cluster.getClusterId()); - if (maintenanceState == MaintenanceState.OFF) { - hostNames.add(host.getHostName()); - } - } - - return new StageWrapper( - StageWrapper.Type.RU_TASKS, - execution.title, - new TaskWrapper(service, component, hostNames, et)); - } - return null; - } - - /** * Populates the manual task, mt, with information about the list of hosts. * @param mt Manual Task * @param hostToComponents Map from host name to list of components http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java index be1f469..27b3a1b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java @@ -208,8 +208,6 @@ public class Grouping { int batchNum = 0; for (Set<String> hostSubset : hostSets) { batchNum++; - TaskWrapper expandedTW = new TaskWrapper(tw.getService(), tw.getComponent(), hostSubset, tw.getParams(), tw.getTasks()); - String stageText = getStageText(verb, ctx.getComponentDisplay(service, pc.name), hostSubset, batchNum, numBatchesNeeded); StageWrapper stage = new StageWrapper( http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java index 86a8f55..8f46b38 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java @@ -34,6 +34,12 @@ public class StageWrapper { private static Gson gson = new Gson(); private String text; + + /** + * Template used in order to format command_detail of any HostRoleCommands that will belong in this tage. + */ + private String originalText; + private Type type; private Map<String, String> params; private List<TaskWrapper> tasks; @@ -69,6 +75,7 @@ public class StageWrapper { public StageWrapper(Type type, String text, Map<String, String> params, List<TaskWrapper> tasks) { this.type = type; this.text = text; + this.originalText = text; this.params = (params == null ? Collections.<String, String>emptyMap() : params); this.tasks = tasks; } @@ -126,13 +133,27 @@ public class StageWrapper { } /** - * @param newText the new text for the stage + * @return the original text for this stage + */ + public String getOriginalText() { + return originalText; + } + + /** + * @param newText the new text for the stage after doing variable substitution */ public void setText(String newText) { text = newText; } /** + * @param newText the new original text for the stage after doing variable substitution + */ + public void setOriginalText(String newText) { + originalText = newText; + } + + /** * Gets the type of stage. All tasks defined for the stage execute this type. * @return the type */ http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java index ec7279c..2cfa656 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java @@ -160,11 +160,11 @@ public abstract class StageWrapperBuilder { } /** - * Consistently formats a string. - * @param prefix - * @param component - * @param batchNum - * @param totalBatches + * Consistently formats a string and includes a component name. + * @param prefix Text to prepend + * @param component May be null if operating on the cluster. + * @param batchNum Current batch number + * @param totalBatches Total number of batches * @return the prepared string */ protected String getStageText(String prefix, String component, Set<String> hosts, int batchNum, int totalBatches) { @@ -174,16 +174,16 @@ public abstract class StageWrapperBuilder { } /** - * Consistently formats a string. - * @param prefix - * @param component - * @param hosts + * Consistently formats a string and includes a component name. + * @param prefix Text to prepend + * @param component Component may be null + * @param hosts Collection of hosts * @return the prepared string */ protected String getStageText(String prefix, String component, Set<String> hosts) { - return String.format("%s %s on %s%s", + return String.format("%s%s on %s%s", prefix, - component, + component == null ? "" : " " + component, 1 == hosts.size() ? hosts.iterator().next() : Integer.valueOf(hosts.size()), 1 == hosts.size() ? "" : " hosts"); } http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/alerts.json b/ambari-server/src/main/resources/alerts.json index 9cffff5..2c979df 100644 --- a/ambari-server/src/main/resources/alerts.json +++ b/ambari-server/src/main/resources/alerts.json @@ -149,6 +149,18 @@ } ] } + }, + { + "name": "version_select", + "label": "Distro/Conf Select Versions", + "description": "This host-level alert is triggered if the distro selector such as hdp-select cannot calculate versions available on this host. This may indicate that /usr/$stack/ directory has links/dirs that do not belong inside of it.", + "interval": 5, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "alert_version_select.py" + } } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode_upgrade.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode_upgrade.py index b55237d..d8c1948 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode_upgrade.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode_upgrade.py @@ -73,7 +73,9 @@ def post_upgrade_check(hdfs_binary): Execute(params.dn_kinit_cmd, user=params.hdfs_user) # verify that the datanode has started and rejoined the HDFS cluster - _check_datanode_startup(hdfs_binary) + # Large Cluster Hack: speed up by assuming all DNs will join. + Logger.info("\n" + ("*" * 40) + "\nSkipping DataNode liveliness check.\n" + "*" * 40) + #_check_datanode_startup(hdfs_binary) def is_datanode_process_running(): http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 17d8107..3aac04f 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -168,8 +168,11 @@ class NameNodeDefault(NameNode): hdfs_binary = self.get_hdfs_binary() namenode_upgrade.prepare_upgrade_check_for_previous_dir() namenode_upgrade.prepare_upgrade_enter_safe_mode(hdfs_binary) - namenode_upgrade.prepare_upgrade_save_namespace(hdfs_binary) - namenode_upgrade.prepare_upgrade_backup_namenode_dir() + # Large Cluster Hack: assume user will run these commands before the upgrade. + Logger.info("\n" + ("*" * 40) + "\nSkipping commands to save NameNode namespace and backup NameNode dir(s).\n" + "*" * 40) + #namenode_upgrade.prepare_upgrade_save_namespace(hdfs_binary) + #namenode_upgrade.prepare_upgrade_backup_namenode_dir() + namenode_upgrade.prepare_upgrade_finalize_previous_upgrades(hdfs_binary) # Call -rollingUpgrade prepare http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py index 1c886f9..a62016a 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py @@ -39,11 +39,15 @@ def post_upgrade_check(): if params.security_enabled and params.nodemanager_kinit_cmd: Execute(params.nodemanager_kinit_cmd, user=params.yarn_user) + # Large Cluster Hack: assume NMs will reach a RUNNING state. + Logger.info("\n" + ("*" * 40) + "\nSkipping check that NodeManager is now in RUNNING state.\n" + "*" * 40) + ''' try: _check_nodemanager_startup() except Fail: show_logs(params.yarn_log_dir, params.yarn_user) raise + ''' @retry(times=30, sleep_time=10, err_class=Fail) http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/host_scripts/alert_version_select.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/host_scripts/alert_version_select.py b/ambari-server/src/main/resources/host_scripts/alert_version_select.py new file mode 100644 index 0000000..9f54b1e --- /dev/null +++ b/ambari-server/src/main/resources/host_scripts/alert_version_select.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import logging +import socket +import json + +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions.stack_select import unsafe_get_stack_versions + +RESULT_STATE_OK = 'OK' +RESULT_STATE_WARNING = 'WARNING' +RESULT_STATE_CRITICAL = 'CRITICAL' +RESULT_STATE_UNKNOWN = 'UNKNOWN' + +STACK_TOOLS = '{{cluster-env/stack_tools}}' + + +logger = logging.getLogger() + + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (STACK_TOOLS,) + + +def execute(configurations={}, parameters={}, host_name=None): + """ + Checks if the stack selector such as hdp-select can find versions installed on this host. E.g., + hdp-select versions + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + msg = [] + try: + if configurations is None: + return (RESULT_STATE_UNKNOWN, ['There were no configurations supplied to the script.']) + + # Check required properties + if STACK_TOOLS not in configurations: + return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script'.format(STACK_TOOLS)]) + + # Of the form, + # { "stack_selector": ["hdp-select", "/usr/bin/hdp-select", "hdp-select"], "conf_selector": ["conf-select", "/usr/bin/conf-select", "conf-select"] } + stack_tools_str = configurations[STACK_TOOLS] + + if stack_tools_str is None: + return (RESULT_STATE_UNKNOWN, ['%s is a required parameter for the script and the value is null' % (STACK_TOOLS)]) + + distro_select = "unknown-distro-select" + try: + stack_tools = json.loads(stack_tools_str) + distro_select = stack_tools["stack_selector"][0] + except: + pass + + if host_name is None: + host_name = socket.getfqdn() + + # This may not exist if the host does not contain any stack components, + # or only contains components like Ambari Metrics and SmartSense + stack_root_dir = Script.get_stack_root() + + if os.path.isdir(stack_root_dir): + (code, out, versions) = unsafe_get_stack_versions() + + if code == 0: + msg.append("Ok. %s on host %s" % (distro_select, host_name)) + if versions is not None and type(versions) is list and len(versions) > 0: + msg.append("Versions: %s" % ", ".join(versions)) + return (RESULT_STATE_OK, ["\n".join(msg)]) + else: + msg.append("Failed, check dir %s for unexpected contents." % stack_root_dir) + if out is not None: + msg.append(out) + + return (RESULT_STATE_CRITICAL, ["\n".join(msg)]) + else: + msg.append("Ok. No stack root %s to check on host %s" % (stack_root_dir, host_name)) + return (RESULT_STATE_OK, ["\n".join(msg)]) + except Exception, e: + return (RESULT_STATE_CRITICAL, [e.message]) + http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml index 0f5f48e..6240684 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -106,6 +106,35 @@ <set key="xasecure.audit.provider.summary.enabled" value="false"/> </definition> + <definition xsi:type="configure" id="hdp_2_2_to_2_5_hdfs_remove_ranger_audit_db" + summary="Remove Ranger HDFS Audit DB"> + <type>ranger-hdfs-audit</type> + + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" + from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" to-key="xasecure.audit.destination.hdfs.dir" + default-value="hdfs://NAMENODE_HOSTNAME:8020/ranger/audit"/> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" + to-key="xasecure.audit.destination.hdfs" default-value="true"/> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" + from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" + default-value="/var/log/hadoop/hdfs/audit/hdfs/spool"/> + <set key="xasecure.audit.destination.solr" value="false"/> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> + <set key="xasecure.audit.destination.solr.zookeepers" value="none"/> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" + value="/var/log/hadoop/hdfs/audit/solr/spool"/> + <set key="xasecure.audit.provider.summary.enabled" value="false"/> + + <transfer operation="delete" delete-key="xasecure.audit.destination.db"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver"/> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir"/> + </definition> + <definition xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_security" summary="Transitioning Ranger HDFS Security"> <type>ranger-hdfs-security</type> @@ -143,6 +172,28 @@ <transfer operation="delete" delete-key="XAAUDIT.DB.DATABASE_NAME"/> <transfer operation="delete" delete-key="XAAUDIT.DB.HOSTNAME"/> </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_namenode_ha_adjustments"> + <type>hdfs-site</type> + <transfer operation="delete" delete-key="dfs.namenode.rpc-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/> + <transfer operation="delete" delete-key="dfs.namenode.http-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/> + <transfer operation="delete" delete-key="dfs.namenode.https-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/> + </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"> + <type>yarn-site</type> + <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/> + <!-- Ideally we need to append spark2_shuffle to the existing value --> + </definition> + </changes> + </component> + </service> + + <service name="SPARK"> + <component name="SPARK_JOBHISTORYSERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver"> + <type>spark-defaults</type> + <transfer operation="delete" delete-key="spark.yarn.max.executor.failures" /> + </definition> </changes> </component> </service> @@ -224,6 +275,16 @@ <transfer operation="delete" delete-key="yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity"/> </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"> + <type>ranger-yarn-audit</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> </changes> </component> </service> @@ -236,6 +297,10 @@ <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService,\
 org.apache.falcon.workflow.WorkflowJobEndNotificationService,\
 org.apache.falcon.service.ProcessSubscriberService,\
 org.apache.falcon.entity.store.ConfigurationStore,\
 org.apache.falcon.rerun.service.RetryService,\
 org.apache.falcon.rerun.service.LateRunService,\
 org.apache.falcon.service.LogCleanupService,\
 org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"> + <type>falcon-startup.properties</type> + <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/> + </definition> </changes> </component> </service> @@ -457,6 +522,43 @@ <type>hive-site</type> <transfer operation="delete" delete-key="datanucleus.rdbms.datastoreAdapterClassName"/> </definition> + + <definition xsi:type="configure" id="hdp_2_2_to_2_5_remove_ranger_hive_audit_db" + summary="Remove Ranger Hive Audit DB"> + <type>ranger-hive-audit</type> + <transfer operation="copy" from-type="ranger-hive-plugin-properties" + from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.dir" + default-value="hdfs://NAMENODE_HOSTNAME:8020/ranger/audit"/> + <transfer operation="copy" from-type="ranger-hive-plugin-properties" + from-key="XAAUDIT.HDFS.IS_ENABLED" + to-key="xasecure.audit.destination.hdfs" + default-value="true"/> + <transfer operation="copy" from-type="ranger-hive-plugin-properties" + from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" + default-value="/var/log/hive/audit/hdfs/spool"/> + <set key="xasecure.audit.destination.solr" value="false"/> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> + <set key="xasecure.audit.destination.solr.zookeepers" value="none"/> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/log/hive/audit/solr/spool"/> + <set key="xasecure.audit.provider.summary.enabled" value="false"/> + + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> + + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_hive_atlas_configs"> + <type>hive-site</type> + <transfer operation="delete" delete-key="atlas.rest.address" /> + <transfer operation="delete" delete-key="atlas.hook.hive.minThreads" /> + <transfer operation="delete" delete-key="atlas.hook.hive.maxThreads" /> + </definition> </changes> </component> @@ -482,6 +584,19 @@ <set key="templeton.hive.extra.files" value="/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"/> </definition> + <definition xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths" summary="Updating Configuration Paths"> + <type>webhcat-site</type> + <replace key="templeton.jar" find="/usr/hdp/current/hive-webhcat" + replace-with="/usr/hdp/${hdp.version}/hive"/> + <replace key="templeton.libjars" find="/usr/hdp/current/zookeeper-client" + replace-with="/usr/hdp/${hdp.version}/zookeeper,/usr/hdp/${hdp.version}/hive/lib/hive-common.jar"/> + <replace key="templeton.hadoop" find="/usr/hdp/current/hadoop-client" + replace-with="/usr/hdp/${hdp.version}/hadoop"/> + <replace key="templeton.hcat" find="/usr/hdp/current/hive-client" + replace-with="/usr/hdp/${hdp.version}/hive"/> + <set key="templeton.hive.extra.files" + value="/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"/> + </definition> </changes> </component> @@ -521,6 +636,11 @@ <set key="tez.lib.uris" value="/hdp/apps/${hdp.version}/tez/tez.tar.gz"/> </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_tez_client_adjust_tez_lib_uris_property"> + <type>tez-site</type> + <set key="tez.lib.uris" value="/hdp/apps/${hdp.version}/tez/tez.tar.gz"/> + </definition> + <definition xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"> <type>tez-site</type> <set key="tez.history.logging.service.class" @@ -614,6 +734,36 @@ <set key="xasecure.audit.provider.summary.enabled" value="false"/> </definition> + <definition xsi:type="configure" id="hdp_2_2_to_2_5_remove_ranger_storm_audit_db" + summary="Remove Ranger Storm Audit DB"> + <type>ranger-storm-audit</type> + <transfer operation="copy" from-type="ranger-storm-plugin-properties" + from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.dir" + default-value="hdfs://NAMENODE_HOSTNAME:8020/ranger/audit"/> + <transfer operation="copy" from-type="ranger-storm-plugin-properties" + from-key="XAAUDIT.HDFS.IS_ENABLED" + to-key="xasecure.audit.destination.hdfs" + default-value="true"/> + <transfer operation="copy" from-type="ranger-storm-plugin-properties" + from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" + default-value="/var/log/storm/audit/hdfs/spool"/> + <set key="xasecure.audit.destination.solr" value="false"/> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> + <set key="xasecure.audit.destination.solr.zookeepers" value="none"/> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/log/storm/audit/solr/spool"/> + <set key="xasecure.audit.provider.summary.enabled" value="false"/> + + <transfer operation="delete" delete-key="xasecure.audit.destination.db" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver" /> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" /> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" /> + </definition> + <definition xsi:type="configure" id="hdp_2_3_0_0_nimbus_remove_deprecated_ranger_properties" summary="Removing Deprecated Ranger Storm Plugin Configurations"> <type>ranger-storm-plugin-properties</type> @@ -650,6 +800,7 @@ <transfer operation="delete" delete-key="storm.topology.submission.notifier.plugin.class" if-key="storm.topology.submission.notifier.plugin.class" if-type="storm-site" if-value=" "/> </definition> + <definition xsi:type="configure" id="increase_storm_zookeeper_timeouts" summary="Increase storm.zookeeper.session.timeout and storm.zookeeper.connection.timeout property"> <type>storm-site</type> @@ -664,6 +815,57 @@ if-type="storm-site" if-value="15000" /> </definition> + + <definition xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0"> + <type>storm-site</type> + <replace key="_storm.thrift.nonsecure.transport" find="backtype.storm.security.auth.SimpleTransportPlugin" + replace-with="org.apache.storm.security.auth.SimpleTransportPlugin" /> + <replace key="_storm.thrift.secure.transport" find="backtype.storm.security.auth.KerberosSaslTransportPlugin" + replace-with="org.apache.storm.security.auth.KerberosSaslTransportPlugin" /> + <replace key="storm.messaging.transport" find="backtype.storm.messaging.netty.Context" + replace-with="org.apache.storm.messaging.netty.Context" /> + <replace key="nimbus.topology.validator" find="backtype.storm.nimbus.DefaultTopologyValidator" + replace-with="org.apache.storm.nimbus.DefaultTopologyValidator" /> + <replace key="topology.spout.wait.strategy" find="backtype.storm.spout.SleepSpoutWaitStrategy" + replace-with="org.apache.storm.spout.SleepSpoutWaitStrategy" /> + <replace key="topology.kryo.factory" find="backtype.storm.serialization.DefaultKryoFactory" + replace-with="org.apache.storm.serialization.DefaultKryoFactory" /> + <replace key="topology.tuple.serializer" find="backtype.storm.serialization.types.ListDelegateSerializer" + replace-with="org.apache.storm.serialization.types.ListDelegateSerializer" /> + <replace key="nimbus.authorizer" find="backtype.storm.security.auth.authorizer.SimpleACLAuthorizer" + replace-with="org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer" /> + <replace key="drpc.authorizer" find="backtype.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer" + replace-with="org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer" /> + <replace key="ui.filter" find="backtype.storm.security.auth.KerberosPrincipalToLocal" + replace-with="org.apache.storm.security.auth.KerberosPrincipalToLocal" /> + <replace key="storm.principal.tolocal" find="backtype.storm.security.auth.KerberosPrincipalToLocal" + replace-with="org.apache.storm.security.auth.KerberosPrincipalToLocal" /> + <set key="client.jartransformer.class" value="org.apache.storm.hack.StormShadeTransformer" /> + </definition> + + <!-- All of these configs are present in Atlas' application.properties file instead and then copied to the hook's atlas-application.properties file. --> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_storm_atlas_configs"> + <type>storm-site</type> + <transfer operation="delete" delete-key="atlas.cluster.name" /> + </definition> + + <definition xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs"> + <condition type="cluster-env" key="security_enabled" value="true"> + <type>storm-site</type> + <key>nimbus.impersonation.authorizer</key> + <value>org.apache.storm.security.auth.authorizer.ImpersonationAuthorizer</value> + </condition> + <condition type="cluster-env" key="security_enabled" value="true"> + <type>storm-site</type> + <key>nimbus.impersonation.acl</key> + <value>"{ {{storm_bare_jaas_principal}} : {hosts: ['*'], groups: ['*']}}"</value> + </condition> + <condition type="cluster-env" key="security_enabled" value="true"> + <type>storm-site</type> + <key>nimbus.admins</key> + <value>"['{{storm_bare_jaas_principal}}', '{{ambari_bare_jaas_principal}}']"</value> + </condition> + </definition> </changes> </component> </service> @@ -838,6 +1040,64 @@ <transfer operation="delete" delete-key="SYNC_SOURCE"/> <transfer operation="delete" delete-key="POLICY_MGR_URL"/> </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_audit_db_flag"> + <type>ranger-env</type> + <transfer operation="delete" delete-key="xasecure.audit.destination.db"/> + </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_audit_db_admin_properties"> + <type>admin-properties</type> + <transfer operation="delete" delete-key="audit_db_name" /> + <transfer operation="delete" delete-key="audit_db_user" /> + <transfer operation="delete" delete-key="audit_db_password" /> + </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_audit_db_ranger_admin_site"> + <type>ranger-admin-site</type> + <set key="ranger.audit.source.type" value="solr"/> + <transfer operation="delete" delete-key="ranger.jpa.audit.jdbc.driver" /> + <transfer operation="delete" delete-key="ranger.jpa.audit.jdbc.url" /> + <transfer operation="delete" delete-key="ranger.jpa.audit.jdbc.user" /> + <transfer operation="delete" delete-key="ranger.jpa.audit.jdbc.password" /> + <transfer operation="delete" delete-key="ranger.jpa.audit.jdbc.credential.alias" /> + <transfer operation="delete" delete-key="ranger.jpa.audit.jdbc.dialect" /> + </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_sso_property"> + <type>ranger-admin-site</type> + <transfer operation="delete" delete-key="ranger.sso.cookiename" /> + <transfer operation="delete" delete-key="ranger.sso.query.param.originalurl" /> + </definition> + <definition xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"> + <condition type="ranger-env" key="is_solrCloud_enabled" value="true"> + <type>ranger-env</type> + <key>is_external_solrCloud_enabled</key> + <value>true</value> + </condition> + </definition> + </changes> + </component> + </service> + + <service name="SQOOP"> + <component name="SQOOP"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_remove_sqoop_atlas_configs"> + <type>sqoop-site</type> + <transfer operation="delete" delete-key="atlas.cluster.name" /> + </definition> + + <!-- Add these configs if the cluster is Kerberized. + Will only be written to the local file system if Atlas is present. --> + <definition xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs"> + <condition type="cluster-env" key="security_enabled" value="true"> + <type>sqoop-atlas-application.properties</type> + <key>atlas.jaas.KafkaClient.option.useTicketCache</key> + <value>true</value> + </condition> + <condition type="cluster-env" key="security_enabled" value="true"> + <type>sqoop-atlas-application.properties</type> + <key>atlas.jaas.KafkaClient.option.renewTicket</key> + <value>true</value> + </condition> + </definition> </changes> </component> </service> @@ -973,6 +1233,36 @@ <set key="xasecure.audit.provider.summary.enabled" value="true"/> </definition> + <definition xsi:type="configure" id="hdp_2_2_to_2_5_remove_ranger_hbase_audit_db" + summary="Remove Ranger HBase Audit DB"> + <type>ranger-hbase-audit</type> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" + from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.dir" + default-value="hdfs://NAMENODE_HOSTNAME:8020/ranger/audit"/> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" + from-key="XAAUDIT.HDFS.IS_ENABLED" + to-key="xasecure.audit.destination.hdfs" + default-value="true"/> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" + from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" + default-value="/var/log/hbase/audit/hdfs/spool"/> + <set key="xasecure.audit.destination.solr" value="false"/> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> + <set key="xasecure.audit.destination.solr.zookeepers" value="none"/> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/log/hbase/audit/solr/spool"/> + <set key="xasecure.audit.provider.summary.enabled" value="true"/> + + <transfer operation="delete" delete-key="xasecure.audit.destination.db"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver"/> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir"/> + </definition> + <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_copy_ranger_policies"> <type>ranger-hbase-security</type> @@ -1175,6 +1465,36 @@ <transfer operation="delete" delete-key="REPOSITORY_NAME"/> <transfer operation="delete" delete-key="POLICY_MGR_URL"/> </definition> + + <definition xsi:type="configure" id="hdp_2_2_to_2_5_remove_ranger_knox_audit_db" + summary="Remove Ranger Knox Audit DB"> + <type>ranger-knox-audit</type> + <transfer operation="copy" from-type="ranger-knox-plugin-properties" + from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.dir" + default-value="hdfs://NAMENODE_HOSTNAME:8020/ranger/audit"/> + <transfer operation="copy" from-type="ranger-knox-plugin-properties" + from-key="XAAUDIT.HDFS.IS_ENABLED" + to-key="xasecure.audit.destination.hdfs" + default-value="true"/> + <transfer operation="copy" from-type="ranger-knox-plugin-properties" + from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" + to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" + default-value="/var/log/knox/audit/hdfs/spool"/> + <set key="xasecure.audit.destination.solr" value="false"/> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"/> + <set key="xasecure.audit.destination.solr.zookeepers" value="none"/> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/log/knox/audit/solr/spool"/> + <set key="xasecure.audit.provider.summary.enabled" value="false"/> + + <transfer operation="delete" delete-key="xasecure.audit.destination.db"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.url"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.user"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.password"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.jdbc.driver"/> + <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file"/> + <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir"/> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/1427726a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index 933183b..a5fcfa0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -76,7 +76,6 @@ <supports-auto-skip-failure>false</supports-auto-skip-failure> <service-check>false</service-check> <parallel-scheduler/> - <service-check>false</service-check> <service name="ATLAS"> <component>ATLAS_SERVER</component> @@ -189,6 +188,10 @@ <component>PHOENIX_QUERY_SERVER</component> </service> + <service name="KAFKA"> + <component>KAFKA_BROKER</component> + </service> + <service name="HDFS"> <component>DATANODE</component> <component>NAMENODE</component> @@ -269,6 +272,7 @@ <skippable>true</skippable> <!-- May fix configuration problems manually --> <supports-auto-skip-failure>false</supports-auto-skip-failure> + <!-- RANGER --> <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger"> <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_env"/> </execute-stage> @@ -313,7 +317,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_admin_db_audit"/> </execute-stage> - <!--HDFS--> + <!-- HDFS --> <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode"> <task xsi:type="configure" id="hdp_2_3_0_0_namenode_ha_adjustments"/> </execute-stage> @@ -342,7 +346,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_hdfs_ranger_hdfs_delete_old_properties"/> </execute-stage> - <!--YARN--> + <!-- YARN --> <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer"> <task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/> </execute-stage> @@ -497,7 +501,7 @@ <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/> </execute-stage> - <!--KNOX--> + <!-- KNOX --> <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway"> <task xsi:type="configure" id="hdp_2_3_0_0_knox_configure_ranger_policy"/> </execute-stage> @@ -626,6 +630,15 @@ </execute-stage> </group> + <group xsi:type="restart" name="KAFKA" title="Kafka"> + <service-check>false</service-check> + <skippable>true</skippable> + <parallel-scheduler/> + <service name="KAFKA"> + <component>KAFKA_BROKER</component> + </service> + </group> + <group xsi:type="restart" name="YARN_AND_MAPR" title="YARN and MapReduce2"> <service-check>false</service-check> <skippable>true</skippable> @@ -695,6 +708,7 @@ <service>ZOOKEEPER</service> <service>RANGER</service> <service>HDFS</service> + <service>KAFKA</service> <service>YARN</service> <service>MAPREDUCE2</service> <service>HBASE</service> @@ -710,6 +724,7 @@ <max-degree-of-parallelism>1</max-degree-of-parallelism> </parallel-scheduler> <service name="HIVE"> + <component>MYSQL_SERVER</component> <component>HIVE_METASTORE</component> <component>HIVE_SERVER</component> <component>WEBHCAT_SERVER</component> @@ -805,15 +820,6 @@ </service> </group> - <group xsi:type="restart" name="KAFKA" title="Kafka"> - <service-check>false</service-check> - <skippable>true</skippable> - <parallel-scheduler/> - <service name="KAFKA"> - <component>KAFKA_BROKER</component> - </service> - </group> - <group xsi:type="restart" name="KNOX" title="Knox"> <service-check>false</service-check> <skippable>true</skippable> @@ -878,7 +884,6 @@ <direction>UPGRADE</direction> <priority> <service>FALCON</service> - <service>KAFKA</service> <service>KNOX</service> <service>STORM</service> <service>SLIDER</service> @@ -977,6 +982,12 @@ </upgrade> </component> + <component name="SECONDARY_NAMENODE"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="DATANODE"> <upgrade> <task xsi:type="restart-task"/> @@ -1008,6 +1019,14 @@ </component> </service> + <service name="KAFKA"> + <component name="KAFKA_BROKER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + </service> + <service name="MAPREDUCE2"> <component name="HISTORYSERVER"> <upgrade> @@ -1101,6 +1120,12 @@ </service> <service name="HIVE"> + <component name="MYSQL_SERVER"> + <upgrade> + <task xsi:type="restart-task"/> + </upgrade> + </component> + <component name="HIVE_METASTORE"> <upgrade> <task xsi:type="restart-task"/> @@ -1194,14 +1219,6 @@ </component> </service> - <service name="KAFKA"> - <component name="KAFKA_BROKER"> - <upgrade> - <task xsi:type="restart-task"/> - </upgrade> - </component> - </service> - <service name="KNOX"> <component name="KNOX_GATEWAY"> <upgrade> @@ -1213,7 +1230,7 @@ <service name="STORM"> <component name="NIMBUS"> <pre-upgrade> - <task xsi:type="execute" summary="Removing Storm data from ZooKeeper"> + <task xsi:type="execute" hosts="any" summary="Removing Storm data from ZooKeeper"> <script>scripts/storm_upgrade.py</script> <function>delete_storm_zookeeper_data</function> </task> @@ -1229,7 +1246,7 @@ <message>Before continuing, please deactivate and kill any currently running topologies.</message> </task> - <task xsi:type="execute" summary="Removing Storm data from ZooKeeper"> + <task xsi:type="execute" hosts="any" summary="Removing Storm data from ZooKeeper"> <script>scripts/storm_upgrade.py</script> <function>delete_storm_zookeeper_data</function> </task>
