This is an automated email from the ASF dual-hosted git repository. dsen pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 675ca9a [AMBARI-23493] Decommission Datanode doesn't work (dsen) 675ca9a is described below commit 675ca9a64f70fae517aeefcadcbab137a239922a Author: Dmytro Sen <d...@apache.org> AuthorDate: Thu Apr 12 11:58:51 2018 +0300 [AMBARI-23493] Decommission Datanode doesn't work (dsen) --- .../AmbariCustomCommandExecutionHelper.java | 16 +++++++++++++ .../org/apache/ambari/server/utils/StageUtils.java | 28 ---------------------- .../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 5 +++- .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 5 +++- .../controller/AmbariManagementControllerTest.java | 9 +++++++ .../apache/ambari/server/utils/StageUtilsTest.java | 4 ---- 6 files changed, 33 insertions(+), 34 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 61b14c4..d396d12 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -129,6 +129,7 @@ public class AmbariCustomCommandExecutionHelper { public final static String DECOM_INCLUDED_HOSTS = "included_hosts"; public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts"; + public final static String ALL_DECOMMISSIONED_HOSTS = "all_decommissioned_hosts"; public final static String DECOM_SLAVE_COMPONENT = "slave_type"; public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only"; public final static String UPDATE_FILES_ONLY = "update_files_only"; @@ -1056,6 +1057,10 @@ public class AmbariCustomCommandExecutionHelper { } Map<String, String> commandParams = new HashMap<>(); + + commandParams.put(ALL_DECOMMISSIONED_HOSTS, + StringUtils.join(calculateDecommissionedNodes(service, slaveCompType), ',')); + if (serviceName.equals(Service.Type.HBASE.name())) { commandParams.put(DECOM_EXCLUDED_HOSTS, StringUtils.join(listOfExcludedHosts, ',')); if ((isDrainOnlyRequest != null) && isDrainOnlyRequest.equals("true")) { @@ -1072,6 +1077,17 @@ public class AmbariCustomCommandExecutionHelper { } } + private Set<String> calculateDecommissionedNodes(Service service, String slaveCompType) throws AmbariException { + Set<String> decommissionedHostsSet = new HashSet<>(); + ServiceComponent serviceComponent = service.getServiceComponent(slaveCompType); + for (ServiceComponentHost serviceComponentHost : serviceComponent.getServiceComponentHosts().values()) { + if (serviceComponentHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) { + decommissionedHostsSet.add(serviceComponentHost.getHostName()); + } + } + return decommissionedHostsSet; + } + private StringBuilder getReadableDecommissionCommandDetail( ActionExecutionContext actionExecutionContext, Set<String> includedHosts, diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index 1130e3d..f023b3a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -58,10 +58,8 @@ import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.HostComponentAdminState; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent; import org.apache.ambari.server.topology.TopologyManager; import org.apache.commons.lang.StringUtils; @@ -92,8 +90,6 @@ public class StageUtils { protected static final String RACKS = "all_racks"; protected static final String IPV4_ADDRESSES = "all_ipv4_ips"; - private static Map<String, String> decommissionedToClusterInfoKeyMap = - new HashMap<>(); private volatile static Gson gson; @Inject @@ -149,12 +145,6 @@ public class StageUtils { StageUtils.configuration = configuration; } - static { - decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts"); - decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts"); - decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts"); - decommissionedToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "decom_hbase_rs_hosts"); - } public static String getActionId(long requestId, long stageId) { return requestId + "-" + stageId; @@ -301,8 +291,6 @@ public class StageUtils { String roleName = getClusterHostInfoKey(componentName); - String decomRoleName = decommissionedToClusterInfoKeyMap.get(componentName); - for (String hostName : serviceComponent.getServiceComponentHosts().keySet()) { if (roleName != null) { @@ -317,22 +305,6 @@ public class StageUtils { //Add index of host to current host role hostsForComponentsHost.add(hostIndex); } - - if (decomRoleName != null) { - ServiceComponentHost scHost = serviceComponent.getServiceComponentHost(hostName); - if (scHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) { - SortedSet<Integer> hostsForComponentsHost = hostRolesInfo.get(decomRoleName); - - if (hostsForComponentsHost == null) { - hostsForComponentsHost = new TreeSet<>(); - hostRolesInfo.put(decomRoleName, hostsForComponentsHost); - } - - int hostIndex = hostsList.indexOf(hostName); - //Add index of host to current host role - hostsForComponentsHost.add(hostIndex); - } - } } } } diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 1a6b2a8..55476f9 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -166,7 +166,10 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file -hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) +if 'all_decommissioned_hosts' in config['commandParams']: + hdfs_exclude_file = config['commandParams']['all_decommissioned_hosts'].split(",") +else: + hdfs_exclude_file = [] exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] slave_hosts = default("/clusterHostInfo/datanode_hosts", []) include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 0de72e3..8df14e1 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -310,7 +310,10 @@ yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc user_group = config['configurations']['cluster-env']['user_group'] #exclude file -exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) +if 'all_decommissioned_hosts' in config['commandParams']: + exclude_hosts = config['commandParams']['all_decommissioned_hosts'].split(",") +else: + exclude_hosts = [] exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") rm_nodes_exclude_dir = os.path.dirname(exclude_file_path) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index af78e29..bf21825 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -2840,6 +2840,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(Role.HBASE_MASTER, command.getRole()); Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); + Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts")); assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); // RS stops @@ -2870,6 +2871,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState()); command = storedTasks.get(0); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); + Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts")); Assert.assertTrue("DECOMMISSION".equals(command.getCustomCommandName())); Assert.assertTrue(("DECOMMISSION, Excluded: " + host2).equals(command.getCommandDetail())); cmdParams = command.getExecutionCommandWrapper().getExecutionCommand().getCommandParams(); @@ -5938,6 +5940,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(Role.NAMENODE, command.getRole()); Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); + Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts")); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); // Decommission the other datanode @@ -5967,6 +5970,10 @@ public class AmbariManagementControllerTest { Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState()); Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); + Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host1)); + Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host2)); + Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host1+","+host2) || + execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host2+","+host1)); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); // Recommission the other datanode (while adding NameNode HA) @@ -6020,6 +6027,7 @@ public class AmbariManagementControllerTest { || !cmdParams.get("update_files_only").equals("true")) { countRefresh++; } + Assert.assertEquals("", cmdParams.get("all_decommissioned_hosts")); } Assert.assertEquals(2, countRefresh); @@ -6765,6 +6773,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(Role.NAMENODE, command.getRole()); Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); + Assert.assertEquals(host1, execCmd.getCommandParams().get("all_decommissioned_hosts")); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java index 8f3fe8e..e901d62 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java @@ -548,10 +548,6 @@ public class StageUtilsTest extends EasyMockSupport { } assertEquals(expectedPingPorts, reindexedPorts); - assertTrue(info.containsKey("decom_tt_hosts")); - Set<String> decommissionedHosts = info.get("decom_tt_hosts"); - assertEquals(2, decommissionedHosts.toString().split(",").length); - // check server hostname field assertTrue(info.containsKey(StageUtils.AMBARI_SERVER_HOST)); Set<String> serverHost = info.get(StageUtils.AMBARI_SERVER_HOST); -- To stop receiving notification emails like this one, please contact d...@apache.org.