This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 5ab00009037ff820d312a747547226225d6b5911 Author: Aravindan Vijayan <avija...@hortonworks.com> AuthorDate: Thu Apr 12 12:07:06 2018 -0700 Revert "[AMBARI-23493] Decommission Datanode doesn't work (dsen)" This reverts commit 675ca9a64f70fae517aeefcadcbab137a239922a. --- .../AmbariCustomCommandExecutionHelper.java | 16 ------------- .../org/apache/ambari/server/utils/StageUtils.java | 28 ++++++++++++++++++++++ .../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 5 +--- .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 5 +--- .../controller/AmbariManagementControllerTest.java | 9 ------- .../apache/ambari/server/utils/StageUtilsTest.java | 4 ++++ 6 files changed, 34 insertions(+), 33 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index d396d12..61b14c4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -129,7 +129,6 @@ public class AmbariCustomCommandExecutionHelper { public final static String DECOM_INCLUDED_HOSTS = "included_hosts"; public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts"; - public final static String ALL_DECOMMISSIONED_HOSTS = "all_decommissioned_hosts"; public final static String DECOM_SLAVE_COMPONENT = "slave_type"; public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only"; public final static String UPDATE_FILES_ONLY = "update_files_only"; @@ -1057,10 +1056,6 @@ public class AmbariCustomCommandExecutionHelper { } Map<String, String> commandParams = new HashMap<>(); - - commandParams.put(ALL_DECOMMISSIONED_HOSTS, - StringUtils.join(calculateDecommissionedNodes(service, slaveCompType), ',')); - if (serviceName.equals(Service.Type.HBASE.name())) { commandParams.put(DECOM_EXCLUDED_HOSTS, StringUtils.join(listOfExcludedHosts, ',')); if ((isDrainOnlyRequest != null) && isDrainOnlyRequest.equals("true")) { @@ -1077,17 +1072,6 @@ public class AmbariCustomCommandExecutionHelper { } } - private Set<String> calculateDecommissionedNodes(Service service, String slaveCompType) throws AmbariException { - Set<String> decommissionedHostsSet = new HashSet<>(); - ServiceComponent serviceComponent = service.getServiceComponent(slaveCompType); - for (ServiceComponentHost serviceComponentHost : serviceComponent.getServiceComponentHosts().values()) { - if (serviceComponentHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) { - decommissionedHostsSet.add(serviceComponentHost.getHostName()); - } - } - return decommissionedHostsSet; - } - private StringBuilder getReadableDecommissionCommandDetail( ActionExecutionContext actionExecutionContext, Set<String> includedHosts, diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index f023b3a..1130e3d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -58,8 +58,10 @@ import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Host; +import org.apache.ambari.server.state.HostComponentAdminState; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; +import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent; import org.apache.ambari.server.topology.TopologyManager; import org.apache.commons.lang.StringUtils; @@ -90,6 +92,8 @@ public class StageUtils { protected static final String RACKS = "all_racks"; protected static final String IPV4_ADDRESSES = "all_ipv4_ips"; + private static Map<String, String> decommissionedToClusterInfoKeyMap = + new HashMap<>(); private volatile static Gson gson; @Inject @@ -145,6 +149,12 @@ public class StageUtils { StageUtils.configuration = configuration; } + static { + decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts"); + decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts"); + decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts"); + decommissionedToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "decom_hbase_rs_hosts"); + } public static String getActionId(long requestId, long stageId) { return requestId + "-" + stageId; @@ -291,6 +301,8 @@ public class StageUtils { String roleName = getClusterHostInfoKey(componentName); + String decomRoleName = decommissionedToClusterInfoKeyMap.get(componentName); + for (String hostName : serviceComponent.getServiceComponentHosts().keySet()) { if (roleName != null) { @@ -305,6 +317,22 @@ public class StageUtils { //Add index of host to current host role hostsForComponentsHost.add(hostIndex); } + + if (decomRoleName != null) { + ServiceComponentHost scHost = serviceComponent.getServiceComponentHost(hostName); + if (scHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) { + SortedSet<Integer> hostsForComponentsHost = hostRolesInfo.get(decomRoleName); + + if (hostsForComponentsHost == null) { + hostsForComponentsHost = new TreeSet<>(); + hostRolesInfo.put(decomRoleName, hostsForComponentsHost); + } + + int hostIndex = hostsList.indexOf(hostName); + //Add index of host to current host role + hostsForComponentsHost.add(hostIndex); + } + } } } } diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 55476f9..1a6b2a8 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -166,10 +166,7 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] falcon_user = config['configurations']['falcon-env']['falcon_user'] #exclude file -if 'all_decommissioned_hosts' in config['commandParams']: - hdfs_exclude_file = config['commandParams']['all_decommissioned_hosts'].split(",") -else: - hdfs_exclude_file = [] +hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] slave_hosts = default("/clusterHostInfo/datanode_hosts", []) include_file_path = default("/configurations/hdfs-site/dfs.hosts", None) diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 8df14e1..0de72e3 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -310,10 +310,7 @@ yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc user_group = config['configurations']['cluster-env']['user_group'] #exclude file -if 'all_decommissioned_hosts' in config['commandParams']: - exclude_hosts = config['commandParams']['all_decommissioned_hosts'].split(",") -else: - exclude_hosts = [] +exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", []) exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude") rm_nodes_exclude_dir = os.path.dirname(exclude_file_path) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index bf21825..af78e29 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -2840,7 +2840,6 @@ public class AmbariManagementControllerTest { Assert.assertEquals(Role.HBASE_MASTER, command.getRole()); Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); - Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts")); assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); // RS stops @@ -2871,7 +2870,6 @@ public class AmbariManagementControllerTest { Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState()); command = storedTasks.get(0); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); - Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts")); Assert.assertTrue("DECOMMISSION".equals(command.getCustomCommandName())); Assert.assertTrue(("DECOMMISSION, Excluded: " + host2).equals(command.getCommandDetail())); cmdParams = command.getExecutionCommandWrapper().getExecutionCommand().getCommandParams(); @@ -5940,7 +5938,6 @@ public class AmbariManagementControllerTest { Assert.assertEquals(Role.NAMENODE, command.getRole()); Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); - Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts")); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); // Decommission the other datanode @@ -5970,10 +5967,6 @@ public class AmbariManagementControllerTest { Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState()); Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); - Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host1)); - Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host2)); - Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host1+","+host2) || - execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host2+","+host1)); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); // Recommission the other datanode (while adding NameNode HA) @@ -6027,7 +6020,6 @@ public class AmbariManagementControllerTest { || !cmdParams.get("update_files_only").equals("true")) { countRefresh++; } - Assert.assertEquals("", cmdParams.get("all_decommissioned_hosts")); } Assert.assertEquals(2, countRefresh); @@ -6773,7 +6765,6 @@ public class AmbariManagementControllerTest { Assert.assertEquals(Role.NAMENODE, command.getRole()); Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand()); Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command")); - Assert.assertEquals(host1, execCmd.getCommandParams().get("all_decommissioned_hosts")); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java index e901d62..8f3fe8e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java @@ -548,6 +548,10 @@ public class StageUtilsTest extends EasyMockSupport { } assertEquals(expectedPingPorts, reindexedPorts); + assertTrue(info.containsKey("decom_tt_hosts")); + Set<String> decommissionedHosts = info.get("decom_tt_hosts"); + assertEquals(2, decommissionedHosts.toString().split(",").length); + // check server hostname field assertTrue(info.containsKey(StageUtils.AMBARI_SERVER_HOST)); Set<String> serverHost = info.get(StageUtils.AMBARI_SERVER_HOST); -- To stop receiving notification emails like this one, please contact avija...@apache.org.