This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new c37a739 [AMBARI-23493] Decommission Datanode doesn't work (dsen)
c37a739 is described below
commit c37a7399c2b5d873af3dff56ecf8cd416a4684ec
Author: Dmytro Sen <[email protected]>
AuthorDate: Mon Apr 16 17:38:17 2018 +0300
[AMBARI-23493] Decommission Datanode doesn't work (dsen)
---
.../AmbariCustomCommandExecutionHelper.java | 16 +++++++++++
.../org/apache/ambari/server/utils/StageUtils.java | 31 +---------------------
.../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 5 +++-
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 5 +++-
.../controller/AmbariManagementControllerTest.java | 9 +++++++
.../apache/ambari/server/utils/StageUtilsTest.java | 4 ---
6 files changed, 34 insertions(+), 36 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 61b14c4..d396d12 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -129,6 +129,7 @@ public class AmbariCustomCommandExecutionHelper {
public final static String DECOM_INCLUDED_HOSTS = "included_hosts";
public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts";
+ public final static String ALL_DECOMMISSIONED_HOSTS =
"all_decommissioned_hosts";
public final static String DECOM_SLAVE_COMPONENT = "slave_type";
public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only";
public final static String UPDATE_FILES_ONLY = "update_files_only";
@@ -1056,6 +1057,10 @@ public class AmbariCustomCommandExecutionHelper {
}
Map<String, String> commandParams = new HashMap<>();
+
+ commandParams.put(ALL_DECOMMISSIONED_HOSTS,
+ StringUtils.join(calculateDecommissionedNodes(service,
slaveCompType), ','));
+
if (serviceName.equals(Service.Type.HBASE.name())) {
commandParams.put(DECOM_EXCLUDED_HOSTS,
StringUtils.join(listOfExcludedHosts, ','));
if ((isDrainOnlyRequest != null) && isDrainOnlyRequest.equals("true"))
{
@@ -1072,6 +1077,17 @@ public class AmbariCustomCommandExecutionHelper {
}
}
+ private Set<String> calculateDecommissionedNodes(Service service, String
slaveCompType) throws AmbariException {
+ Set<String> decommissionedHostsSet = new HashSet<>();
+ ServiceComponent serviceComponent =
service.getServiceComponent(slaveCompType);
+ for (ServiceComponentHost serviceComponentHost :
serviceComponent.getServiceComponentHosts().values()) {
+ if (serviceComponentHost.getComponentAdminState() ==
HostComponentAdminState.DECOMMISSIONED) {
+ decommissionedHostsSet.add(serviceComponentHost.getHostName());
+ }
+ }
+ return decommissionedHostsSet;
+ }
+
private StringBuilder getReadableDecommissionCommandDetail(
ActionExecutionContext actionExecutionContext, Set<String> includedHosts,
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index b6287e6..9abc62b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -58,10 +58,8 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.ActionExecutionContext;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
-import org.apache.ambari.server.state.ServiceComponentHost;
import
org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent;
import org.apache.ambari.server.topology.TopologyManager;
import org.apache.commons.lang.StringUtils;
@@ -94,8 +92,6 @@ public class StageUtils {
private static Map<String, String> componentToClusterInfoKeyMap =
new HashMap<>();
- private static Map<String, String> decommissionedToClusterInfoKeyMap =
- new HashMap<>();
private volatile static Gson gson;
@Inject
@@ -181,13 +177,6 @@ public class StageUtils {
componentToClusterInfoKeyMap.put("ACCUMULO_TSERVER",
"accumulo_tserver_hosts");
}
- static {
- decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts");
- decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts");
- decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts");
- decommissionedToClusterInfoKeyMap.put("HBASE_REGIONSERVER",
"decom_hbase_rs_hosts");
- }
-
public static String getActionId(long requestId, long stageId) {
return requestId + "-" + stageId;
}
@@ -334,9 +323,7 @@ public class StageUtils {
additionalComponentToClusterInfoKeyMap.put(componentName, roleName);
}
- String decomRoleName =
decommissionedToClusterInfoKeyMap.get(componentName);
-
- if (roleName == null && decomRoleName == null) {
+ if (roleName == null) {
continue;
}
@@ -354,22 +341,6 @@ public class StageUtils {
//Add index of host to current host role
hostsForComponentsHost.add(hostIndex);
}
-
- if (decomRoleName != null) {
- ServiceComponentHost scHost =
serviceComponent.getServiceComponentHost(hostName);
- if (scHost.getComponentAdminState() ==
HostComponentAdminState.DECOMMISSIONED) {
- SortedSet<Integer> hostsForComponentsHost =
hostRolesInfo.get(decomRoleName);
-
- if (hostsForComponentsHost == null) {
- hostsForComponentsHost = new TreeSet<>();
- hostRolesInfo.put(decomRoleName, hostsForComponentsHost);
- }
-
- int hostIndex = hostsList.indexOf(hostName);
- //Add index of host to current host role
- hostsForComponentsHost.add(hostIndex);
- }
- }
}
}
}
diff --git
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 1a6b2a8..55476f9 100644
---
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -166,7 +166,10 @@ hdfs_user_keytab =
config['configurations']['hadoop-env']['hdfs_user_keytab']
falcon_user = config['configurations']['falcon-env']['falcon_user']
#exclude file
-hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
+if 'all_decommissioned_hosts' in config['commandParams']:
+ hdfs_exclude_file =
config['commandParams']['all_decommissioned_hosts'].split(",")
+else:
+ hdfs_exclude_file = []
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
slave_hosts = default("/clusterHostInfo/datanode_hosts", [])
include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
diff --git
a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 0de72e3..8df14e1 100644
---
a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -310,7 +310,10 @@ yarn_job_summary_log =
format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc
user_group = config['configurations']['cluster-env']['user_group']
#exclude file
-exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
+if 'all_decommissioned_hosts' in config['commandParams']:
+ exclude_hosts =
config['commandParams']['all_decommissioned_hosts'].split(",")
+else:
+ exclude_hosts = []
exclude_file_path =
default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 084efdd..1ff1f5d 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -2840,6 +2840,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.HBASE_MASTER, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION",
execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host2,
execCmd.getCommandParams().get("all_decommissioned_hosts"));
assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY),
response.getRequestContext());
// RS stops
@@ -2870,6 +2871,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
command = storedTasks.get(0);
Assert.assertEquals("DECOMMISSION",
execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host2,
execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertTrue("DECOMMISSION".equals(command.getCustomCommandName()));
Assert.assertTrue(("DECOMMISSION, Excluded: " +
host2).equals(command.getCommandDetail()));
cmdParams =
command.getExecutionCommandWrapper().getExecutionCommand().getCommandParams();
@@ -5938,6 +5940,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.NAMENODE, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION",
execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host2,
execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY),
response.getRequestContext());
// Decommission the other datanode
@@ -5967,6 +5970,10 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED,
scHost.getComponentAdminState());
Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
Assert.assertEquals("DECOMMISSION",
execCmd.getCommandParams().get("custom_command"));
+
Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host1));
+
Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host2));
+
Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host1+","+host2)
||
+
execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host2+","+host1));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY),
response.getRequestContext());
// Recommission the other datanode (while adding NameNode HA)
@@ -6020,6 +6027,7 @@ public class AmbariManagementControllerTest {
|| !cmdParams.get("update_files_only").equals("true")) {
countRefresh++;
}
+ Assert.assertEquals("", cmdParams.get("all_decommissioned_hosts"));
}
Assert.assertEquals(2, countRefresh);
@@ -6765,6 +6773,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.NAMENODE, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION",
execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host1,
execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY),
response.getRequestContext());
}
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
index 3e8d348..e5c8943 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
@@ -548,10 +548,6 @@ public class StageUtilsTest extends EasyMockSupport {
}
assertEquals(expectedPingPorts, reindexedPorts);
- assertTrue(info.containsKey("decom_tt_hosts"));
- Set<String> decommissionedHosts = info.get("decom_tt_hosts");
- assertEquals(2, decommissionedHosts.toString().split(",").length);
-
// check server hostname field
assertTrue(info.containsKey(StageUtils.AMBARI_SERVER_HOST));
Set<String> serverHost = info.get(StageUtils.AMBARI_SERVER_HOST);
--
To stop receiving notification emails like this one, please contact
[email protected].
