This is an automated email from the ASF dual-hosted git repository.

avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/trunk by this push:
     new c37a739  [AMBARI-23493] Decommission Datanode doesn't work (dsen)
c37a739 is described below

commit c37a7399c2b5d873af3dff56ecf8cd416a4684ec
Author: Dmytro Sen <d...@apache.org>
AuthorDate: Mon Apr 16 17:38:17 2018 +0300

    [AMBARI-23493] Decommission Datanode doesn't work (dsen)
---
 .../AmbariCustomCommandExecutionHelper.java        | 16 +++++++++++
 .../org/apache/ambari/server/utils/StageUtils.java | 31 +---------------------
 .../HDFS/2.1.0.2.0/package/scripts/params_linux.py |  5 +++-
 .../YARN/2.1.0.2.0/package/scripts/params_linux.py |  5 +++-
 .../controller/AmbariManagementControllerTest.java |  9 +++++++
 .../apache/ambari/server/utils/StageUtilsTest.java |  4 ---
 6 files changed, 34 insertions(+), 36 deletions(-)

diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 61b14c4..d396d12 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -129,6 +129,7 @@ public class AmbariCustomCommandExecutionHelper {
 
   public final static String DECOM_INCLUDED_HOSTS = "included_hosts";
   public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts";
+  public final static String ALL_DECOMMISSIONED_HOSTS = 
"all_decommissioned_hosts";
   public final static String DECOM_SLAVE_COMPONENT = "slave_type";
   public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only";
   public final static String UPDATE_FILES_ONLY = "update_files_only";
@@ -1056,6 +1057,10 @@ public class AmbariCustomCommandExecutionHelper {
       }
 
       Map<String, String> commandParams = new HashMap<>();
+
+      commandParams.put(ALL_DECOMMISSIONED_HOSTS,
+          StringUtils.join(calculateDecommissionedNodes(service, 
slaveCompType), ','));
+
       if (serviceName.equals(Service.Type.HBASE.name())) {
         commandParams.put(DECOM_EXCLUDED_HOSTS, 
StringUtils.join(listOfExcludedHosts, ','));
         if ((isDrainOnlyRequest != null) && isDrainOnlyRequest.equals("true")) 
{
@@ -1072,6 +1077,17 @@ public class AmbariCustomCommandExecutionHelper {
     }
   }
 
+  private Set<String> calculateDecommissionedNodes(Service service, String 
slaveCompType) throws AmbariException {
+    Set<String> decommissionedHostsSet = new HashSet<>();
+    ServiceComponent serviceComponent = 
service.getServiceComponent(slaveCompType);
+    for (ServiceComponentHost serviceComponentHost : 
serviceComponent.getServiceComponentHosts().values()) {
+      if (serviceComponentHost.getComponentAdminState() == 
HostComponentAdminState.DECOMMISSIONED) {
+        decommissionedHostsSet.add(serviceComponentHost.getHostName());
+      }
+    }
+    return decommissionedHostsSet;
+  }
+
 
   private StringBuilder getReadableDecommissionCommandDetail(
       ActionExecutionContext actionExecutionContext, Set<String> includedHosts,
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java 
b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index b6287e6..9abc62b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -58,10 +58,8 @@ import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.ActionExecutionContext;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.HostComponentAdminState;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
-import org.apache.ambari.server.state.ServiceComponentHost;
 import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent;
 import org.apache.ambari.server.topology.TopologyManager;
 import org.apache.commons.lang.StringUtils;
@@ -94,8 +92,6 @@ public class StageUtils {
 
   private static Map<String, String> componentToClusterInfoKeyMap =
     new HashMap<>();
-  private static Map<String, String> decommissionedToClusterInfoKeyMap =
-    new HashMap<>();
   private volatile static Gson gson;
 
   @Inject
@@ -181,13 +177,6 @@ public class StageUtils {
     componentToClusterInfoKeyMap.put("ACCUMULO_TSERVER", 
"accumulo_tserver_hosts");
   }
 
-  static {
-    decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts");
-    decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts");
-    decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts");
-    decommissionedToClusterInfoKeyMap.put("HBASE_REGIONSERVER", 
"decom_hbase_rs_hosts");
-  }
-
   public static String getActionId(long requestId, long stageId) {
     return requestId + "-" + stageId;
   }
@@ -334,9 +323,7 @@ public class StageUtils {
           additionalComponentToClusterInfoKeyMap.put(componentName, roleName);
         }
 
-        String decomRoleName = 
decommissionedToClusterInfoKeyMap.get(componentName);
-
-        if (roleName == null && decomRoleName == null) {
+        if (roleName == null) {
           continue;
         }
 
@@ -354,22 +341,6 @@ public class StageUtils {
             //Add index of host to current host role
             hostsForComponentsHost.add(hostIndex);
           }
-
-          if (decomRoleName != null) {
-            ServiceComponentHost scHost = 
serviceComponent.getServiceComponentHost(hostName);
-            if (scHost.getComponentAdminState() == 
HostComponentAdminState.DECOMMISSIONED) {
-              SortedSet<Integer> hostsForComponentsHost = 
hostRolesInfo.get(decomRoleName);
-
-              if (hostsForComponentsHost == null) {
-                hostsForComponentsHost = new TreeSet<>();
-                hostRolesInfo.put(decomRoleName, hostsForComponentsHost);
-              }
-
-              int hostIndex = hostsList.indexOf(hostName);
-              //Add index of host to current host role
-              hostsForComponentsHost.add(hostIndex);
-            }
-          }
         }
       }
     }
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 1a6b2a8..55476f9 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -166,7 +166,10 @@ hdfs_user_keytab = 
config['configurations']['hadoop-env']['hdfs_user_keytab']
 falcon_user = config['configurations']['falcon-env']['falcon_user']
 
 #exclude file
-hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
+if 'all_decommissioned_hosts' in config['commandParams']:
+  hdfs_exclude_file = 
config['commandParams']['all_decommissioned_hosts'].split(",")
+else:
+  hdfs_exclude_file = []
 exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
 slave_hosts = default("/clusterHostInfo/datanode_hosts", [])
 include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
diff --git 
a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
 
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 0de72e3..8df14e1 100644
--- 
a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ 
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -310,7 +310,10 @@ yarn_job_summary_log = 
format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc
 user_group = config['configurations']['cluster-env']['user_group']
 
 #exclude file
-exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
+if 'all_decommissioned_hosts' in config['commandParams']:
+  exclude_hosts = 
config['commandParams']['all_decommissioned_hosts'].split(",")
+else:
+  exclude_hosts = []
 exclude_file_path = 
default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
 rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
 
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 084efdd..1ff1f5d 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -2840,6 +2840,7 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(Role.HBASE_MASTER, command.getRole());
     Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
     Assert.assertEquals("DECOMMISSION", 
execCmd.getCommandParams().get("custom_command"));
+    Assert.assertEquals(host2, 
execCmd.getCommandParams().get("all_decommissioned_hosts"));
     assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), 
response.getRequestContext());
 
     // RS stops
@@ -2870,6 +2871,7 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
     command = storedTasks.get(0);
     Assert.assertEquals("DECOMMISSION", 
execCmd.getCommandParams().get("custom_command"));
+    Assert.assertEquals(host2, 
execCmd.getCommandParams().get("all_decommissioned_hosts"));
     Assert.assertTrue("DECOMMISSION".equals(command.getCustomCommandName()));
     Assert.assertTrue(("DECOMMISSION, Excluded: " + 
host2).equals(command.getCommandDetail()));
     cmdParams = 
command.getExecutionCommandWrapper().getExecutionCommand().getCommandParams();
@@ -5938,6 +5940,7 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(Role.NAMENODE, command.getRole());
     Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
     Assert.assertEquals("DECOMMISSION", 
execCmd.getCommandParams().get("custom_command"));
+    Assert.assertEquals(host2, 
execCmd.getCommandParams().get("all_decommissioned_hosts"));
     Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), 
response.getRequestContext());
 
     // Decommission the other datanode
@@ -5967,6 +5970,10 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, 
scHost.getComponentAdminState());
     Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
     Assert.assertEquals("DECOMMISSION", 
execCmd.getCommandParams().get("custom_command"));
+    
Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host1));
+    
Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host2));
+    
Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host1+","+host2)
 ||
+      
execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host2+","+host1));
     Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), 
response.getRequestContext());
 
     // Recommission the other datanode  (while adding NameNode HA)
@@ -6020,6 +6027,7 @@ public class AmbariManagementControllerTest {
           || !cmdParams.get("update_files_only").equals("true")) {
         countRefresh++;
       }
+      Assert.assertEquals("", cmdParams.get("all_decommissioned_hosts"));
     }
     Assert.assertEquals(2, countRefresh);
 
@@ -6765,6 +6773,7 @@ public class AmbariManagementControllerTest {
     Assert.assertEquals(Role.NAMENODE, command.getRole());
     Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
     Assert.assertEquals("DECOMMISSION", 
execCmd.getCommandParams().get("custom_command"));
+    Assert.assertEquals(host1, 
execCmd.getCommandParams().get("all_decommissioned_hosts"));
     Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), 
response.getRequestContext());
   }
 
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
index 3e8d348..e5c8943 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
@@ -548,10 +548,6 @@ public class StageUtilsTest extends EasyMockSupport {
     }
     assertEquals(expectedPingPorts, reindexedPorts);
 
-    assertTrue(info.containsKey("decom_tt_hosts"));
-    Set<String> decommissionedHosts = info.get("decom_tt_hosts");
-    assertEquals(2, decommissionedHosts.toString().split(",").length);
-
     // check server hostname field
     assertTrue(info.containsKey(StageUtils.AMBARI_SERVER_HOST));
     Set<String> serverHost = info.get(StageUtils.AMBARI_SERVER_HOST);

-- 
To stop receiving notification emails like this one, please contact
avija...@apache.org.

Reply via email to