AMBARI-18270. Host delete should support a force option to delete all components (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7ab43807 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7ab43807 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7ab43807 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 7ab438079f0e1a12fab52bc8abbdc73040473195 Parents: 392a752 Author: Sumit Mohanty <[email protected]> Authored: Wed Sep 28 22:55:50 2016 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Sep 28 22:55:50 2016 -0700 ---------------------------------------------------------------------- .../internal/HostResourceProvider.java | 130 ++++++++++++++----- .../server/controller/internal/RequestImpl.java | 2 +- .../AmbariManagementControllerTest.java | 106 +++++++++++++++ .../internal/HostResourceProviderTest.java | 15 ++- 4 files changed, 219 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7ab43807/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java index 4673c73..73bc908 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java @@ -41,6 +41,7 @@ import org.apache.ambari.server.controller.HostRequest; import org.apache.ambari.server.controller.HostResponse; import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.ServiceComponentHostRequest; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.NoSuchResourceException; import org.apache.ambari.server.controller.spi.Predicate; @@ -64,12 +65,14 @@ import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.InvalidTopologyTemplateException; import org.apache.ambari.server.topology.LogicalRequest; import org.apache.ambari.server.topology.TopologyManager; +import org.apache.ambari.server.update.HostUpdateHelper; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +133,6 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { public static final String HOST_RECOVERY_SUMMARY_PROPERTY_ID = PropertyHelper.getPropertyId("Hosts", "recovery_summary"); public static final String HOST_STATE_PROPERTY_ID = - PropertyHelper.getPropertyId("Hosts", "host_state"); public static final String HOST_LAST_AGENT_ENV_PROPERTY_ID = PropertyHelper.getPropertyId("Hosts", "last_agent_env"); @@ -152,6 +154,8 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { public static final String HOST_RACK_INFO_NO_CATEGORY_PROPERTY_ID = PropertyHelper.getPropertyId(null, "rack_info"); + protected static final String FORCE_DELETE_COMPONENTS = "force_delete_components"; + private static Set<String> pkPropertyIds = new HashSet<String>(Arrays.asList(new String[]{ @@ -335,6 +339,10 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { final Set<HostRequest> requests = new HashSet<>(); + Map<String, String> requestInfoProperties = request.getRequestInfoProperties(); + final boolean forceDelete = requestInfoProperties.containsKey(FORCE_DELETE_COMPONENTS) && + requestInfoProperties.get(FORCE_DELETE_COMPONENTS).equals("true"); + for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { requests.add(getRequest(propertyMap)); } @@ -342,11 +350,13 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { DeleteStatusMetaData deleteStatusMetaData = modifyResources(new Command<DeleteStatusMetaData>() { @Override public DeleteStatusMetaData invoke() throws AmbariException { - return deleteHosts(requests, request.isDryRunRequest()); + return deleteHosts(requests, request.isDryRunRequest(), forceDelete); } }); - notifyDelete(Resource.Type.Host, predicate); + if(!request.isDryRunRequest()) { + notifyDelete(Resource.Type.Host, predicate); + } return getRequestStatus(null, null, deleteStatusMetaData); } @@ -834,7 +844,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { } @Transactional - protected DeleteStatusMetaData deleteHosts(Set<HostRequest> requests, boolean dryRun) + protected DeleteStatusMetaData deleteHosts(Set<HostRequest> requests, boolean dryRun, boolean forceDelete) throws AmbariException { AmbariManagementController controller = getManagementController(); @@ -849,7 +859,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { } try { - validateHostInDeleteFriendlyState(hostRequest, clusters); + validateHostInDeleteFriendlyState(hostRequest, clusters, forceDelete); okToRemove.add(hostRequest); } catch (Exception ex) { deleteStatusMetaData.addException(hostName, ex); @@ -867,16 +877,19 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { //Do not break behavior for existing clients where delete request contains only 1 host. //Response for these requests will have empty body with appropriate error code. - if (deleteStatusMetaData.getDeletedKeys().size() + deleteStatusMetaData.getExceptionForKeys().size() == 1) { - if (deleteStatusMetaData.getDeletedKeys().size() == 1) { - return null; - } - for (Map.Entry<String, Exception> entry : deleteStatusMetaData.getExceptionForKeys().entrySet()) { - Exception ex = entry.getValue(); - if (ex instanceof AmbariException) { - throw (AmbariException)ex; - } else { - throw new AmbariException(ex.getMessage(), ex); + //dryRun is a new feature so its ok to unify the behavior + if (!dryRun) { + if (deleteStatusMetaData.getDeletedKeys().size() + deleteStatusMetaData.getExceptionForKeys().size() == 1) { + if (deleteStatusMetaData.getDeletedKeys().size() == 1) { + return null; + } + for (Map.Entry<String, Exception> entry : deleteStatusMetaData.getExceptionForKeys().entrySet()) { + Exception ex = entry.getValue(); + if (ex instanceof AmbariException) { + throw (AmbariException) ex; + } else { + throw new AmbariException(ex.getMessage(), ex); + } } } } @@ -888,6 +901,38 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { for (HostRequest hostRequest : requests) { // Assume the user also wants to delete it entirely, including all clusters. String hostname = hostRequest.getHostname(); + + // delete all host components + Set<ServiceComponentHostRequest> schrs = new HashSet<>(); + for(Cluster cluster : clusters.getClustersForHost(hostname)) { + List<ServiceComponentHost> list = cluster.getServiceComponentHosts(hostname); + for (ServiceComponentHost sch : list) { + ServiceComponentHostRequest schr = new ServiceComponentHostRequest(cluster.getClusterName(), + sch.getServiceName(), + sch.getServiceComponentName(), + sch.getHostName(), + null); + schrs.add(schr); + } + } + DeleteStatusMetaData componentDeleteStatus = null; + if(schrs.size() > 0) { + try { + componentDeleteStatus = getManagementController().deleteHostComponents(schrs); + } catch (Exception ex) { + deleteStatusMetaData.addException(hostname, ex); + } + } + + if (componentDeleteStatus != null) { + for (String key : componentDeleteStatus.getDeletedKeys()) { + deleteStatusMetaData.addDeletedKey(key); + } + for (String key : componentDeleteStatus.getExceptionForKeys().keySet()) { + deleteStatusMetaData.addException(key, componentDeleteStatus.getExceptionForKeys().get(key)); + } + } + try { clusters.deleteHost(hostname); deleteStatusMetaData.addDeletedKey(hostname); @@ -905,37 +950,57 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { } } - private void validateHostInDeleteFriendlyState(HostRequest hostRequest, Clusters clusters ) throws AmbariException { + private void validateHostInDeleteFriendlyState(HostRequest hostRequest, Clusters clusters, boolean forceDelete) throws AmbariException { Set<String> clusterNamesForHost = new HashSet<>(); String hostName = hostRequest.getHostname(); - if (null != hostRequest.getClusterName()) { - clusterNamesForHost.add(hostRequest.getClusterName()); - } else { - Set<Cluster> clustersForHost = clusters.getClustersForHost(hostRequest.getHostname()); - if (null != clustersForHost) { - for (Cluster c : clustersForHost) { - clusterNamesForHost.add(c.getClusterName()); - } + if (null != hostRequest.getClusterName()) { + clusterNamesForHost.add(hostRequest.getClusterName()); + } else { + Set<Cluster> clustersForHost = clusters.getClustersForHost(hostRequest.getHostname()); + if (null != clustersForHost) { + for (Cluster c : clustersForHost) { + clusterNamesForHost.add(c.getClusterName()); } } + } - for (String clusterName : clusterNamesForHost) { - Cluster cluster = clusters.getCluster(clusterName); + for (String clusterName : clusterNamesForHost) { + Cluster cluster = clusters.getCluster(clusterName); - List<ServiceComponentHost> list = cluster.getServiceComponentHosts(hostName); + List<ServiceComponentHost> list = cluster.getServiceComponentHosts(hostName); - if (!list.isEmpty()) { - List<String> componentsToRemove = new ArrayList<>(); - for (ServiceComponentHost sch : list) { - componentsToRemove.add(sch.getServiceComponentName()); + if (!list.isEmpty()) { + List<String> componentsToRemove = new ArrayList<>(); + List<String> componentsStarted = new ArrayList<>(); + for (ServiceComponentHost sch : list) { + componentsToRemove.add(sch.getServiceComponentName()); + if (sch.getState() == State.STARTED) { + componentsStarted.add(sch.getServiceComponentName()); } + } + if (forceDelete) { + // error if components are running + if (!componentsStarted.isEmpty()) { + StringBuilder reason = new StringBuilder("Cannot remove host ") + .append(hostName) + .append(" from ") + .append(hostRequest.getClusterName()) + .append( + ". The following roles exist, and these components must be stopped: "); + + reason.append(StringUtils.join(componentsToRemove, ", ")); + + throw new AmbariException(reason.toString()); + } + } else { if (!componentsToRemove.isEmpty()) { StringBuilder reason = new StringBuilder("Cannot remove host ") .append(hostName) .append(" from ") .append(hostRequest.getClusterName()) - .append(". The following roles exist, and these components must be stopped if running, and then deleted: "); + .append( + ". The following roles exist, and these components must be stopped if running, and then deleted: "); reason.append(StringUtils.join(componentsToRemove, ", ")); @@ -943,6 +1008,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { } } } + } } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/7ab43807/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestImpl.java index 36ad4c3..aaeefd7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestImpl.java @@ -85,7 +85,7 @@ public class RequestImpl implements Request { * @param requestInfoProperties request properties; may be null * @param mapTemporalInfo temporal info */ - public RequestImpl(Set<String> propertyIds, Set<Map<String, Object>> properties, + public RequestImpl(Set<String> propertyIds, Set<Map<String, Object>> properties, Map<String, String> requestInfoProperties, Map<String,TemporalInfo> mapTemporalInfo) { this(propertyIds, properties, requestInfoProperties, mapTemporalInfo, null, null); } http://git-wip-us.apache.org/repos/asf/ambari/blob/7ab43807/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index db5adff..197b925 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -8949,6 +8949,112 @@ public class AmbariManagementControllerTest { } @Test + public void testDeleteHostComponentWithForce() throws Exception { + String cluster1 = getUniqueName(); + + createCluster(cluster1); + + Cluster cluster = clusters.getCluster(cluster1); + cluster.setDesiredStackVersion(new StackId("HDP-0.1")); + + String serviceName = "HDFS"; + createService(cluster1, serviceName, null); + String componentName1 = "NAMENODE"; + String componentName2 = "DATANODE"; + String componentName3 = "HDFS_CLIENT"; + + createServiceComponent(cluster1, serviceName, componentName1, State.INIT); + createServiceComponent(cluster1, serviceName, componentName2, State.INIT); + createServiceComponent(cluster1, serviceName, componentName3, State.INIT); + + String host1 = getUniqueName(); // Host will belong to the cluster and contain components + + addHostToCluster(host1, cluster1); + + // Add components to host1 + createServiceComponentHost(cluster1, serviceName, componentName1, host1, null); + createServiceComponentHost(cluster1, serviceName, componentName2, host1, null); + createServiceComponentHost(cluster1, serviceName, componentName3, host1, null); + + // Install + installService(cluster1, serviceName, false, false); + + // Treat host components on host1 as up and healthy + Map<String, ServiceComponentHost> hostComponents = cluster.getService(serviceName).getServiceComponent(componentName1).getServiceComponentHosts(); + for (Map.Entry<String, ServiceComponentHost> entry : hostComponents.entrySet()) { + ServiceComponentHost cHost = entry.getValue(); + cHost.handleEvent(new ServiceComponentHostInstallEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis(), cluster.getDesiredStackVersion().getStackId())); + cHost.handleEvent(new ServiceComponentHostOpSucceededEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis())); + } + hostComponents = cluster.getService(serviceName).getServiceComponent(componentName2).getServiceComponentHosts(); + for (Map.Entry<String, ServiceComponentHost> entry : hostComponents.entrySet()) { + ServiceComponentHost cHost = entry.getValue(); + cHost.handleEvent(new ServiceComponentHostInstallEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis(), cluster.getDesiredStackVersion().getStackId())); + cHost.handleEvent(new ServiceComponentHostOpSucceededEvent(cHost.getServiceComponentName(), cHost.getHostName(), System.currentTimeMillis())); + } + + // Case 1: Attempt delete when components still exist + Set<HostRequest> requests = new HashSet<HostRequest>(); + requests.clear(); + requests.add(new HostRequest(host1, cluster1, null)); + try { + HostResourceProviderTest.deleteHosts(controller, requests, false, false); + fail("Expect failure deleting hosts when components exist and have not been deleted."); + } catch (Exception e) { + LOG.info("Exception is - " + e.getMessage()); + Assert.assertTrue(e.getMessage().contains("these components must be stopped if running, and then deleted")); + } + + Service s = cluster.getService(serviceName); + s.getServiceComponent("DATANODE").getServiceComponentHost(host1).setState(State.STARTED); + try { + HostResourceProviderTest.deleteHosts(controller, requests, false, true); + fail("Expect failure deleting hosts when components exist and have not been stopped."); + } catch (Exception e) { + LOG.info("Exception is - " + e.getMessage()); + Assert.assertTrue(e.getMessage().contains("these components must be stopped:")); + } + + DeleteStatusMetaData data = null; + try { + data = HostResourceProviderTest.deleteHosts(controller, requests, true, true); + Assert.assertTrue(data.getDeletedKeys().size() == 0); + } catch (Exception e) { + LOG.info("Exception is - " + e.getMessage()); + fail("Do not expect failure deleting hosts when components exist and are stopped."); + } + + LOG.info("Test dry run of delete with all host components"); + s.getServiceComponent("DATANODE").getServiceComponentHost(host1).setState(State.INSTALLED); + try { + data = HostResourceProviderTest.deleteHosts(controller, requests, true, true); + Assert.assertTrue(data.getDeletedKeys().size() == 1); + } catch (Exception e) { + LOG.info("Exception is - " + e.getMessage()); + fail("Do not expect failure deleting hosts when components exist and are stopped."); + } + + LOG.info("Test successful delete with all host components"); + s.getServiceComponent("DATANODE").getServiceComponentHost(host1).setState(State.INSTALLED); + try { + data = HostResourceProviderTest.deleteHosts(controller, requests, false, true); + Assert.assertNotNull(data); + Assert.assertTrue(4 == data.getDeletedKeys().size()); + Assert.assertTrue(0 == data.getExceptionForKeys().size()); + } catch (Exception e) { + LOG.info("Exception is - " + e.getMessage()); + fail("Do not expect failure deleting hosts when components exist and are stopped."); + } + // Verify host does not exist + try { + clusters.getHost(host1); + Assert.fail("Expected a HostNotFoundException."); + } catch (HostNotFoundException e) { + // expected + } + } + + @Test public void testDeleteHost() throws Exception { String cluster1 = getUniqueName(); http://git-wip-us.apache.org/repos/asf/ambari/blob/7ab43807/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostResourceProviderTest.java index 260ff92..08dd591 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostResourceProviderTest.java @@ -1340,7 +1340,20 @@ public class HostResourceProviderTest extends EasyMockSupport { HostResourceProvider provider = getHostProvider(controller); HostResourceProvider.setTopologyManager(topologyManager); - provider.deleteHosts(requests, false); + provider.deleteHosts(requests, false, false); + } + + public static DeleteStatusMetaData deleteHosts(AmbariManagementController controller, + Set<HostRequest> requests, boolean dryRun, boolean forceDelete) + throws AmbariException { + TopologyManager topologyManager = EasyMock.createNiceMock(TopologyManager.class); + expect(topologyManager.getRequests(Collections.EMPTY_LIST)).andReturn(Collections.EMPTY_LIST).anyTimes(); + + replay(topologyManager); + + HostResourceProvider provider = getHostProvider(controller); + HostResourceProvider.setTopologyManager(topologyManager); + return provider.deleteHosts(requests, dryRun, forceDelete); } public static void updateHosts(AmbariManagementController controller, Set<HostRequest> requests)
