fixing forceful termination to terminate the cluster instances as well
Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/d3c03f56 Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/d3c03f56 Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/d3c03f56 Branch: refs/heads/tenantisolation Commit: d3c03f56b327ec4c296bd6f0d5294f238fec1bea Parents: f5c4c8e Author: reka <[email protected]> Authored: Thu Jun 25 15:51:07 2015 +0530 Committer: reka <[email protected]> Committed: Thu Jun 25 16:32:02 2015 +0530 ---------------------------------------------------------------------- .../applications/dependency/DependencyTree.java | 12 +++- .../monitor/component/GroupMonitor.java | 11 ++++ .../component/ParentComponentMonitor.java | 22 +++++++ .../services/impl/AutoscalerServiceImpl.java | 67 +++++++++++--------- 4 files changed, 81 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/d3c03f56/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/dependency/DependencyTree.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/dependency/DependencyTree.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/dependency/DependencyTree.java index f1291b4..04af9b1 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/dependency/DependencyTree.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/dependency/DependencyTree.java @@ -213,8 +213,9 @@ public class DependencyTree { * @param childContexts contains the children of the node * @return all the children of the given node */ - public List<ApplicationChildContext> findAllChildrenOfAppContext(List<ApplicationChildContext> applicationContexts, - List<ApplicationChildContext> childContexts) { + private List<ApplicationChildContext> findAllChildrenOfAppContext( + List<ApplicationChildContext> applicationContexts, + List<ApplicationChildContext> childContexts) { for (ApplicationChildContext context : applicationContexts) { childContexts.add(context); findAllChildrenOfAppContext(context.getApplicationChildContextList(), childContexts); @@ -222,6 +223,13 @@ public class DependencyTree { return childContexts; } + public List<ApplicationChildContext> findAllChildrenOfAppContext() { + List<ApplicationChildContext> allChildrenOfAppContext = + new ArrayList<ApplicationChildContext>(); + return findAllChildrenOfAppContext(this.primaryApplicationContextList, + allChildrenOfAppContext); + } + public void setTerminationBehavior(TerminationBehavior terminationBehavior) { this.terminationBehavior = terminationBehavior; } http://git-wip-us.apache.org/repos/asf/stratos/blob/d3c03f56/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java index 2d25e97..37d1d5f 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.stratos.autoscaler.algorithms.PartitionAlgorithm; import org.apache.stratos.autoscaler.applications.ApplicationHolder; +import org.apache.stratos.autoscaler.applications.dependency.context.ApplicationChildContext; import org.apache.stratos.autoscaler.applications.topic.ApplicationBuilder; import org.apache.stratos.autoscaler.context.AutoscalerContext; import org.apache.stratos.autoscaler.context.InstanceContext; @@ -514,6 +515,16 @@ public class GroupMonitor extends ParentComponentMonitor { if (instance != null) { // If this parent instance is terminating, then based on child notification, // it has to decide its state rather than starting a the children recovery + + ApplicationMonitor applicationMonitor = AutoscalerContext.getInstance(). + getAppMonitor(appId); + //In case if the group instance is not in terminating while application is + // terminating, changing the status to terminating + if(applicationMonitor.isTerminating() && instance.getStatus().getCode() < 3) { + //Sending group instance terminating event + ApplicationBuilder.handleGroupTerminatingEvent(appId, id, instanceId); + } + if (instance.getStatus() == GroupStatus.Terminating || instance.getStatus() == GroupStatus.Terminated) { ServiceReferenceHolder.getInstance().getGroupStatusProcessorChain().process(id, http://git-wip-us.apache.org/repos/asf/stratos/blob/d3c03f56/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java index cbfc1ea..717edc3 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java @@ -656,6 +656,13 @@ public abstract class ParentComponentMonitor extends Monitor { } } + if(log.isDebugEnabled()) { + log.debug(String.format("Calculating the group instances status for [application] " + + "%s [group] %s [group-instance] %s [required-status] %s [no-of-instances] %s", + appId, childId, instanceId, requiredStatus.toString(), + noOfInstancesOfRequiredStatus)); + } + if (!groupInstances.isEmpty()) { ParentLevelNetworkPartitionContext networkPartitionContext = (ParentLevelNetworkPartitionContext) ((GroupMonitor) monitor). @@ -664,12 +671,27 @@ public abstract class ParentComponentMonitor extends Monitor { //if terminated all the instances in this instances map should be in terminated state if (noOfInstancesOfRequiredStatus == this.inactiveInstancesMap.size() && requiredStatus == GroupStatus.Terminated) { + if(log.isDebugEnabled()) { + log.debug(String.format("Group instances in required status for [application] " + + "%s [group] %s [group-instance] %s [required-status] %s", + appId, childId, instanceId, GroupStatus.Terminated.toString())); + } return true; } else if (noOfInstancesOfRequiredStatus >= minInstances) { + if(log.isDebugEnabled()) { + log.debug(String.format("Group instances in required status for [application] " + + "%s [group] %s [group-instance] %s [required-status] %s", + appId, childId, instanceId, requiredStatus.toString())); + } return true; } else { //of only one is inActive implies that the whole group is Inactive if (requiredStatus == GroupStatus.Inactive && noOfInstancesOfRequiredStatus >= 1) { + if(log.isDebugEnabled()) { + log.debug(String.format("Group instances in required status for [application] " + + "%s [group] %s [group-instance] %s [required-status] %s", + appId, childId, instanceId, GroupStatus.Inactive.toString())); + } return true; } } http://git-wip-us.apache.org/repos/asf/stratos/blob/d3c03f56/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java index e6baf1e..9dd67bb 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java @@ -498,7 +498,8 @@ public class AutoscalerServiceImpl implements AutoscalerService { ApplicationMonitor appMonitor = asCtx.getAppMonitor(applicationId); if (appMonitor == null) { - log.info(String.format("Could not find application monitor for the application %s, hence returning", applicationId)); + log.info(String.format("Could not find application monitor for the application %s, " + + "hence returning", applicationId)); return false; } if (!force) { @@ -517,14 +518,16 @@ public class AutoscalerServiceImpl implements AutoscalerService { if (appMonitor.isTerminating()) { if (appMonitor.isForce()) { - log.warn(String.format("Force un-deployment is already in progress, hence not invoking again " + + log.warn(String.format("Force un-deployment is already in progress, " + + "hence not invoking again " + "[application-id] %s", applicationId)); return false; } else { log.info(String.format("Previous graceful un-deployment is in progress for " + - "[application-id] %s , thus terminating instances directly", applicationId)); + "[application-id] %s , thus terminating instances directly", + applicationId)); appMonitor.setForce(true); - terminateAllApplicationMembersForcefully(applicationId); + terminateAllMembersAndClustersForcefully(applicationId); } } else { log.info(String.format("Forcefully un-deploying the application " + applicationId)); @@ -542,7 +545,8 @@ public class AutoscalerServiceImpl implements AutoscalerService { log.info("Starting to undeploy application: [application-id] " + applicationId); } - ApplicationContext applicationContext = AutoscalerContext.getInstance().getApplicationContext(applicationId); + ApplicationContext applicationContext = AutoscalerContext.getInstance(). + getApplicationContext(applicationId); Application application = ApplicationHolder.getApplications().getApplication(applicationId); if ((applicationContext == null) || (application == null)) { String msg = String.format("Application not found: [application-id] %s", applicationId); @@ -923,7 +927,7 @@ public class AutoscalerServiceImpl implements AutoscalerService { return PolicyManager.getInstance().getApplicationPolicies(); } - private void terminateAllApplicationMembersForcefully(String applicationId) { + private void terminateAllMembersAndClustersForcefully(String applicationId) { if (StringUtils.isEmpty(applicationId)) { throw new IllegalArgumentException("Application Id cannot be empty"); } @@ -955,38 +959,43 @@ public class AutoscalerServiceImpl implements AutoscalerService { } //If there are no members in cluster Instance, send cluster Terminated Event - Collection<ClusterInstance> allClusterInstances = cluster.getClusterInstances(); - for (ClusterInstance clusterInstance : allClusterInstances) { - boolean anyMembersInClusterInstance = false; - - for (Member member : cluster.getMembers()) { - if (member.getClusterInstanceId().equals(clusterInstance.getInstanceId())) { - anyMembersInClusterInstance = true; - } + //Stopping the cluster monitor thread + ClusterMonitor clusterMonitor = AutoscalerContext.getInstance(). + getClusterMonitor(clusterId); + if(clusterMonitor != null) { + clusterMonitor.destroy(); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Cluster monitor cannot be found for [application] %s " + + "[cluster] %s", applicationId, clusterId)); } - - if (anyMembersInClusterInstance == false) { + } + if(cluster != null) { + Collection<ClusterInstance> allClusterInstances = cluster.getClusterInstances(); + for (ClusterInstance clusterInstance : allClusterInstances) { ClusterStatusEventPublisher.sendClusterTerminatedEvent(applicationId, cluster.getServiceName(), clusterId, clusterInstance.getInstanceId()); } - } - List<String> memberListToTerminate = new LinkedList<String>(); - for (Member member : cluster.getMembers()) { - memberListToTerminate.add(member.getMemberId()); - } + List<String> memberListToTerminate = new LinkedList<String>(); + for (Member member : cluster.getMembers()) { + memberListToTerminate.add(member.getMemberId()); + } - for (String memberIdToTerminate : memberListToTerminate) { - try { - log.info(String.format("Terminating member forcefully [member-id] %s of the cluster [cluster-id] %s " + - "[application-id] %s", memberIdToTerminate, clusterId, application)); - AutoscalerCloudControllerClient.getInstance().terminateInstanceForcefully(memberIdToTerminate); - } catch (Exception e) { - log.error(String.format("Forceful termination of member %s has failed, but continuing forceful " + - "deletion of other members", memberIdToTerminate)); + for (String memberIdToTerminate : memberListToTerminate) { + try { + log.info(String.format("Terminating member forcefully [member-id] %s of the cluster [cluster-id] %s " + + "[application-id] %s", memberIdToTerminate, clusterId, application)); + AutoscalerCloudControllerClient.getInstance().terminateInstanceForcefully(memberIdToTerminate); + } catch (Exception e) { + log.error(String.format("Forceful termination of member %s has failed, but continuing forceful " + + "deletion of other members", memberIdToTerminate)); + } } } + } + }
