Repository: stratos Updated Branches: refs/heads/master 605d2875d -> fc95e5bfc
fixing undeployment and restart issue when group scaling happens Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/fc95e5bf Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/fc95e5bf Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/fc95e5bf Branch: refs/heads/master Commit: fc95e5bfcd2c3cc89f20046b26539222c49b7c67 Parents: 605d287 Author: reka <[email protected]> Authored: Thu Jul 2 14:06:09 2015 +0530 Committer: reka <[email protected]> Committed: Thu Jul 2 14:06:09 2015 +0530 ---------------------------------------------------------------------- .../applications/topic/ApplicationBuilder.java | 3 +- .../context/cluster/ClusterInstanceContext.java | 10 + .../AutoscalerTopologyEventReceiver.java | 3 + .../autoscaler/monitor/MonitorFactory.java | 2 +- .../monitor/cluster/ClusterMonitor.java | 190 +++++++++++-------- .../monitor/component/ApplicationMonitor.java | 4 +- .../component/ParentComponentMonitor.java | 7 +- .../services/impl/AutoscalerServiceImpl.java | 3 - 8 files changed, 133 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/topic/ApplicationBuilder.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/topic/ApplicationBuilder.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/topic/ApplicationBuilder.java index 36ada2d..5b5ec90 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/topic/ApplicationBuilder.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/applications/topic/ApplicationBuilder.java @@ -307,7 +307,8 @@ public class ApplicationBuilder { // stopping application thread applicationMonitor.destroy(); AutoscalerContext.getInstance().removeAppMonitor(applicationId); - + // Remove network partition algorithm context + AutoscalerContext.getInstance().removeNetworkPartitionAlgorithmContext(applicationId); // update application status in application context applicationContext.setStatus(ApplicationContext.STATUS_CREATED); AutoscalerContext.getInstance().updateApplicationContext(applicationContext); http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/context/cluster/ClusterInstanceContext.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/context/cluster/ClusterInstanceContext.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/context/cluster/ClusterInstanceContext.java index a90d929..d1964b3 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/context/cluster/ClusterInstanceContext.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/context/cluster/ClusterInstanceContext.java @@ -170,6 +170,16 @@ public class ClusterInstanceContext extends InstanceContext { return activeMemberCount; } + public int getPendingMemberCount() { + + int activeMemberCount = 0; + for (ClusterLevelPartitionContext partitionContext : partitionCtxts.values()) { + + activeMemberCount += partitionContext.getPendingMembers().size(); + } + return activeMemberCount; + } + public int getNonTerminatedMemberCount() { int nonTerminatedMemberCount = 0; http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/event/receiver/topology/AutoscalerTopologyEventReceiver.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/event/receiver/topology/AutoscalerTopologyEventReceiver.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/event/receiver/topology/AutoscalerTopologyEventReceiver.java index 34098ee..4e6d8fd 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/event/receiver/topology/AutoscalerTopologyEventReceiver.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/event/receiver/topology/AutoscalerTopologyEventReceiver.java @@ -288,6 +288,9 @@ public class AutoscalerTopologyEventReceiver { //Terminating the pending members monitor.terminatePendingMembers(clusterInstanceId, clusterInstance.getNetworkPartitionId()); + //Move all members to terminating pending list + monitor.moveMembersToTerminatingPending(clusterInstanceId, + clusterInstance.getNetworkPartitionId()); } else { monitor.notifyParentMonitor(ClusterStatus.Terminating, clusterInstanceId); monitor.terminateAllMembers(clusterInstanceId, clusterInstance.getNetworkPartitionId()); http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/MonitorFactory.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/MonitorFactory.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/MonitorFactory.java index 2e6b1b8..b8ad83c 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/MonitorFactory.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/MonitorFactory.java @@ -149,7 +149,7 @@ public class MonitorFactory { //Find whether any other instances exists in group // which has not been added to in-memory model in the restart ApplicationMonitor applicationMonitor = AutoscalerContext.getInstance().getAppMonitor(appId); - if(applicationMonitor != null && applicationMonitor.isTerminating()) { + if(applicationMonitor != null && applicationMonitor.isRestarting()) { Collection<Instance> instances = parentMonitor.getInstances(); for(Instance instance : instances) { if(!instanceIds.contains(instance.getInstanceId())) { http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java index 2f0c4ac..241f6cd 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java @@ -30,6 +30,7 @@ import org.apache.stratos.autoscaler.context.cluster.ClusterInstanceContext; import org.apache.stratos.autoscaler.context.member.MemberStatsContext; import org.apache.stratos.autoscaler.context.partition.ClusterLevelPartitionContext; import org.apache.stratos.autoscaler.context.partition.network.ClusterLevelNetworkPartitionContext; +import org.apache.stratos.autoscaler.context.partition.network.NetworkPartitionContext; import org.apache.stratos.autoscaler.event.publisher.ClusterStatusEventPublisher; import org.apache.stratos.autoscaler.event.publisher.InstanceNotificationPublisher; import org.apache.stratos.autoscaler.exception.InvalidArgumentException; @@ -466,6 +467,25 @@ public class ClusterMonitor extends Monitor { executorService.execute(monitoringRunnable); } + if (instance.getStatus() == ClusterStatus.Terminating) { + //Move active members to Termination pending + if (instanceContext.getActiveMembers() > 0) { + //Sending cluster cleanup event + InstanceNotificationPublisher.getInstance(). + sendInstanceCleanupEventForCluster(clusterId, + instance.getInstanceId()); + moveMembersToTerminatingPending(networkPartitionContext.getId(), + instance.getInstanceId()); + } + + if (instanceContext.getPendingMemberCount() > 0) { + //Terminating the pending members + terminatePendingMembers(networkPartitionContext.getId(), + instance.getInstanceId()); + } + + } + for (final ClusterLevelPartitionContext partitionContext : instanceContext.getPartitionCtxts()) { Runnable monitoringRunnable = new Runnable() { @Override @@ -520,7 +540,7 @@ public class ClusterMonitor extends Monitor { @Override public void destroy() { //shutting down the scheduler - if(schedulerFuture != null) { + if (schedulerFuture != null) { schedulerFuture.cancel(true); } @@ -1350,56 +1370,108 @@ public class ClusterMonitor extends Monitor { Thread memberTerminator = new Thread(new Runnable() { public void run() { + NetworkPartitionContext networkPartitionContext = + getAllNetworkPartitionCtxts().get(networkPartitionId); + ClusterInstanceContext instanceContext = - (ClusterInstanceContext) getAllNetworkPartitionCtxts().get(networkPartitionId) - .getInstanceContext(instanceId); - boolean allMovedToObsolete = true; - for (ClusterLevelPartitionContext partitionContext : instanceContext.getPartitionCtxts()) { - if (log.isInfoEnabled()) { - log.info("Starting to terminate all members in cluster [" + getClusterId() + "] " + - "Network Partition [" + instanceContext.getNetworkPartitionId() + "], Partition [" + - partitionContext.getPartitionId() + "]"); - } + (ClusterInstanceContext) networkPartitionContext. + getInstanceContext(instanceId); + if (instanceContext != null) { + boolean allMovedToObsolete = true; + for (ClusterLevelPartitionContext partitionContext : instanceContext.getPartitionCtxts()) { + if (log.isInfoEnabled()) { + log.info("Starting to terminate all members in cluster [" + getClusterId() + "] " + + "Network Partition [" + instanceContext.getNetworkPartitionId() + "], Partition [" + + partitionContext.getPartitionId() + "]"); + } - if (AutoscalerContext.getInstance().getAppMonitor(getAppId()).isForce()) { - log.info(String.format("Terminating all remaining members of partition [partition-id] %s [application-id] %s", partitionContext.getPartitionId(), getAppId())); - partitionContext.terminateAllRemainingInstances(); - } - //Need to terminate pending members - Iterator<MemberContext> pendingIterator = partitionContext.getPendingMembers().listIterator(); - List<String> pendingMemberIdList = new ArrayList<String>(); - while (pendingIterator.hasNext()) { - MemberContext pendingMemberContext = pendingIterator.next(); - pendingMemberIdList.add(pendingMemberContext.getMemberId()); + if (AutoscalerContext.getInstance().getAppMonitor(getAppId()).isForce()) { + log.info(String.format("Terminating all remaining members of partition [partition-id] %s [application-id] %s", partitionContext.getPartitionId(), getAppId())); + partitionContext.terminateAllRemainingInstances(); + } + //Need to terminate pending members + Iterator<MemberContext> pendingIterator = partitionContext.getPendingMembers().listIterator(); + List<String> pendingMemberIdList = new ArrayList<String>(); + while (pendingIterator.hasNext()) { + MemberContext pendingMemberContext = pendingIterator.next(); + pendingMemberIdList.add(pendingMemberContext.getMemberId()); - } - for (String memberId : pendingMemberIdList) { - // pending members - if (log.isDebugEnabled()) { - log.debug("Moving pending member [member id] " + memberId + " to obsolete list"); } - partitionContext.movePendingMemberToObsoleteMembers(memberId); + for (String memberId : pendingMemberIdList) { + // pending members + if (log.isDebugEnabled()) { + log.debug("Moving pending member [member id] " + memberId + " to obsolete list"); + } + partitionContext.movePendingMemberToObsoleteMembers(memberId); + } + + allMovedToObsolete = partitionContext.getTotalMemberCount() == 0; } - /* - if (partitionContext.getTotalMemberCount() == 0) { - allMovedToObsolete = allMovedToObsolete && true; - } else { - allMovedToObsolete = false; + if (allMovedToObsolete) { + monitor.monitor(); } - */ - allMovedToObsolete = partitionContext.getTotalMemberCount() == 0; } - if (allMovedToObsolete) { - monitor.monitor(); - } } }, "Member Terminator - [cluster id] " + getClusterId()); memberTerminator.start(); } + public void moveMembersToTerminatingPending(final String instanceId, final String networkPartitionId) { + final ClusterMonitor monitor = this; + Thread memberTerminator = new Thread(new Runnable() { + public void run() { + + NetworkPartitionContext networkPartitionContext = + getAllNetworkPartitionCtxts().get(networkPartitionId); + + ClusterInstanceContext instanceContext = + (ClusterInstanceContext) networkPartitionContext. + getInstanceContext(instanceId); + if (instanceContext != null) { + for (ClusterLevelPartitionContext partitionContext : instanceContext.getPartitionCtxts()) { + if (log.isInfoEnabled()) { + log.info("Starting to move all members in cluster [" + getClusterId() + "] " + + "Network Partition [" + instanceContext.getNetworkPartitionId() + "], Partition [" + + partitionContext.getPartitionId() + "] to termination pending list"); + } + + //Need to terminate pending members + Iterator<MemberContext> activeIterator = partitionContext.getActiveMembers().listIterator(); + List<String> activeMemberIdList = new ArrayList<String>(); + while (activeIterator.hasNext()) { + MemberContext activeMemberContext = activeIterator.next(); + activeMemberIdList.add(activeMemberContext.getMemberId()); + + } + for (String memberId : activeMemberIdList) { + // pending members + if (log.isDebugEnabled()) { + log.debug("Moving pending member [member id] " + memberId + " to obsolete list"); + } + partitionContext.moveActiveMemberToTerminationPendingMembers(memberId); + if (partitionContext.getMemberStatsContext(memberId) != null) { + partitionContext.removeMemberStatsContext(memberId); + } + if (log.isDebugEnabled()) { + log.debug(String.format("Member has been moved as pending termination, " + + "and member stat context is removed: " + + "[member] %s", memberId)); + } + } + + } + } + + + } + }, "Active to Terminating pending adder - [cluster id] " + getClusterId()); + + memberTerminator.start(); + } + public Map<String, ClusterLevelNetworkPartitionContext> getAllNetworkPartitionCtxts() { return (this.clusterContext).getNetworkPartitionCtxts(); @@ -1521,52 +1593,6 @@ public class ClusterMonitor extends Monitor { } - /** - * Move all the members of the cluster instance to termiantion pending - * - * @param instanceId - */ - public void moveMembersFromActiveToPendingTermination(String instanceId) { - - //TODO take read lock for network partition context - //FIXME to iterate properly - for (ClusterLevelNetworkPartitionContext networkPartitionContext : - (this.clusterContext).getNetworkPartitionCtxts().values()) { - ClusterInstanceContext clusterInstanceContext = - (ClusterInstanceContext) networkPartitionContext.getInstanceContext(instanceId); - if (clusterInstanceContext != null) { - for (ClusterLevelPartitionContext partitionContext : clusterInstanceContext.getPartitionCtxts()) { - List<String> members = new ArrayList<String>(); - - Iterator<MemberContext> iterator = partitionContext.getActiveMembers().listIterator(); - while (iterator.hasNext()) { - MemberContext activeMember = iterator.next(); - members.add(activeMember.getMemberId()); - } - - for (String memberId : members) { - partitionContext.moveActiveMemberToTerminationPendingMembers( - memberId); - } - List<String> pendingMembers = new ArrayList<String>(); - - Iterator<MemberContext> pendingIterator = partitionContext.getPendingMembers().listIterator(); - while (pendingIterator.hasNext()) { - MemberContext activeMember = pendingIterator.next(); - pendingMembers.add(activeMember.getMemberId()); - } - for (String memberId : members) { - // pending members - if (log.isDebugEnabled()) { - log.debug("Moving pending member [member id] " + memberId + " the obsolete list"); - } - partitionContext.movePendingMemberToObsoleteMembers(memberId); - } - } - } - } - } - public String getDeploymentPolicyId() { return deploymentPolicyId; } http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java index e2f91ec..a2abe81 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java @@ -569,11 +569,13 @@ public class ApplicationMonitor extends ParentComponentMonitor { getInstanceByNetworkPartitionId(context.getId()); if (appInstance != null) { //use the existing instance in the Topology to create the data + if(!isRestarting) { + this.setRestarting(true); + } instanceId = handleApplicationInstanceCreation(application, context, appInstance); initialStartup = false; } else { //create new app instance as it doesn't exist in the Topology - this.setRestarting(true); instanceId = handleApplicationInstanceCreation(application, context, null); } http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java index 7e32022..588b49f 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java @@ -244,7 +244,12 @@ public abstract class ParentComponentMonitor extends Monitor { Monitor monitor = aliasToActiveChildMonitorsMap.get(context.getId()); // Creating new instance for (String instanceId : parentInstanceIds) { - monitor.createInstanceOnDemand(instanceId); + if(monitor.getInstancesByParentInstanceId(instanceId) == null) { + monitor.createInstanceOnDemand(instanceId); + } else { + log.info(String.format("Instance has already exists for [application] " + + "%s [component] %s", getAppId(), context.getId())); + } } } } http://git-wip-us.apache.org/repos/asf/stratos/blob/fc95e5bf/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java index 6d40cf6..9e52e61 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/services/impl/AutoscalerServiceImpl.java @@ -532,9 +532,6 @@ public class AutoscalerServiceImpl implements AutoscalerService { // Remove application signup(s) in stratos manager removeApplicationSignUp(applicationContext); - // Remove network partition algorithm context - AutoscalerContext.getInstance().removeNetworkPartitionAlgorithmContext(applicationId); - ApplicationBuilder.handleApplicationUnDeployedEvent(applicationId); if (log.isInfoEnabled()) {
