Repository: stratos Updated Branches: refs/heads/master 58bea52be -> 1c21daaee
fxing locking issue when updating hierarchical groups Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/1c21daae Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/1c21daae Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/1c21daae Branch: refs/heads/master Commit: 1c21daaeea7b27ad0a0141a70b32e9443e78e309 Parents: 966a437 Author: reka <[email protected]> Authored: Mon Jun 22 00:09:31 2015 +0530 Committer: reka <[email protected]> Committed: Mon Jun 22 00:09:51 2015 +0530 ---------------------------------------------------------------------- .../stratos/autoscaler/monitor/Monitor.java | 1 - .../monitor/component/ApplicationMonitor.java | 100 ++++++---- .../monitor/component/GroupMonitor.java | 200 +++++++++++-------- .../component/ParentComponentMonitor.java | 14 +- 4 files changed, 189 insertions(+), 126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/1c21daae/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/Monitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/Monitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/Monitor.java index e6560f2..a32e024 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/Monitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/Monitor.java @@ -65,7 +65,6 @@ public abstract class Monitor implements EventHandler, Runnable { */ public abstract boolean createInstanceOnDemand(String instanceId); - /** * Return the id of the monitor * http://git-wip-us.apache.org/repos/asf/stratos/blob/1c21daae/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java index 60bd84d..d7fd545 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ApplicationMonitor.java @@ -29,8 +29,8 @@ import org.apache.stratos.autoscaler.applications.topic.ApplicationBuilder; import org.apache.stratos.autoscaler.context.AutoscalerContext; import org.apache.stratos.autoscaler.context.InstanceContext; import org.apache.stratos.autoscaler.context.application.ApplicationInstanceContext; -import org.apache.stratos.autoscaler.context.partition.network.ParentLevelNetworkPartitionContext; import org.apache.stratos.autoscaler.context.partition.network.NetworkPartitionContext; +import org.apache.stratos.autoscaler.context.partition.network.ParentLevelNetworkPartitionContext; import org.apache.stratos.autoscaler.exception.application.DependencyBuilderException; import org.apache.stratos.autoscaler.exception.application.MonitorNotFoundException; import org.apache.stratos.autoscaler.exception.application.TopologyInConsistentException; @@ -315,53 +315,60 @@ public class ApplicationMonitor extends ParentComponentMonitor { } @Override - public void onChildStatusEvent(MonitorStatusEvent statusEvent) { - String childId = statusEvent.getId(); - String instanceId = statusEvent.getInstanceId(); - LifeCycleState status1 = statusEvent.getStatus(); - //Events coming from parent are In_Active(in faulty detection), Scaling events, termination - if (status1 == ClusterStatus.Active || status1 == GroupStatus.Active) { - onChildActivatedEvent(childId, instanceId); - - } else if (status1 == ClusterStatus.Inactive || status1 == GroupStatus.Inactive) { - this.markInstanceAsInactive(childId, instanceId); - onChildInactiveEvent(childId, instanceId); - - } else if (status1 == ClusterStatus.Terminating || status1 == GroupStatus.Terminating) { - //mark the child monitor as inActive in the map - markInstanceAsTerminating(childId, instanceId); - - } else if (status1 == ClusterStatus.Terminated || status1 == GroupStatus.Terminated) { - //Check whether all dependent goes Terminated and then start them in parallel. - removeInstanceFromFromInactiveMap(childId, instanceId); - removeInstanceFromFromTerminatingMap(childId, instanceId); - - ApplicationInstance instance = (ApplicationInstance) instanceIdToInstanceMap.get(instanceId); - if (instance != null) { - if (isTerminating() || instance.getStatus() == ApplicationStatus.Terminating || - instance.getStatus() == ApplicationStatus.Terminated) { - ServiceReferenceHolder.getInstance().getGroupStatusProcessorChain().process(this.id, - appId, instanceId); - } else { - Monitor monitor = this.getMonitor(childId); - boolean active = false; - if (monitor instanceof GroupMonitor) { - //Checking whether the Group is still active in case the faulty member - // identified after scaling up - active = verifyGroupStatus(childId, instanceId, GroupStatus.Active); - } - if (!active) { - onChildTerminatedEvent(childId, instanceId); + public void onChildStatusEvent(final MonitorStatusEvent statusEvent) { + Runnable monitoringRunnable = new Runnable() { + @Override + public void run() { + String childId = statusEvent.getId(); + String instanceId = statusEvent.getInstanceId(); + LifeCycleState status1 = statusEvent.getStatus(); + //Events coming from parent are In_Active(in faulty detection), Scaling events, termination + if (status1 == ClusterStatus.Active || status1 == GroupStatus.Active) { + onChildActivatedEvent(childId, instanceId); + + } else if (status1 == ClusterStatus.Inactive || status1 == GroupStatus.Inactive) { + markInstanceAsInactive(childId, instanceId); + onChildInactiveEvent(childId, instanceId); + + } else if (status1 == ClusterStatus.Terminating || status1 == GroupStatus.Terminating) { + //mark the child monitor as inActive in the map + markInstanceAsTerminating(childId, instanceId); + + } else if (status1 == ClusterStatus.Terminated || status1 == GroupStatus.Terminated) { + //Check whether all dependent goes Terminated and then start them in parallel. + removeInstanceFromFromInactiveMap(childId, instanceId); + removeInstanceFromFromTerminatingMap(childId, instanceId); + + ApplicationInstance instance = (ApplicationInstance) instanceIdToInstanceMap.get(instanceId); + if (instance != null) { + if (isTerminating() || instance.getStatus() == ApplicationStatus.Terminating || + instance.getStatus() == ApplicationStatus.Terminated) { + ServiceReferenceHolder.getInstance().getGroupStatusProcessorChain().process(id, + appId, instanceId); + } else { + Monitor monitor = getMonitor(childId); + boolean active = false; + if (monitor instanceof GroupMonitor) { + //Checking whether the Group is still active in case the faulty member + // identified after scaling up + active = verifyGroupStatus(childId, instanceId, GroupStatus.Active); + } + if (!active) { + onChildTerminatedEvent(childId, instanceId); + } else { + log.info("[Group Instance] " + instanceId + " is still active " + + "upon termination of the [child ] " + childId); + } + } } else { - log.info("[Group Instance] " + instanceId + " is still active " + - "upon termination of the [child ] " + childId); + log.warn("The required instance cannot be found in the the [GroupMonitor] " + + id); } } - } else { - log.warn("The required instance cannot be found in the the [GroupMonitor] " + - this.id); } - } + }; + executorService.execute(monitoringRunnable); + } @Override @@ -687,4 +694,9 @@ public class ApplicationMonitor extends ParentComponentMonitor { public void setForce(boolean force) { this.force = force; } + + @Override + public boolean createInstanceOnTermination(String instanceId) { + return false; + } } http://git-wip-us.apache.org/repos/asf/stratos/blob/1c21daae/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java index dd83d9a..7b91205 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/GroupMonitor.java @@ -28,8 +28,8 @@ import org.apache.stratos.autoscaler.context.InstanceContext; import org.apache.stratos.autoscaler.context.group.GroupInstanceContext; import org.apache.stratos.autoscaler.context.partition.GroupLevelPartitionContext; import org.apache.stratos.autoscaler.context.partition.PartitionContext; -import org.apache.stratos.autoscaler.context.partition.network.ParentLevelNetworkPartitionContext; import org.apache.stratos.autoscaler.context.partition.network.NetworkPartitionContext; +import org.apache.stratos.autoscaler.context.partition.network.ParentLevelNetworkPartitionContext; import org.apache.stratos.autoscaler.exception.application.DependencyBuilderException; import org.apache.stratos.autoscaler.exception.application.MonitorNotFoundException; import org.apache.stratos.autoscaler.exception.application.TopologyInConsistentException; @@ -159,7 +159,7 @@ public class GroupMonitor extends ParentComponentMonitor { = (ParentLevelNetworkPartitionContext) networkPartitionContext; Collection<Instance> parentInstances = parent.getInstances(); - for(Instance parentInstance : parentInstances) { + for (Instance parentInstance : parentInstances) { int nonTerminatedInstancesCount = parentLevelNetworkPartitionContext. getNonTerminatedInstancesCount(parentInstance.getInstanceId()); int minInstances = parentLevelNetworkPartitionContext. @@ -183,7 +183,7 @@ public class GroupMonitor extends ParentComponentMonitor { getActiveInstancesCount(); if (activeAppInstances > 0) { //Creating new group instance based on the existing parent instances - if(log.isDebugEnabled()) { + if (log.isDebugEnabled()) { log.debug("Creating a group instance of [application] " + appId + " [group] " + id + " as the the minimum required instances not met"); @@ -200,13 +200,13 @@ public class GroupMonitor extends ParentComponentMonitor { if (activeInstances > maxInstances) { int instancesToBeTerminated = activeInstances - maxInstances; List<InstanceContext> contexts = - ((ParentLevelNetworkPartitionContext)networkPartitionContext). - getInstanceIdToInstanceContextMap(parentInstance.getInstanceId()); + ((ParentLevelNetworkPartitionContext) networkPartitionContext). + getInstanceIdToInstanceContextMap(parentInstance.getInstanceId()); List<InstanceContext> contextList = new ArrayList<InstanceContext>(contexts); for (int i = 0; i < instancesToBeTerminated; i++) { InstanceContext instanceContext = contextList.get(i); //scale down only when extra instances found - if(log.isDebugEnabled()) { + if (log.isDebugEnabled()) { log.debug("Terminating a group instance of [application] " + appId + " [group] " + id + " as it exceeded the " + "maximum no of instances by " + instancesToBeTerminated); @@ -427,22 +427,18 @@ public class GroupMonitor extends ParentComponentMonitor { // then have to use parent instance Id when notifying the parent // as parent doesn't aware if more than one group instances are there if (this.isGroupScalingEnabled() || minGroupInstances > 1 || maxGroupInstances > 1) { - try { - ApplicationHolder.acquireReadLock(); - Application application = ApplicationHolder.getApplications(). - getApplication(this.appId); - if (application != null) { - //Notifying the parent using parent's instance Id, - // as it has group scaling enabled. Notify parent - log.info("[Group] " + this.id + " is notifying the [parent] " + - this.parent.getId() + " [instance] " + parentInstanceId); - MonitorStatusEventBuilder.handleGroupStatusEvent(this.parent, - status, this.id, parentInstanceId); + Application application = ApplicationHolder.getApplications(). + getApplication(this.appId); + if (application != null) { + //Notifying the parent using parent's instance Id, + // as it has group scaling enabled. Notify parent + log.info("[Group] " + this.id + " is notifying the [parent] " + + this.parent.getId() + " [instance] " + parentInstanceId); + MonitorStatusEventBuilder.handleGroupStatusEvent(this.parent, + status, this.id, parentInstanceId); - } - } finally { - ApplicationHolder.releaseReadLock(); } + } else { // notifying the parent log.info("[Group] " + this.id + " is notifying the [parent] " @@ -463,36 +459,43 @@ public class GroupMonitor extends ParentComponentMonitor { } @Override - public void onChildStatusEvent(MonitorStatusEvent statusEvent) { - String childId = statusEvent.getId(); - String instanceId = statusEvent.getInstanceId(); - LifeCycleState status1 = statusEvent.getStatus(); - //Events coming from parent are In_Active(in faulty detection), Scaling events, termination - if (status1 == GroupStatus.Active) { - //Verifying whether all the minimum no of instances of child - // became active to take next action - boolean isChildActive = verifyGroupStatus(childId, instanceId, GroupStatus.Active); - if (isChildActive) { - onChildActivatedEvent(childId, instanceId); - } else { - log.info("Waiting for other group instances to be active"); - } + public void onChildStatusEvent(final MonitorStatusEvent statusEvent) { + Runnable monitoringRunnable = new Runnable() { + @Override + public void run() { + String childId = statusEvent.getId(); + String instanceId = statusEvent.getInstanceId(); + LifeCycleState status1 = statusEvent.getStatus(); + //Events coming from parent are In_Active(in faulty detection), Scaling events, termination + if (status1 == GroupStatus.Active) { + //Verifying whether all the minimum no of instances of child + // became active to take next action + boolean isChildActive = verifyGroupStatus(childId, instanceId, GroupStatus.Active); + if (isChildActive) { + onChildActivatedEvent(childId, instanceId); + } else { + log.info("Waiting for other group instances to be active"); + } - } else if (status1 == ClusterStatus.Active) { - onChildActivatedEvent(childId, instanceId); + } else if (status1 == ClusterStatus.Active) { + onChildActivatedEvent(childId, instanceId); - } else if (status1 == ClusterStatus.Inactive || status1 == GroupStatus.Inactive) { - markInstanceAsInactive(childId, instanceId); - onChildInactiveEvent(childId, instanceId); + } else if (status1 == ClusterStatus.Inactive || status1 == GroupStatus.Inactive) { + markInstanceAsInactive(childId, instanceId); + onChildInactiveEvent(childId, instanceId); - } else if (status1 == ClusterStatus.Terminating || status1 == GroupStatus.Terminating) { - //mark the child monitor as inactive in the map - markInstanceAsTerminating(childId, instanceId); + } else if (status1 == ClusterStatus.Terminating || status1 == GroupStatus.Terminating) { + //mark the child monitor as inactive in the map + markInstanceAsTerminating(childId, instanceId); + + } else if (status1 == ClusterStatus.Terminated || status1 == GroupStatus.Terminated) { + //Act upon child instance termination + onTerminationOfInstance(childId, instanceId); + } + } + }; + executorService.execute(monitoringRunnable); - } else if (status1 == ClusterStatus.Terminated || status1 == GroupStatus.Terminated) { - //Act upon child instance termination - onTerminationOfInstance(childId, instanceId); - } } /** @@ -538,45 +541,52 @@ public class GroupMonitor extends ParentComponentMonitor { @Override - public void onParentStatusEvent(MonitorStatusEvent statusEvent) + public void onParentStatusEvent(final MonitorStatusEvent statusEvent) throws MonitorNotFoundException { - String instanceId = statusEvent.getInstanceId(); - // send the ClusterTerminating event - if (statusEvent.getStatus() == GroupStatus.Terminating || - statusEvent.getStatus() == ApplicationStatus.Terminating) { - //Get all the instances which related to this instanceId - GroupInstance instance = (GroupInstance) this.instanceIdToInstanceMap.get(instanceId); - if (instance != null) { - if (log.isInfoEnabled()) { - log.info(String.format("Publishing Group terminating event for [application] " + - "%s [group] %s [instance] %s", appId, id, instanceId)); - } - ApplicationBuilder.handleGroupTerminatingEvent(appId, id, instanceId); - } else { - //Using parentId need to get the all the children and ask them to terminate - // as they can't exist without the parent - List<String> instanceIds = this.getInstancesByParentInstanceId(instanceId); - if (!instanceIds.isEmpty()) { - for (String instanceId1 : instanceIds) { + Runnable monitoringRunnable = new Runnable() { + @Override + public void run() { + String instanceId = statusEvent.getInstanceId(); + // send the ClusterTerminating event + if (statusEvent.getStatus() == GroupStatus.Terminating || + statusEvent.getStatus() == ApplicationStatus.Terminating) { + //Get all the instances which related to this instanceId + GroupInstance instance = (GroupInstance) instanceIdToInstanceMap.get(instanceId); + if (instance != null) { if (log.isInfoEnabled()) { - log.info(String.format("Publishing Group terminating event for" + - " [application] %s [group] %s [instance] %s", - appId, id, instanceId1)); + log.info(String.format("Publishing Group terminating event for [application] " + + "%s [group] %s [instance] %s", appId, id, instanceId)); + } + ApplicationBuilder.handleGroupTerminatingEvent(appId, id, instanceId); + } else { + //Using parentId need to get the all the children and ask them to terminate + // as they can't exist without the parent + List<String> instanceIds = getInstancesByParentInstanceId(instanceId); + if (!instanceIds.isEmpty()) { + for (String instanceId1 : instanceIds) { + if (log.isInfoEnabled()) { + log.info(String.format("Publishing Group terminating event for" + + " [application] %s [group] %s [instance] %s", + appId, id, instanceId1)); + } + ApplicationBuilder.handleGroupTerminatingEvent(appId, id, instanceId1); + } } - ApplicationBuilder.handleGroupTerminatingEvent(appId, id, instanceId1); + } + } else if (statusEvent.getStatus() == ClusterStatus.Created || + statusEvent.getStatus() == GroupStatus.Created) { + //starting a new instance of this monitor + if (log.isDebugEnabled()) { + log.debug("Creating a [group-instance] for [application] " + appId + " [group] " + + id + " as the parent scaled by group or application bursting"); + } + createInstanceOnDemand(statusEvent.getInstanceId()); } - } - } else if (statusEvent.getStatus() == ClusterStatus.Created || - statusEvent.getStatus() == GroupStatus.Created) { - //starting a new instance of this monitor - if(log.isDebugEnabled()) { - log.debug("Creating a [group-instance] for [application] " + appId + " [group] " + - id + " as the parent scaled by group or application bursting"); - } - createInstanceOnDemand(statusEvent.getInstanceId()); - } + }; + executorService.execute(monitoringRunnable); + } @Override @@ -976,7 +986,7 @@ public class GroupMonitor extends ParentComponentMonitor { < groupMax) { //Check whether group level deployment policy is there String deploymentPolicyId = AutoscalerUtil.getDeploymentPolicyIdByAlias(appId, id); - if(deploymentPolicyId != null) { + if (deploymentPolicyId != null) { // Get partitionContext to create instance in partitionContext = getPartitionContext(parentLevelNetworkPartitionContext, parentPartitionId); @@ -1051,4 +1061,34 @@ public class GroupMonitor extends ParentComponentMonitor { stopScheduler(); } + @Override + public boolean createInstanceOnTermination(String parentInstanceId) { + // Get parent instance context + Instance parentInstanceContext = getParentInstanceContext(parentInstanceId); + + Group group = ApplicationHolder.getApplications(). + getApplication(this.appId).getGroupRecursively(this.id); + + ApplicationMonitor appMonitor = AutoscalerContext.getInstance().getAppMonitor(appId); + + log.info("Starting to Create a group instance of [application] " + appId + " [group] " + + id + " upon termination of an group instance for [parent-instance] " + + parentInstanceId); + // Get existing or create new GroupLevelNetworkPartitionContext + ParentLevelNetworkPartitionContext parentLevelNetworkPartitionContext = + getGroupLevelNetworkPartitionContext(group.getUniqueIdentifier(), + this.appId, parentInstanceContext); + + int groupMin = group.getGroupMinInstances(); + //have to create one more instance + if (parentLevelNetworkPartitionContext.getNonTerminatedInstancesCount(parentInstanceId) + < groupMin) { + if (!appMonitor.isTerminating()) { + createInstanceOnDemand(parentInstanceId); + return true; + } + } + return false; + + } } http://git-wip-us.apache.org/repos/asf/stratos/blob/1c21daae/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java index e4eea04..cbfc1ea 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/component/ParentComponentMonitor.java @@ -115,6 +115,14 @@ public abstract class ParentComponentMonitor extends Monitor { } /** + * This will create Instance on demand as requested by monitors + * + * @param instanceId instance Id of the instance to be created + * @return whether it is created or not + */ + public abstract boolean createInstanceOnTermination(String instanceId); + + /** * Starting the scheduler for the monitor */ public void startScheduler() { @@ -199,7 +207,11 @@ public abstract class ParentComponentMonitor extends Monitor { //starting a new instance of the child Monitor monitor = aliasToActiveChildMonitorsMap.get(context.getId()); //Creating the new instance - monitor.createInstanceOnDemand(instanceId); + if(monitor instanceof ParentComponentMonitor) { + ((ParentComponentMonitor) monitor).createInstanceOnTermination(instanceId); + } else { + monitor.createInstanceOnDemand(instanceId); + } } } }
