Updated Branches: refs/heads/master 1f5a3ad95 -> f7dab5f83
STRATOS-371 and STRATOS-372 Project: http://git-wip-us.apache.org/repos/asf/incubator-stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-stratos/commit/f7dab5f8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-stratos/tree/f7dab5f8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-stratos/diff/f7dab5f8 Branch: refs/heads/master Commit: f7dab5f830a79b4239847f763aa8ead8ca3ec6e1 Parents: 1f5a3ad Author: Lahiru Sandaruwan <[email protected]> Authored: Sat Jan 18 20:21:15 2014 +0530 Committer: Lahiru Sandaruwan <[email protected]> Committed: Sat Jan 18 20:21:15 2014 +0530 ---------------------------------------------------------------------- .../stratos/autoscaler/PartitionContext.java | 12 +- .../autoscaler/algorithm/OneAfterAnother.java | 16 +- .../autoscaler/algorithm/RoundRobin.java | 159 ++++++++++--------- .../autoscaler/rule/RuleTasksDelegator.java | 8 +- .../distribution/src/main/conf/scaling.drl | 9 +- 5 files changed, 119 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-stratos/blob/f7dab5f8/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/PartitionContext.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/PartitionContext.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/PartitionContext.java index 0d80d00..06d115e 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/PartitionContext.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/PartitionContext.java @@ -50,7 +50,9 @@ public class PartitionContext implements Serializable{ private Partition partition; // private int currentActiveMemberCount = 0; private int minimumMemberCount = 0; - + private int pendingMembersFailureCount = 0; + private final int PENDING_MEMBER_FAILURE_THRESHOLD = 5; + // properties private Properties properties; @@ -170,6 +172,7 @@ public class PartitionContext implements Serializable{ iterator.remove(); // add to the activated list this.activeMembers.add(pendingMember); + pendingMembersFailureCount = 0; if (log.isDebugEnabled()) { log.debug(String.format("Pending member is removed and added to the " + "activated member list. [Member Id] %s",memberId)); @@ -380,11 +383,18 @@ public class PartitionContext implements Serializable{ } long pendingTime = System.currentTimeMillis() - pendingMember.getInitTime(); if (pendingTime >= expiryTime) { + + iterator.remove(); log.info("Pending state of member: " + pendingMember.getMemberId() + " is expired. " + "Adding as an obsoleted member."); // member should be terminated ctxt.addObsoleteMember(pendingMember.getMemberId()); + pendingMembersFailureCount++; + if( pendingMembersFailureCount > PENDING_MEMBER_FAILURE_THRESHOLD){ + setExpiryTime(expiryTime * 2);//Doubles the expiry time after the threshold of failure exceeded + //TODO Implement an alerting system: STRATOS-369 + } } } } http://git-wip-us.apache.org/repos/asf/incubator-stratos/blob/f7dab5f8/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/OneAfterAnother.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/OneAfterAnother.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/OneAfterAnother.java index 1fddf77..b3d8d0d 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/OneAfterAnother.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/OneAfterAnother.java @@ -45,14 +45,15 @@ public class OneAfterAnother implements AutoscaleAlgorithm { private static final Log log = LogFactory.getLog(OneAfterAnother.class); public Partition getNextScaleUpPartition(NetworkPartitionContext networkPartitionContext, String clusterId) { + try { + if (log.isDebugEnabled()) + log.debug(String.format("Searching for a partition to up down %s [network partition] %s", + networkPartitionContext.getId())) ; int currentPartitionIndex = networkPartitionContext.getCurrentPartitionIndex(); List<?> partitions = Arrays.asList(networkPartitionContext.getPartitions()); int noOfPartitions = partitions.size(); - if (log.isDebugEnabled()) { - log.debug(String.format("Selecting a partition from 'One After Another' algorithm, " + - "%s partitions in the [network partition]: %s ", noOfPartitions, networkPartitionContext.getId())); - } + for (int i = currentPartitionIndex; i < noOfPartitions; i++) { if (partitions.get(currentPartitionIndex) instanceof Partition) { @@ -88,7 +89,12 @@ public class OneAfterAnother implements AutoscaleAlgorithm { } public Partition getNextScaleDownPartition(NetworkPartitionContext networkPartitionContext, String clusterId) { + try { + + if (log.isDebugEnabled()) + log.debug(String.format("Searching for a partition to scale down %s [network partition] %s", + networkPartitionContext.getId())) ; int currentPartitionIndex = networkPartitionContext.getCurrentPartitionIndex(); List<?> partitions = Arrays.asList(networkPartitionContext.getPartitions()); @@ -110,7 +116,7 @@ public class OneAfterAnother implements AutoscaleAlgorithm { if (currentPartitionIndex == 0) { if (log.isDebugEnabled()) log.debug(String.format("Partition %s reached with no space to scale down," + - "[current] %s [min] %s", currentPartitionId, currentlyActiveMemberCount, + "[active] %s [min] %s", currentPartitionId, currentlyActiveMemberCount, currentPartition.getPartitionMin())); return null; } http://git-wip-us.apache.org/repos/asf/incubator-stratos/blob/f7dab5f8/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/RoundRobin.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/RoundRobin.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/RoundRobin.java index fc2f891..aa9b0bb 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/RoundRobin.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/algorithm/RoundRobin.java @@ -37,97 +37,108 @@ public class RoundRobin implements AutoscaleAlgorithm{ private static final Log log = LogFactory.getLog(RoundRobin.class); public Partition getNextScaleUpPartition(NetworkPartitionContext networkPartitionContext, String clusterId){ + try{ + + if (log.isDebugEnabled()) + log.debug(String.format("Searching for a partition to scale up %s [network partition] %s", + networkPartitionContext.getId())) ; + List<?> partitions = Arrays.asList(networkPartitionContext.getPartitions()); + int noOfPartitions = partitions.size(); + + for(int i=0; i < noOfPartitions; i++) + { + int currentPartitionIndex = networkPartitionContext.getCurrentPartitionIndex(); + if (partitions.get(currentPartitionIndex) instanceof Partition) { + Partition currentPartition = (Partition) partitions.get(currentPartitionIndex); + String currentPartitionId = currentPartition.getId(); + + // point to next partition + int nextPartitionIndex = currentPartitionIndex == noOfPartitions - 1 ? 0 : currentPartitionIndex+1; + networkPartitionContext.setCurrentPartitionIndex(nextPartitionIndex); + int nonTerminatedMemberCountOfPartition = networkPartitionContext.getNonTerminatedMemberCountOfPartition(currentPartitionId); + if(nonTerminatedMemberCountOfPartition < currentPartition.getPartitionMax()){ + // current partition is free + if (log.isDebugEnabled()) + log.debug(String.format("A free space found for scale up in partition %s [current] %s [max] %s", + currentPartitionId, networkPartitionContext.getNonTerminatedMemberCountOfPartition(currentPartitionId), + currentPartition.getPartitionMax())) ; + return currentPartition; + } - List<?> partitions = Arrays.asList(networkPartitionContext.getPartitions()); - int noOfPartitions = partitions.size(); - if(log.isDebugEnabled()){ - log.debug(String.format("Selecting a partition from 'Round Robin' algorithm, " + - "%s partitions in the [network partition]: %s ", noOfPartitions, networkPartitionContext.getId())); - } - for(int i=0; i < noOfPartitions; i++) - { - int currentPartitionIndex = networkPartitionContext.getCurrentPartitionIndex(); - if (partitions.get(currentPartitionIndex) instanceof Partition) { - Partition currentPartition = (Partition) partitions.get(currentPartitionIndex); - String currentPartitionId = currentPartition.getId(); - - // point to next partition - int nextPartitionIndex = currentPartitionIndex == noOfPartitions - 1 ? 0 : currentPartitionIndex+1; - networkPartitionContext.setCurrentPartitionIndex(nextPartitionIndex); - int nonTerminatedMemberCountOfPartition = networkPartitionContext.getNonTerminatedMemberCountOfPartition(currentPartitionId); - if(nonTerminatedMemberCountOfPartition < currentPartition.getPartitionMax()){ - // current partition is free - if (log.isDebugEnabled()) - log.debug(String.format("A free space found for scale up in partition %s [current] %s [max] %s", - currentPartitionId, networkPartitionContext.getNonTerminatedMemberCountOfPartition(currentPartitionId), - currentPartition.getPartitionMax())) ; - return currentPartition; - } - - if(log.isDebugEnabled()) - log.debug("No free space for a new instance in partition " + currentPartition.getId()); + if(log.isDebugEnabled()) + log.debug("No free space for a new instance in partition " + currentPartition.getId()); - } - } + } + } - // none of the partitions were free. - if(log.isDebugEnabled()) { - log.debug("No free partition found at network partition " + networkPartitionContext); - } - return null; + // none of the partitions were free. + if(log.isDebugEnabled()) { + log.debug("No free partition found at network partition " + networkPartitionContext); + } + } catch (Exception e) { + log.error("Could not find next scale up partition", e); + } + return null; } @Override public Partition getNextScaleDownPartition(NetworkPartitionContext networkPartitionContext, String clusterId) { + try{ + if (log.isDebugEnabled()) + log.debug(String.format("Searching for a partition to scale up %s [network partition] %s", + networkPartitionContext.getId())) ; + List<?> partitions = Arrays.asList(networkPartitionContext.getPartitions()); + int noOfPartitions = partitions.size(); + + for (int i = 0; i < noOfPartitions; i++) { + int currentPartitionIndex = networkPartitionContext.getCurrentPartitionIndex(); + // point to next partition + if (currentPartitionIndex == 0) { + + currentPartitionIndex = noOfPartitions - 1; + } else { + + currentPartitionIndex = currentPartitionIndex - 1; + } - List<?> partitions = Arrays.asList(networkPartitionContext.getPartitions()); - int noOfPartitions = partitions.size(); - - for (int i = 0; i < noOfPartitions; i++) { - int currentPartitionIndex = networkPartitionContext.getCurrentPartitionIndex(); - // point to next partition - if (currentPartitionIndex == 0) { - - currentPartitionIndex = noOfPartitions - 1; - } else { - - currentPartitionIndex = currentPartitionIndex - 1; - } - - // Set next partition as current partition in Autoscaler Context - networkPartitionContext.setCurrentPartitionIndex(currentPartitionIndex); - - if (partitions.get(currentPartitionIndex) instanceof Partition) { + // Set next partition as current partition in Autoscaler Context + networkPartitionContext.setCurrentPartitionIndex(currentPartitionIndex); - Partition currentPartition = (Partition) partitions.get(currentPartitionIndex); - String currentPartitionId = currentPartition.getId(); + if (partitions.get(currentPartitionIndex) instanceof Partition) { - // has more than minimum instances. - int currentlyActiveMemberCount = networkPartitionContext.getActiveMemberCount(currentPartitionId); - if (currentlyActiveMemberCount > currentPartition.getPartitionMin()) { - // current partition is free - if (log.isDebugEnabled()) - log.debug(String.format("A free space found for scale down in partition %s [current] %s [min] %s", - currentPartitionId, currentlyActiveMemberCount, currentPartition.getPartitionMin())) ; - return currentPartition; - }else { + Partition currentPartition = (Partition) partitions.get(currentPartitionIndex); + String currentPartitionId = currentPartition.getId(); - if (currentPartitionIndex == 0) { + // has more than minimum instances. + int currentlyActiveMemberCount = networkPartitionContext.getActiveMemberCount(currentPartitionId); + if (currentlyActiveMemberCount > currentPartition.getPartitionMin()) { + // current partition is free if (log.isDebugEnabled()) - log.debug(String.format("Partition %s reached with no space to scale down," + - "[current] %s [min] %s", currentPartitionId, currentlyActiveMemberCount, - currentPartition.getPartitionMin())); - return null; + log.debug(String.format("A free space found for scale down in partition %s [current] %s [min] %s", + currentPartitionId, currentlyActiveMemberCount, currentPartition.getPartitionMin())) ; + return currentPartition; + }else { + + if (currentPartitionIndex == 0) { + if (log.isDebugEnabled()) + log.debug(String.format("Partition %s reached with no space to scale down," + + "[current] %s [min] %s", currentPartitionId, currentlyActiveMemberCount, + currentPartition.getPartitionMin())); + return null; + } } } } - } - if (log.isDebugEnabled()) - log.debug("No partition found for scale down at network partition " + - networkPartitionContext.getId()); - // none of the partitions were free. + if (log.isDebugEnabled()) + log.debug("No partition found for scale down at network partition " + + networkPartitionContext.getId()); + // none of the partitions were free. + + } catch (Exception e) { + log.error("Could not find next scale up partition", e); + } return null; } http://git-wip-us.apache.org/repos/asf/incubator-stratos/blob/f7dab5f8/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/rule/RuleTasksDelegator.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/rule/RuleTasksDelegator.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/rule/RuleTasksDelegator.java index 823b4d6..9f8262d 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/rule/RuleTasksDelegator.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/rule/RuleTasksDelegator.java @@ -74,7 +74,13 @@ public class RuleTasksDelegator { clusterId, lbClusterId, partitionContext.getNetworkPartitionId()); if (memberContext != null) { - partitionContext.addPendingMember(memberContext); + partitionContext.addPendingMember(memberContext); + if(log.isDebugEnabled()){ + log.debug(String.format("Pending member added, [member] %s [partition] %s", memberContext.getMemberId(), + memberContext.getPartition().getId())); + } + } else if(log.isDebugEnabled()){ + log.debug("Returned member context is null, did not add to pending members"); } } catch (Throwable e) { http://git-wip-us.apache.org/repos/asf/incubator-stratos/blob/f7dab5f8/products/autoscaler/modules/distribution/src/main/conf/scaling.drl ---------------------------------------------------------------------- diff --git a/products/autoscaler/modules/distribution/src/main/conf/scaling.drl b/products/autoscaler/modules/distribution/src/main/conf/scaling.drl index 4955e0d..d6c3a78 100644 --- a/products/autoscaler/modules/distribution/src/main/conf/scaling.drl +++ b/products/autoscaler/modules/distribution/src/main/conf/scaling.drl @@ -108,21 +108,21 @@ dialect "mvel" Partition partition = autoscaleAlgorithm.getNextScaleUpPartition($networkPartitionContext, clusterId); if(partition != null){ log.info("[scale-up] Partition available, hence trying to spawn an instance to scale up!" ); - log.debug("[scaling up] " + " [partition] " + partition.getId() + " [cluster] " + clusterId ); + log.debug("[scale-up] " + " [partition] " + partition.getId() + " [cluster] " + clusterId ); $delegator.delegateSpawn($networkPartitionContext.getPartitionCtxt(partition.getId()), clusterId, lbRef); } } else if(scaleDown){ if($networkPartitionContext.getScaleDownRequestsCount() > 5 ){ - + log.debug("[scale-down] Reached scale down requests threshold [cluster] " + clusterId); $networkPartitionContext.resetScaleDownRequestsCount(); MemberStatsContext selectedMemberStatsContext = null; double lowestOverallLoad = 0.0; boolean foundAValue = false; Partition partition = autoscaleAlgorithm.getNextScaleDownPartition($networkPartitionContext, clusterId); if(partition != null){ - log.info("[scaling down] Partition available to scale down "); - log.debug("[scaling down] " + " [partition] " + partition.getId() + " [cluster] " + clusterId); + log.info("[scale-down] Partition available to scale down "); + log.debug("[scale-down] " + " [partition] " + partition.getId() + " [cluster] " + clusterId); partitionContext = $networkPartitionContext.getPartitionCtxt(partition.getId()); for(MemberStatsContext memberStatsContext: partitionContext.getMemberStatsContexts().values()){ @@ -165,6 +165,7 @@ dialect "mvel" } } else{ $networkPartitionContext.increaseScaleDownRequestsCount(); + log.debug("[scale-down] Not reached scale down requests threshold. " + " [cluster] " + clusterId); } } } else{
