This is an automated email from the ASF dual-hosted git repository. cbickel pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new 9bb2617 Back-off threshold and choose home invoker as default. (#2417) 9bb2617 is described below commit 9bb26173e71e2c71667810970e900c291d5a8ac3 Author: Markus Thömmes <markusthoem...@me.com> AuthorDate: Mon Jun 26 17:33:09 2017 +0200 Back-off threshold and choose home invoker as default. (#2417) * Back-off threshold and choose home invoker as default. The loadbalancer falls back to picking the home invoker for the action iff all invokers are loaded above a defined threshold. That threshold is backed off 3 times to prevent a "all hell breaks loose" scenario in a sustained high load. --- .../core/loadBalancer/LoadBalancerService.scala | 25 ++++++++++++---------- .../test/LoadBalancerServiceObjectTests.scala | 25 +++++++++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala index 5390a3a..d2e8b64 100644 --- a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala +++ b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala @@ -336,8 +336,8 @@ object LoadBalancerService { /** * Scans through all invokers and searches for an invoker, that has a queue length - * below the defined threshold. Iff no "underloaded" invoker was found it will - * default to the least loaded invoker in the list. + * below the defined threshold. The threshold is subject to a 3 times back off. Iff + * no "underloaded" invoker was found it will default to the home invoker. * * @param availableInvokers a list of available (healthy) invokers to search in * @param activationsPerInvoker a map of the number of outstanding activations per invoker @@ -359,30 +359,33 @@ object LoadBalancerService { val step = stepSizes(hash % stepSizes.size) @tailrec - def search(targetInvoker: Int, seenInvokers: Int): A = { + def search(targetInvoker: Int, iteration: Int = 1): A = { // map the computed index to the actual invoker index val invokerName = availableInvokers(targetInvoker) // send the request to the target invoker if it has capacity... - if (activationsPerInvoker.get(invokerName).getOrElse(0) < invokerBusyThreshold) { + if (activationsPerInvoker.get(invokerName).getOrElse(0) < invokerBusyThreshold * iteration) { invokerName } else { - // ... otherwise look for a less loaded invoker by stepping through a pre computed + // ... otherwise look for a less loaded invoker by stepping through a pre-computed // list of invokers; there are two possible outcomes: // 1. the search lands on a new invoker that has capacity, choose it // 2. walked through the entire list and found no better invoker than the - // "home invoker", choose the least loaded invoker + // "home invoker", force the home invoker val newTarget = (targetInvoker + step) % numInvokers - if (newTarget == homeInvoker || seenInvokers > numInvokers) { - // fall back to the invoker with the least load. - activationsPerInvoker.minBy(_._2)._1 + if (newTarget == homeInvoker) { + if (iteration < 3) { + search(newTarget, iteration + 1) + } else { + availableInvokers(homeInvoker) + } } else { - search(newTarget, seenInvokers + 1) + search(newTarget, iteration) } } } - Some(search(homeInvoker, 0)) + Some(search(homeInvoker)) } else { None } diff --git a/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala b/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala index 4f5433c..9b8bafc 100644 --- a/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala +++ b/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala @@ -112,15 +112,30 @@ class LoadBalancerServiceObjectTests extends FlatSpec with Matchers { 1, hash) shouldBe Some(hashInto(invs, hash + step + step)) } - it should "choose the least loaded invoker if all invokers are overloaded to begin with" in { + it should "multiply its threshold in 3 iterations to find an invoker with a good warm-chance" in { val invokerCount = 3 val invs = invokers(invokerCount) - val hash = 1 + val hash = 0 // home is 0, stepsize is 1 + + // even though invoker1 is not the home invoker in this case, it gets chosen over + // the others because it's the first one encountered by the iteration mechanism to be below + // the threshold of 3 * 16 invocations + LoadBalancerService.schedule( + invs, + Map("invoker0" -> 33, "invoker1" -> 36, "invoker2" -> 33), + 16, + hash) shouldBe Some("invoker0") + } + + it should "choose the home invoker if all invokers are overloaded even above the muliplied threshold" in { + val invokerCount = 3 + val invs = invokers(invokerCount) + val hash = 0 // home is 0, stepsize is 1 LoadBalancerService.schedule( invs, - Map("invoker0" -> 3, "invoker1" -> 3, "invoker2" -> 2), - 1, - hash) shouldBe Some("invoker2") + Map("invoker0" -> 51, "invoker1" -> 50, "invoker2" -> 49), + 16, + hash) shouldBe Some("invoker0") } } -- To stop receiving notification emails like this one, please contact ['"commits@openwhisk.apache.org" <commits@openwhisk.apache.org>'].