markusthoemmes closed pull request #3165: Throttle log messages when
rescheduling containers (update)
URL: https://github.com/apache/incubator-openwhisk/pull/3165
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
index 61172ddd7a..3e83fcac5a 100644
--- a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
+++ b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
@@ -18,13 +18,11 @@
package whisk.core.containerpool
import scala.collection.immutable
-
import akka.actor.Actor
import akka.actor.ActorRef
import akka.actor.ActorRefFactory
import akka.actor.Props
import whisk.common.AkkaLogging
-
import whisk.common.TransactionId
import whisk.core.entity.ByteSize
import whisk.core.entity.CodeExec
@@ -33,6 +31,8 @@ import whisk.core.entity.ExecutableWhiskAction
import whisk.core.entity.size._
import whisk.core.connector.MessageFeed
+import scala.concurrent.duration._
+
sealed trait WorkerState
case object Busy extends WorkerState
case object Free extends WorkerState
@@ -71,6 +71,7 @@ class ContainerPool(childFactory: ActorRefFactory => ActorRef,
var freePool = immutable.Map.empty[ActorRef, ContainerData]
var busyPool = immutable.Map.empty[ActorRef, ContainerData]
var prewarmedPool = immutable.Map.empty[ActorRef, ContainerData]
+ val logMessageInterval = 10.seconds
prewarmConfig.foreach { config =>
logging.info(this, s"pre-warming ${config.count} ${config.exec.kind}
containers")(TransactionId.invokerWarmup)
@@ -117,8 +118,18 @@ class ContainerPool(childFactory: ActorRefFactory =>
ActorRef,
// this can also happen if createContainer fails to start a new
container, or
// if a job is rescheduled but the container it was allocated to has
not yet destroyed itself
// (and a new container would over commit the pool)
- logging.error(this, "Rescheduling Run message, too many message in
the pool")(r.msg.transid)
- self ! r
+ val isErrorLogged =
r.retryLogDeadline.map(_.isOverdue).getOrElse(true)
+ val retryLogDeadline = if (isErrorLogged) {
+ logging.error(
+ this,
+ s"Rescheduling Run message, too many message in the pool,
freePoolSize: ${freePool.size}, " +
+ s"busyPoolSize: ${busyPool.size}, maxActiveContainers
$maxActiveContainers, " +
+ s"userNamespace: ${r.msg.user.namespace}, action:
${r.action}")(r.msg.transid)
+ Some(logMessageInterval.fromNow)
+ } else {
+ r.retryLogDeadline
+ }
+ self ! Run(r.action, r.msg, retryLogDeadline)
}
// Container is free to take more work
diff --git
a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
index c3f89c59e3..93d4798ff4 100644
--- a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
+++ b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
@@ -61,7 +61,7 @@ case class WarmedData(container: Container,
// Events received by the actor
case class Start(exec: CodeExec[_], memoryLimit: ByteSize)
-case class Run(action: ExecutableWhiskAction, msg: ActivationMessage)
+case class Run(action: ExecutableWhiskAction, msg: ActivationMessage,
retryLogDeadline: Option[Deadline] = None)
case object Remove
// Events sent by the actor
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
