rabbah closed pull request #3148: Make memory limits as config variables
URL: https://github.com/apache/incubator-openwhisk/pull/3148
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ansible/roles/controller/tasks/deploy.yml
b/ansible/roles/controller/tasks/deploy.yml
index 5f9687ef7b..2400bf87bd 100644
--- a/ansible/roles/controller/tasks/deploy.yml
+++ b/ansible/roles/controller/tasks/deploy.yml
@@ -82,6 +82,10 @@
"LIMITS_TRIGGERS_FIRES_PERMINUTE": "{{ limits.firesPerMinute }}"
"LIMITS_ACTIONS_SEQUENCE_MAXLENGTH": "{{ limits.sequenceMaxLength }}"
+ "CONFIG_whisk_memory_min": "{{ limit_action_memory_min | default() }}"
+ "CONFIG_whisk_memory_max": "{{ limit_action_memory_max | default() }}"
+ "CONFIG_whisk_memory_std": "{{ limit_action_memory_std | default() }}"
+
"RUNTIMES_MANIFEST": "{{ runtimesManifest | to_json }}"
"CONTROLLER_LOCALBOOKKEEPING": "{{ controller.localBookkeeping }}"
"AKKA_CLUSTER_SEED_NODES": "{{seed_nodes_list | join(' ') }}"
diff --git a/ansible/roles/invoker/tasks/deploy.yml
b/ansible/roles/invoker/tasks/deploy.yml
index 3da0b30ef2..7199125669 100644
--- a/ansible/roles/invoker/tasks/deploy.yml
+++ b/ansible/roles/invoker/tasks/deploy.yml
@@ -155,6 +155,9 @@
-e CONFIG_kamon_statsd_port='{{ metrics.kamon.port }}'
-e CONFIG_whisk_spi_LogStoreProvider='{{ userLogs.spi }}'
-e CONFIG_logback_log_level='{{ invoker.loglevel }}'
+ -e CONFIG_whisk_memory_min='{{ limit_action_memory_min | default() }}'
+ -e CONFIG_whisk_memory_max='{{ limit_action_memory_max | default() }}'
+ -e CONFIG_whisk_memory_std='{{ limit_action_memory_std | default() }}'
-v /sys/fs/cgroup:/sys/fs/cgroup
-v /run/runc:/run/runc
-v {{ whisk_logs_dir }}/invoker{{
groups['invokers'].index(inventory_hostname) }}:/logs
diff --git a/common/scala/src/main/resources/application.conf
b/common/scala/src/main/resources/application.conf
index 3ba04c845d..5556db6646 100644
--- a/common/scala/src/main/resources/application.conf
+++ b/common/scala/src/main/resources/application.conf
@@ -87,4 +87,11 @@ whisk {
stride = 1
stride = ${?CONTROLLER_INSTANCES}
}
+
+ # action memory configuration
+ memory {
+ min = "128M"
+ max = "512M"
+ std = "256M"
+ }
}
diff --git a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala
b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala
index c9fc5031bd..0967c07b9f 100644
--- a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala
+++ b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala
@@ -243,6 +243,8 @@ object ConfigKeys {
val kafka = "whisk.kafka"
val kafkaTopics = s"$kafka.topics"
+ val memory = "whisk.memory"
+
val db = "whisk.db"
val docker = "whisk.docker"
diff --git a/common/scala/src/main/scala/whisk/core/entity/Limits.scala
b/common/scala/src/main/scala/whisk/core/entity/Limits.scala
index 81a9902827..a2294e0d3d 100644
--- a/common/scala/src/main/scala/whisk/core/entity/Limits.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/Limits.scala
@@ -37,7 +37,7 @@ protected[entity] abstract class Limits {
* Limits on a specific action. Includes the following properties
* {
* timeout: maximum duration in msecs an action is allowed to consume in
[100 msecs, 5 minutes],
- * memory: maximum memory in megabytes an action is allowed to consume in
[128 MB, 512 MB],
+ * memory: maximum memory in megabytes an action is allowed to consume
within system limit, default [128 MB, 512 MB],
* logs: maximum logs line in megabytes an action is allowed to generate [10
MB]
* }
*
diff --git a/common/scala/src/main/scala/whisk/core/entity/MemoryLimit.scala
b/common/scala/src/main/scala/whisk/core/entity/MemoryLimit.scala
index a1c251740a..929bb166c3 100644
--- a/common/scala/src/main/scala/whisk/core/entity/MemoryLimit.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/MemoryLimit.scala
@@ -27,10 +27,14 @@ import spray.json.JsValue
import spray.json.RootJsonFormat
import spray.json.deserializationError
import whisk.core.entity.size.SizeInt
+import whisk.core.ConfigKeys
+import pureconfig._
+
+case class MemoryLimitConfig(min: String, max: String, std: String)
/**
- * MemoyLimit encapsulates allowed memory for an action. The limit must be
within a
- * permissible range (currently [128MB, 512MB]).
+ * MemoryLimit encapsulates allowed memory for an action. The limit must be
within a
+ * permissible range (by default [128MB, 512MB]).
*
* It is a value type (hence == is .equals, immutable and cannot be assigned
null).
* The constructor is private so that argument requirements are checked and
normalized
@@ -41,12 +45,14 @@ import whisk.core.entity.size.SizeInt
protected[entity] class MemoryLimit private (val megabytes: Int) extends
AnyVal {}
protected[core] object MemoryLimit extends ArgNormalizer[MemoryLimit] {
- protected[core] val MIN_MEMORY = 128 MB
- protected[core] val MAX_MEMORY = 512 MB
- protected[core] val STD_MEMORY = 256 MB
+ private val memoryConfig =
loadConfigOrThrow[MemoryLimitConfig](ConfigKeys.memory)
+
+ protected[core] val minMemory = ByteSize.fromString(memoryConfig.min)
+ protected[core] val maxMemory = ByteSize.fromString(memoryConfig.max)
+ protected[core] val stdMemory = ByteSize.fromString(memoryConfig.std)
- /** Gets TimeLimit with default duration */
- protected[core] def apply(): MemoryLimit = MemoryLimit(STD_MEMORY)
+ /** Gets MemoryLimit with default value */
+ protected[core] def apply(): MemoryLimit = MemoryLimit(stdMemory)
/**
* Creates MemoryLimit for limit, iff limit is within permissible range.
@@ -57,8 +63,8 @@ protected[core] object MemoryLimit extends
ArgNormalizer[MemoryLimit] {
*/
@throws[IllegalArgumentException]
protected[core] def apply(megabytes: ByteSize): MemoryLimit = {
- require(megabytes >= MIN_MEMORY, s"memory $megabytes below allowed
threshold of $MIN_MEMORY")
- require(megabytes <= MAX_MEMORY, s"memory $megabytes exceeds allowed
threshold of $MAX_MEMORY")
+ require(megabytes >= minMemory, s"memory $megabytes below allowed
threshold of $minMemory")
+ require(megabytes <= maxMemory, s"memory $megabytes exceeds allowed
threshold of $maxMemory")
new MemoryLimit(megabytes.toMB.toInt);
}
diff --git a/tests/src/test/scala/whisk/core/cli/test/WskBasicUsageTests.scala
b/tests/src/test/scala/whisk/core/cli/test/WskBasicUsageTests.scala
index 963eb4c67c..a2ed63420d 100644
--- a/tests/src/test/scala/whisk/core/cli/test/WskBasicUsageTests.scala
+++ b/tests/src/test/scala/whisk/core/cli/test/WskBasicUsageTests.scala
@@ -717,7 +717,7 @@ class WskBasicUsageTests extends TestHelpers with
WskTestHelpers {
// Limits to assert, standard values if CLI omits certain values
val limits = JsObject(
"timeout" -> timeout.getOrElse(STD_DURATION).toMillis.toJson,
- "memory" -> memory.getOrElse(STD_MEMORY).toMB.toInt.toJson,
+ "memory" -> memory.getOrElse(stdMemory).toMB.toInt.toJson,
"logs" -> logs.getOrElse(STD_LOGSIZE).toMB.toInt.toJson)
val name = "ActionLimitTests" + Instant.now.toEpochMilli
@@ -747,7 +747,7 @@ class WskBasicUsageTests extends TestHelpers with
WskTestHelpers {
// Assert for valid permutations that the values are set correctly
for {
time <- Seq(None, Some(MIN_DURATION), Some(MAX_DURATION))
- mem <- Seq(None, Some(MIN_MEMORY), Some(MAX_MEMORY))
+ mem <- Seq(None, Some(minMemory), Some(maxMemory))
log <- Seq(None, Some(MIN_LOGSIZE), Some(MAX_LOGSIZE))
} testLimit(time, mem, log)
diff --git a/tests/src/test/scala/whisk/core/entity/test/SchemaTests.scala
b/tests/src/test/scala/whisk/core/entity/test/SchemaTests.scala
index 504db38c0e..ffe78cd0d2 100644
--- a/tests/src/test/scala/whisk/core/entity/test/SchemaTests.scala
+++ b/tests/src/test/scala/whisk/core/entity/test/SchemaTests.scala
@@ -601,16 +601,16 @@ class SchemaTests extends FlatSpec with BeforeAndAfter
with ExecHelpers with Mat
val json = Seq[JsValue](
JsObject(
"timeout" -> TimeLimit.STD_DURATION.toMillis.toInt.toJson,
- "memory" -> MemoryLimit.STD_MEMORY.toMB.toInt.toJson,
+ "memory" -> MemoryLimit.stdMemory.toMB.toInt.toJson,
"logs" -> LogLimit.STD_LOGSIZE.toMB.toInt.toJson),
JsObject(
"timeout" -> TimeLimit.STD_DURATION.toMillis.toInt.toJson,
- "memory" -> MemoryLimit.STD_MEMORY.toMB.toInt.toJson,
+ "memory" -> MemoryLimit.stdMemory.toMB.toInt.toJson,
"logs" -> LogLimit.STD_LOGSIZE.toMB.toInt.toJson,
"foo" -> "bar".toJson),
JsObject(
"timeout" -> TimeLimit.STD_DURATION.toMillis.toInt.toJson,
- "memory" -> MemoryLimit.STD_MEMORY.toMB.toInt.toJson))
+ "memory" -> MemoryLimit.stdMemory.toMB.toInt.toJson))
val limits = json.map(ActionLimits.serdes.read)
assert(limits(0) == ActionLimits())
assert(limits(1) == ActionLimits())
@@ -626,19 +626,19 @@ class SchemaTests extends FlatSpec with BeforeAndAfter
with ExecHelpers with Mat
JsObject(),
JsNull,
JsObject("timeout" -> TimeLimit.STD_DURATION.toMillis.toInt.toJson),
- JsObject("memory" -> MemoryLimit.STD_MEMORY.toMB.toInt.toJson),
+ JsObject("memory" -> MemoryLimit.stdMemory.toMB.toInt.toJson),
JsObject("logs" -> (LogLimit.STD_LOGSIZE.toMB.toInt + 1).toJson),
JsObject(
"TIMEOUT" -> TimeLimit.STD_DURATION.toMillis.toInt.toJson,
- "MEMORY" -> MemoryLimit.STD_MEMORY.toMB.toInt.toJson),
+ "MEMORY" -> MemoryLimit.stdMemory.toMB.toInt.toJson),
JsObject(
"timeout" -> (TimeLimit.STD_DURATION.toMillis.toDouble + .01).toJson,
- "memory" -> (MemoryLimit.STD_MEMORY.toMB.toDouble + .01).toJson),
+ "memory" -> (MemoryLimit.stdMemory.toMB.toDouble + .01).toJson),
JsObject("timeout" -> null, "memory" -> null),
JsObject("timeout" -> JsNull, "memory" -> JsNull),
JsObject(
"timeout" -> TimeLimit.STD_DURATION.toMillis.toString.toJson,
- "memory" -> MemoryLimit.STD_MEMORY.toMB.toInt.toString.toJson))
+ "memory" -> MemoryLimit.stdMemory.toMB.toInt.toString.toJson))
limits.foreach { p =>
a[DeserializationException] should be thrownBy
ActionLimits.serdes.read(p)
@@ -674,7 +674,7 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with
ExecHelpers with Mat
LogLimit())
an[IllegalArgumentException] should be thrownBy ActionLimits(
TimeLimit(),
- MemoryLimit(MemoryLimit.MIN_MEMORY - 1.B),
+ MemoryLimit(MemoryLimit.minMemory - 1.B),
LogLimit())
an[IllegalArgumentException] should be thrownBy ActionLimits(
TimeLimit(),
@@ -687,7 +687,7 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with
ExecHelpers with Mat
LogLimit())
an[IllegalArgumentException] should be thrownBy ActionLimits(
TimeLimit(),
- MemoryLimit(MemoryLimit.MAX_MEMORY + 1.B),
+ MemoryLimit(MemoryLimit.maxMemory + 1.B),
LogLimit())
an[IllegalArgumentException] should be thrownBy ActionLimits(
TimeLimit(),
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
