adding config storm.cgroup.memory.limit.tolerance.margin.mb
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/42ce11ee Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/42ce11ee Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/42ce11ee Branch: refs/heads/master Commit: 42ce11ee1135c13f3efd075bb2d42f9ce995bc23 Parents: 0f77402 Author: Boyang Jerry Peng <[email protected]> Authored: Fri Feb 12 12:41:37 2016 -0600 Committer: Boyang Jerry Peng <[email protected]> Committed: Fri Feb 12 12:41:37 2016 -0600 ---------------------------------------------------------------------- conf/defaults.yaml | 3 ++- storm-core/src/clj/org/apache/storm/daemon/supervisor.clj | 6 ++++-- storm-core/src/jvm/org/apache/storm/Config.java | 5 +++++ 3 files changed, 11 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/42ce11ee/conf/defaults.yaml ---------------------------------------------------------------------- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index b88d478..166b249 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -263,7 +263,7 @@ topology.state.checkpoint.interval.ms: 1000 # topology priority describing the importance of the topology in decreasing importance starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases). # Recommended range of 0-29 but no hard limit set. topology.priority: 29 -topology.component.resources.onheap.memory.mb: 256.0 +topology.component.resources.onheap.memory.mb: 128.0 topology.component.resources.offheap.memory.mb: 0.0 topology.component.cpu.pcore.percent: 10.0 topology.worker.max.heap.size.mb: 768.0 @@ -298,3 +298,4 @@ storm.cgroup.hierarchy.name: "storm" # Also determines whether the unit tests for cgroup runs. If cgroup.enable is set to false the unit tests for cgroups will not run storm.supervisor.cgroup.rootdir: "storm" storm.cgroup.cgexec.cmd: "/bin/cgexec" +storm.cgroup.memory.limit.tolerance.margin.mb: 128.0 http://git-wip-us.apache.org/repos/asf/storm/blob/42ce11ee/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj index cb6bafc..dd29afe 100644 --- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj @@ -1087,7 +1087,9 @@ (Utils/addToClasspath topo-classpath)) top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS) - mem-onheap (int (Math/ceil (.get_mem_on_heap resources))) + mem-onheap (if (and (.get_mem_on_heap resources) (> (.get_mem_on_heap resources) 0)) ;; not nil and not zero + (int (Math/ceil (.get_mem_on_heap resources))) ;; round up + (storm-conf WORKER-HEAP-MEMORY-MB)) ;; otherwise use default value mem-offheap (int (Math/ceil (.get_mem_off_heap resources))) @@ -1160,7 +1162,7 @@ command (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE) (do (.reserveResourcesForWorker (:resource-isolation-manager supervisor) worker-id - {"cpu" cpu "memory" (+ mem-onheap mem-offheap)}) + {"cpu" cpu "memory" (+ mem-onheap mem-offheap (int (Math/ceil (conf STORM-CGROUP-MEMORY-MB-LIMIT-TOLERANCE-MARGIN))))}) (.getLaunchCommand (:resource-isolation-manager supervisor) worker-id (java.util.ArrayList. (java.util.Arrays/asList (to-array command))))) command)] http://git-wip-us.apache.org/repos/asf/storm/blob/42ce11ee/storm-core/src/jvm/org/apache/storm/Config.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java index ebe435c..a4d3409 100644 --- a/storm-core/src/jvm/org/apache/storm/Config.java +++ b/storm-core/src/jvm/org/apache/storm/Config.java @@ -2256,6 +2256,11 @@ public class Config extends HashMap<String, Object> { @isString public static String STORM_CGROUP_CGEXEC_CMD = "storm.cgroup.cgexec.cmd"; + /** + * The amount of memory a worker can exceed its allocation before cgroup will kill it + */ + @isPositiveNumber + public static String STORM_CGROUP_MEMORY_MB_LIMIT_TOLERANCE_MARGIN = "storm.cgroup.memory.limit.tolerance.margin.mb"; public static void setClasspath(Map conf, String cp) { conf.put(Config.TOPOLOGY_CLASSPATH, cp);
