Updated Branches: refs/heads/master 503a98fad -> 9c2f669a5
Allowed specification of mininum slot count. To ensure Hadoop jobs begin promptly, we can specify a minimum number of 'hot slots' to be available for use. This addresses the TaskTracker spin up delay that exists with Hadoop on Mesos. This can be a nuisance with lower latency applications, such as ad-hoc Hive queries. Review: https://reviews.apache.org/r/12921 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9c2f669a Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9c2f669a Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9c2f669a Branch: refs/heads/master Commit: 9c2f669a582a708d433937ec1d06687b3bb86609 Parents: 503a98f Author: Brenden Matthews <[email protected]> Authored: Mon Jun 17 09:38:43 2013 -0700 Committer: Brenden Matthews <[email protected]> Committed: Thu Aug 1 18:31:32 2013 -0700 ---------------------------------------------------------------------- .../apache/hadoop/mapred/MesosScheduler.java | 21 ++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/9c2f669a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java index 1923d49..027389d 100644 --- a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java +++ b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java @@ -375,9 +375,22 @@ public class MesosScheduler extends TaskScheduler implements Scheduler { } } + // To ensure Hadoop jobs begin promptly, we can specify a minimum number + // of 'hot slots' to be available for use. This addresses the + // TaskTracker spin up delay that exists with Hadoop on Mesos. This can + // be a nuisance with lower latency applications, such as ad-hoc Hive + // queries. + int minimumMapSlots = conf.getInt("mapred.mesos.total.map.slots.minimum", 0); + int minimumReduceSlots = + conf.getInt("mapred.mesos.total.reduce.slots.minimum", 0); + // Compute how many slots we need to allocate. - int neededMapSlots = Math.max(0, pendingMaps - (idleMapSlots + inactiveMapSlots)); - int neededReduceSlots = Math.max(0, pendingReduces - (idleReduceSlots + inactiveReduceSlots)); + int neededMapSlots = Math.max( + minimumMapSlots - (idleMapSlots + inactiveMapSlots), + pendingMaps - (idleMapSlots + inactiveMapSlots)); + int neededReduceSlots = Math.max( + minimumReduceSlots - (idleReduceSlots + inactiveReduceSlots), + pendingReduces - (idleReduceSlots + inactiveReduceSlots)); LOG.info(join("\n", Arrays.asList( "JobTracker Status", @@ -403,6 +416,10 @@ public class MesosScheduler extends TaskScheduler implements Scheduler { continue; } + // Ensure these values aren't < 0. + neededMapSlots = Math.max(0, neededMapSlots); + neededReduceSlots = Math.max(0, neededReduceSlots); + double cpus = -1.0; double mem = -1.0; double disk = -1.0;
