Repository: aurora Updated Branches: refs/heads/master e8a61f68c -> 92fc094f8
Resources: moving calculation methods from Resources Bugs closed: AURORA-1415 Reviewed at https://reviews.apache.org/r/37139/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/92fc094f Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/92fc094f Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/92fc094f Branch: refs/heads/master Commit: 92fc094f8267e680fe2dc29d5a42fb56cf8cbbff Parents: e8a61f6 Author: Maxim Khutornenko <[email protected]> Authored: Fri Aug 7 15:36:20 2015 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Fri Aug 7 15:36:20 2015 -0700 ---------------------------------------------------------------------- .../apache/aurora/scheduler/ResourceSlot.java | 79 +++++++++++--- .../scheduler/configuration/Resources.java | 107 ------------------- .../aurora/scheduler/ResourceSlotTest.java | 24 +++++ .../scheduler/configuration/ResourcesTest.java | 46 -------- .../mesos/MesosTaskFactoryImplTest.java | 4 +- 5 files changed, 93 insertions(+), 167 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java b/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java index ecadb3e..eb14ffe 100644 --- a/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java +++ b/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java @@ -18,8 +18,10 @@ import java.util.List; import java.util.Set; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; import com.twitter.common.quantity.Amount; import com.twitter.common.quantity.Data; @@ -65,7 +67,7 @@ public final class ResourceSlot { private static ResourceSlot from(Resources resources, ExecutorSettings executorSettings) { // Apply a flat 'tax' of executor overhead resources to the task. - Resources requiredTaskResources = Resources.sum( + Resources requiredTaskResources = sum( resources, executorSettings.getExecutorOverhead()); @@ -135,18 +137,37 @@ public final class ResourceSlot { } public static ResourceSlot sum(Iterable<ResourceSlot> rs) { - Resources r = Resources.sum(Iterables.transform(rs, new Function<ResourceSlot, Resources>() { - @Override - public Resources apply(ResourceSlot input) { - return input.resources; - } - })); + Resources sum = Resources.NONE; + + for (ResourceSlot r : rs) { + double numCpus = sum.getNumCpus() + r.getNumCpus(); + Amount<Long, Data> disk = + Amount.of(sum.getDisk().as(Data.BYTES) + r.getDisk().as(Data.BYTES), Data.BYTES); + Amount<Long, Data> ram = + Amount.of(sum.getRam().as(Data.BYTES) + r.getRam().as(Data.BYTES), Data.BYTES); + int ports = sum.getNumPorts() + r.getNumPorts(); + sum = new Resources(numCpus, ram, disk, ports); + } + + return new ResourceSlot(sum); + } - return new ResourceSlot(r); + @VisibleForTesting + public static Resources sum(Resources a, Resources b) { + return sum(ImmutableList.of(new ResourceSlot(a), new ResourceSlot(b))).resources; } public static ResourceSlot subtract(ResourceSlot a, Resources b) { - return new ResourceSlot(Resources.subtract(a.resources, b)); + return new ResourceSlot(subtract(a.resources, b)); + } + + @VisibleForTesting + static Resources subtract(Resources a, Resources b) { + return new Resources( + a.getNumCpus() - b.getNumCpus(), + Amount.of(a.getRam().as(Data.MB) - b.getRam().as(Data.MB), Data.MB), + Amount.of(a.getDisk().as(Data.MB) - b.getDisk().as(Data.MB), Data.MB), + a.getNumPorts() - b.getNumPorts()); } public List<Protos.Resource> toResourceList(Set<Integer> selectedPorts) { @@ -156,7 +177,41 @@ public final class ResourceSlot { public static final Ordering<ResourceSlot> ORDER = new Ordering<ResourceSlot>() { @Override public int compare(ResourceSlot left, ResourceSlot right) { - return Resources.RESOURCE_ORDER.compare(left.resources, right.resources); + return RESOURCE_ORDER.compare(left.resources, right.resources); } }; + + /** + * A Resources object is greater than another iff _all_ of its resource components are greater + * or equal. A Resources object compares as equal if some but not all components are greater than + * or equal to the other. + */ + public static final Ordering<Resources> RESOURCE_ORDER = new Ordering<Resources>() { + @Override + public int compare(Resources left, Resources right) { + int diskC = left.getDisk().compareTo(right.getDisk()); + int ramC = left.getRam().compareTo(right.getRam()); + int portC = Integer.compare(left.getNumPorts(), right.getNumPorts()); + int cpuC = Double.compare(left.getNumCpus(), right.getNumCpus()); + + FluentIterable<Integer> vector = + FluentIterable.from(ImmutableList.of(diskC, ramC, portC, cpuC)); + + if (vector.allMatch(IS_ZERO)) { + return 0; + } + + if (vector.filter(Predicates.not(IS_ZERO)).allMatch(e -> e > 0)) { + return 1; + } + + if (vector.filter(Predicates.not(IS_ZERO)).allMatch(e -> e < 0)) { + return -1; + } + + return 0; + } + }; + + private static final Predicate<Integer> IS_ZERO = e -> e == 0; } http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java b/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java index ea7ce24..6db3efb 100644 --- a/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java +++ b/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java @@ -21,15 +21,12 @@ import java.util.Set; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.base.Predicates; import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; -import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.twitter.common.quantity.Amount; import com.twitter.common.quantity.Data; @@ -87,20 +84,6 @@ public class Resources { } /** - * Tests whether this bundle of resources is greater than or equal to another bundle of resources. - * - * @param other Resources being compared to. - * @return {@code true} if all resources in this bundle are greater than or equal to the - * equivalents from {@code other}, otherwise {@code false}. - */ - public boolean greaterThanOrEqual(Resources other) { - return numCpus >= other.numCpus - && disk.as(Data.MB) >= other.disk.as(Data.MB) - && ram.as(Data.MB) >= other.ram.as(Data.MB) - && numPorts >= other.numPorts; - } - - /** * Adapts this resources object to a list of mesos resources. * * @param selectedPorts The ports selected, to be applied as concrete task ranges. @@ -207,43 +190,6 @@ public class Resources { public static final Resources NONE = new Resources(0, Amount.of(0L, Data.BITS), Amount.of(0L, Data.BITS), 0); - /** - * a - b. - */ - public static Resources subtract(Resources a, Resources b) { - return new Resources( - a.getNumCpus() - b.getNumCpus(), - Amount.of(a.getRam().as(Data.MB) - b.getRam().as(Data.MB), Data.MB), - Amount.of(a.getDisk().as(Data.MB) - b.getDisk().as(Data.MB), Data.MB), - a.getNumPorts() - b.getNumPorts()); - } - - /** - * sum(a, b). - */ - public static Resources sum(Resources a, Resources b) { - return sum(ImmutableList.of(a, b)); - } - - /** - * sum(rs). - */ - public static Resources sum(Iterable<Resources> rs) { - Resources sum = NONE; - - for (Resources r : rs) { - double numCpus = sum.getNumCpus() + r.getNumCpus(); - Amount<Long, Data> disk = - Amount.of(sum.getDisk().as(Data.BYTES) + r.getDisk().as(Data.BYTES), Data.BYTES); - Amount<Long, Data> ram = - Amount.of(sum.getRam().as(Data.BYTES) + r.getRam().as(Data.BYTES), Data.BYTES); - int ports = sum.getNumPorts() + r.getNumPorts(); - sum = new Resources(numCpus, ram, disk, ports); - } - - return sum; - } - private static int getNumAvailablePorts(List<Resource> resource) { int offeredPorts = 0; for (Range range : getPortRanges(resource)) { @@ -401,57 +347,4 @@ public class Resources { Collections.shuffle(availablePorts); return ImmutableSet.copyOf(availablePorts.subList(0, numPorts)); } - - /** - * A Resources object is greater than another iff _all_ of its resource components are greater - * or equal. A Resources object compares as equal if some but not all components are greater than - * or equal to the other. - */ - public static final Ordering<Resources> RESOURCE_ORDER = new Ordering<Resources>() { - @Override - public int compare(Resources left, Resources right) { - int diskC = left.getDisk().compareTo(right.getDisk()); - int ramC = left.getRam().compareTo(right.getRam()); - int portC = Integer.compare(left.getNumPorts(), right.getNumPorts()); - int cpuC = Double.compare(left.getNumCpus(), right.getNumCpus()); - - FluentIterable<Integer> vector = - FluentIterable.from(ImmutableList.of(diskC, ramC, portC, cpuC)); - - if (vector.allMatch(IS_ZERO)) { - return 0; - } - - if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_POSITIVE)) { - return 1; - } - - if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_NEGATIVE)) { - return -1; - } - - return 0; - } - }; - - private static final Predicate<Integer> IS_POSITIVE = new Predicate<Integer>() { - @Override - public boolean apply(Integer input) { - return input > 0; - } - }; - - private static final Predicate<Integer> IS_NEGATIVE = new Predicate<Integer>() { - @Override - public boolean apply(Integer input) { - return input < 0; - } - }; - - private static final Predicate<Integer> IS_ZERO = new Predicate<Integer>() { - @Override - public boolean apply(Integer input) { - return input == 0; - } - }; } http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java b/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java index b4dbab7..896833d 100644 --- a/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java +++ b/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java @@ -23,6 +23,15 @@ import static org.junit.Assert.assertEquals; public class ResourceSlotTest { + private static final Resources NEGATIVE_ONE = + new Resources(-1.0, Amount.of(-1L, Data.MB), Amount.of(-1L, Data.MB), -1); + private static final Resources ONE = + new Resources(1.0, Amount.of(1L, Data.MB), Amount.of(1L, Data.MB), 1); + private static final Resources TWO = + new Resources(2.0, Amount.of(2L, Data.MB), Amount.of(2L, Data.MB), 2); + private static final Resources THREE = + new Resources(3.0, Amount.of(3L, Data.MB), Amount.of(3L, Data.MB), 3); + @Test public void testMaxElements() { Resources highRAM = new Resources(1, Amount.of(8L, Data.GB), Amount.of(10L, Data.MB), 0); @@ -34,4 +43,19 @@ public class ResourceSlotTest { assertEquals(result.getDisk(), Amount.of(10L, Data.GB)); assertEquals(result.getNumPorts(), 1); } + + @Test + public void testSubtract() { + assertEquals(ONE, ResourceSlot.subtract(TWO, ONE)); + assertEquals(TWO, ResourceSlot.subtract(THREE, ONE)); + assertEquals(NEGATIVE_ONE, ResourceSlot.subtract(ONE, TWO)); + assertEquals(NEGATIVE_ONE, ResourceSlot.subtract(TWO, THREE)); + } + + @Test + public void testSum() { + assertEquals(TWO, ResourceSlot.sum(ONE, ONE)); + assertEquals(THREE, ResourceSlot.sum(ONE, TWO)); + assertEquals(THREE, ResourceSlot.sum(TWO, ONE)); + } } http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java index 115dc08..9335141 100644 --- a/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java +++ b/src/test/java/org/apache/aurora/scheduler/configuration/ResourcesTest.java @@ -33,9 +33,7 @@ import org.apache.mesos.Protos.Value.Type; import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class ResourcesTest { @@ -87,30 +85,6 @@ public class ResourcesTest { assertEquals(ImmutableSet.of(), Resources.getPorts(createOffer(portsResource), 0)); } - private static final Resources NEGATIVE_ONE = - new Resources(-1.0, Amount.of(-1L, Data.MB), Amount.of(-1L, Data.MB), -1); - private static final Resources ONE = - new Resources(1.0, Amount.of(1L, Data.MB), Amount.of(1L, Data.MB), 1); - private static final Resources TWO = - new Resources(2.0, Amount.of(2L, Data.MB), Amount.of(2L, Data.MB), 2); - private static final Resources THREE = - new Resources(3.0, Amount.of(3L, Data.MB), Amount.of(3L, Data.MB), 3); - - @Test - public void testSum() { - assertEquals(TWO, Resources.sum(ONE, ONE)); - assertEquals(THREE, Resources.sum(ONE, TWO)); - assertEquals(THREE, Resources.sum(TWO, ONE)); - } - - @Test - public void testSubtract() { - assertEquals(ONE, Resources.subtract(TWO, ONE)); - assertEquals(TWO, Resources.subtract(THREE, ONE)); - assertEquals(NEGATIVE_ONE, Resources.subtract(ONE, TWO)); - assertEquals(NEGATIVE_ONE, Resources.subtract(TWO, THREE)); - } - @Test(expected = Resources.InsufficientResourcesException.class) public void testPortRangeScarcity() { Resource portsResource = createPortRange(Pair.of(1, 2)); @@ -177,11 +151,6 @@ public class ResourcesTest { .setDiskMb(2048) .setRequestedPorts(ImmutableSet.of("http", "debug"))); - private static void assertLeftIsLarger(Resources left, Resources right) { - assertTrue(left.greaterThanOrEqual(right)); - assertFalse(right.greaterThanOrEqual(left)); - } - @Test public void testAccessors() { Resources resources = Resources.from(TASK); @@ -189,21 +158,6 @@ public class ResourcesTest { assertEquals(Amount.of(TASK.getRamMb(), Data.MB), resources.getRam()); assertEquals(Amount.of(TASK.getDiskMb(), Data.MB), resources.getDisk()); assertEquals(TASK.getRequestedPorts().size(), resources.getNumPorts()); - - assertTrue(resources.greaterThanOrEqual(resources)); - assertLeftIsLarger( - resources, - Resources.from(ITaskConfig.build(TASK.newBuilder().setNumCpus(0.5)))); - assertLeftIsLarger( - resources, - Resources.from(ITaskConfig.build(TASK.newBuilder().setRamMb(512)))); - assertLeftIsLarger( - resources, - Resources.from(ITaskConfig.build(TASK.newBuilder().setDiskMb(1024)))); - assertLeftIsLarger( - resources, - Resources.from( - ITaskConfig.build(TASK.newBuilder().setRequestedPorts(ImmutableSet.of("http"))))); } @Test http://git-wip-us.apache.org/repos/asf/aurora/blob/92fc094f/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java index 7f18bbc..d19f5af 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java @@ -166,7 +166,7 @@ public class MesosTaskFactoryImplTest { private void checkTaskResources(ITaskConfig task, TaskInfo taskInfo) { assertEquals( - Resources.sum(Resources.from(task), config.getExecutorOverhead()), + ResourceSlot.sum(Resources.from(task), config.getExecutorOverhead()), getTotalTaskResources(taskInfo)); } @@ -238,6 +238,6 @@ public class MesosTaskFactoryImplTest { private static Resources getTotalTaskResources(TaskInfo task) { Resources taskResources = Resources.from(task.getResourcesList()); Resources executorResources = Resources.from(task.getExecutor().getResourcesList()); - return Resources.sum(taskResources, executorResources); + return ResourceSlot.sum(taskResources, executorResources); } }
