Repository: aurora Updated Branches: refs/heads/master 3cbff4117 -> 9ad153f0f
Fixing resource slot counting. Reviewed at https://reviews.apache.org/r/47561/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9ad153f0 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9ad153f0 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9ad153f0 Branch: refs/heads/master Commit: 9ad153f0f474649669b0c9fa2debd516a9a4db5a Parents: 3cbff41 Author: Maxim Khutornenko <[email protected]> Authored: Thu May 19 08:27:00 2016 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Thu May 19 08:27:00 2016 -0700 ---------------------------------------------------------------------- .../org/apache/aurora/benchmark/Offers.java | 4 +-- .../aurora/benchmark/SchedulingBenchmarks.java | 3 +++ .../java/org/apache/aurora/benchmark/Tasks.java | 10 +++++++ .../aurora/scheduler/resources/ResourceBag.java | 28 +++++++++++++------- .../scheduler/resources/ResourceBagTest.java | 17 +++++++++--- .../scheduler/stats/SlotSizeCounterTest.java | 22 +++++++++++++++ 6 files changed, 70 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/Offers.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/Offers.java b/src/jmh/java/org/apache/aurora/benchmark/Offers.java index 1dabe15..144f47e 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/Offers.java +++ b/src/jmh/java/org/apache/aurora/benchmark/Offers.java @@ -99,8 +99,8 @@ final class Offers { Protos.Offer offer = Protos.Offer.newBuilder() .addAllResources(ImmutableSet.of( makeScalar(CPUS, cpu), - makeScalar(RAM_MB, ram.getValue()), - makeScalar(DISK_MB, disk.getValue()), + makeScalar(RAM_MB, ram.as(Data.MB)), + makeScalar(DISK_MB, disk.as(Data.MB)), makeRange( PORTS, IntStream.range(1, ports).boxed().collect(Collectors.toSet())))) http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java index 1aa484c..65f5edc 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java +++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java @@ -33,6 +33,7 @@ import org.apache.aurora.common.quantity.Time; import org.apache.aurora.common.stats.StatsProvider; import org.apache.aurora.common.util.Clock; import org.apache.aurora.common.util.testing.FakeClock; +import org.apache.aurora.gen.ServerInfo; import org.apache.aurora.scheduler.HostOffer; import org.apache.aurora.scheduler.TaskIdGenerator; import org.apache.aurora.scheduler.TierModule; @@ -60,6 +61,7 @@ import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; import org.apache.aurora.scheduler.storage.db.DbUtil; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; +import org.apache.aurora.scheduler.storage.entities.IServerInfo; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -157,6 +159,7 @@ public class SchedulingBenchmarks { bind(Clock.class).toInstance(clock); bind(StatsProvider.class).toInstance(new FakeStatsProvider()); bind(EventSink.class).toInstance(eventBus::post); + bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo("jmh", ""))); } } ); http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/jmh/java/org/apache/aurora/benchmark/Tasks.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/Tasks.java b/src/jmh/java/org/apache/aurora/benchmark/Tasks.java index 4c4b03d..e548a09 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/Tasks.java +++ b/src/jmh/java/org/apache/aurora/benchmark/Tasks.java @@ -15,6 +15,7 @@ package org.apache.aurora.benchmark; import java.util.Set; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -32,6 +33,10 @@ import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; +import static org.apache.aurora.gen.Resource.diskMb; +import static org.apache.aurora.gen.Resource.numCpus; +import static org.apache.aurora.gen.Resource.ramMb; + /** * Task factory. */ @@ -140,12 +145,17 @@ final class Tasks { builder.getAssignedTask() .setInstanceId(i) .setTaskId(taskId); + builder.getAssignedTask().setAssignedPorts(ImmutableMap.of()); builder.getAssignedTask().getTask() .setConstraints(constraints.build()) .setNumCpus(cpu) .setRamMb(ram.as(Data.MB)) .setDiskMb(disk.as(Data.MB)) .setProduction(isProduction) + .setResources(ImmutableSet.of( + numCpus(cpu), + ramMb(ram.as(Data.MB)), + diskMb(disk.as(Data.MB)))) .setRequestedPorts(ImmutableSet.of()); tasks.add(IScheduledTask.build(builder)); } http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java b/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java index 3cff946..211f1fc 100644 --- a/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java +++ b/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java @@ -15,12 +15,14 @@ package org.apache.aurora.scheduler.resources; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BinaryOperator; import java.util.function.Predicate; import java.util.stream.Stream; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import static java.util.stream.Collectors.toMap; @@ -132,7 +134,9 @@ public class ResourceBag { /** * Divides this by other bag contents. - * + * <p> + * Note: any missing or zero resource values in {@code other} will result in + * {@code java.lang.Double.POSITIVE_INFINITY}. * @param other Other bag to divide by. * @return Result of division. */ @@ -173,16 +177,22 @@ public class ResourceBag { .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); } + /** + * Applies {@code operator} to this and {@code other} resource values. Any missing resource type + * values on either side will get substituted with 0.0 before applying the operator. + * + * @param other Other ResourceBag. + * @param operator Operator to apply. + * @return Operation result. + */ private ResourceBag binaryOp(ResourceBag other, BinaryOperator<Double> operator) { ImmutableMap.Builder<ResourceType, Double> builder = ImmutableMap.builder(); - for (Map.Entry<ResourceType, Double> entry : resourceVectors.entrySet()) { - // Apply binary operator only on matching keys from the other. If there is no match, keep the - // current value unchanged. - builder.put( - entry.getKey(), - other.getResourceVectors().containsKey(entry.getKey()) - ? operator.apply(entry.getValue(), other.getResourceVectors().get(entry.getKey())) - : entry.getValue()); + Set<ResourceType> resourceTypes = + Sets.union(resourceVectors.keySet(), other.getResourceVectors().keySet()); + for (ResourceType type : resourceTypes) { + Double left = resourceVectors.getOrDefault(type, 0.0); + Double right = other.getResourceVectors().getOrDefault(type, 0.0); + builder.put(type, operator.apply(left, right)); } return new ResourceBag(builder.build()); http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java b/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java index b8e6474..c163826 100644 --- a/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java +++ b/src/test/java/org/apache/aurora/scheduler/resources/ResourceBagTest.java @@ -17,6 +17,8 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; +import static java.lang.Double.POSITIVE_INFINITY; + import static org.apache.aurora.scheduler.resources.ResourceBag.IS_NEGATIVE; import static org.apache.aurora.scheduler.resources.ResourceBag.LARGE; import static org.apache.aurora.scheduler.resources.ResourceBag.MEDIUM; @@ -58,12 +60,21 @@ public class ResourceBagTest { @Test public void testKeyMismatch() { assertEquals( - new ResourceBag(ImmutableMap.of(CPUS, 9.0)), + new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)), new ResourceBag(ImmutableMap.of(CPUS, 1.0)).add(LARGE)); + // Check add with mismatched keys is still commutative. + assertEquals( + new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0)), + LARGE.add(new ResourceBag(ImmutableMap.of(CPUS, 1.0)))); + + // Check divide with missing values on the right. assertEquals( - LARGE.add(new ResourceBag(ImmutableMap.of(CPUS, 1.0))), - new ResourceBag(ImmutableMap.of(CPUS, 9.0, RAM_MB, 16384.0, DISK_MB, 32768.0))); + new ResourceBag(ImmutableMap.of( + CPUS, 1.0, + RAM_MB, POSITIVE_INFINITY, + DISK_MB, POSITIVE_INFINITY)), + LARGE.divide(new ResourceBag(ImmutableMap.of(CPUS, 8.0)))); } @Test http://git-wip-us.apache.org/repos/asf/aurora/blob/9ad153f0/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java b/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java index e529e67..6c532e6 100644 --- a/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java +++ b/src/test/java/org/apache/aurora/scheduler/stats/SlotSizeCounterTest.java @@ -142,6 +142,28 @@ public class SlotSizeCounterTest extends EasyMockTest { } @Test + public void testMissingResourceVector() { + expectStatExport(); + expectGetSlots( + new MachineResource( + ResourceTestUtil.bag(ImmutableMap.of(RAM_MB, 65536.0, DISK_MB, 65536.0)), + false, + false)); + + control.replay(); + + slotCounter.run(); + assertEquals(0, smallCounter.get()); + assertEquals(0, smallDedicatedCounter.get()); + assertEquals(0, smallRevocableCounter.get()); + assertEquals(0, smallDedicatedRevocableCounter.get()); + assertEquals(0, largeCounter.get()); + assertEquals(0, largeDedicatedCounter.get()); + assertEquals(0, largeRevocableCounter.get()); + assertEquals(0, largeDedicatedRevocableCounter.get()); + } + + @Test public void testCountSlots() { expectStatExport(); expectGetSlots(
