Repository: aurora Updated Branches: refs/heads/master 04b6858a0 -> 89da936f3
Adding minimal implementation of the external tier config. Bugs closed: AURORA-1437 Reviewed at https://reviews.apache.org/r/37825/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/89da936f Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/89da936f Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/89da936f Branch: refs/heads/master Commit: 89da936f3d28743e307c7c4fed0bff6fead7ceca Parents: 04b6858 Author: Maxim Khutornenko <[email protected]> Authored: Mon Aug 31 15:18:51 2015 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Mon Aug 31 15:18:51 2015 -0700 ---------------------------------------------------------------------- .../upstart/aurora-scheduler-kerberos.conf | 3 +- examples/vagrant/upstart/aurora-scheduler.conf | 3 +- .../org/apache/aurora/benchmark/Offers.java | 5 ++- .../aurora/scheduler/SchedulerModule.java | 46 ++++++++++++++++++++ .../org/apache/aurora/scheduler/TierInfo.java | 10 +++-- .../apache/aurora/scheduler/TierManager.java | 45 ++++++++++++++++++- .../aurora/scheduler/base/TaskTestUtil.java | 2 + .../aurora/scheduler/ResourceSlotTest.java | 8 ++-- .../apache/aurora/scheduler/ResourcesTest.java | 6 ++- .../aurora/scheduler/SchedulerModuleTest.java | 44 +++++++++++++++++++ .../aurora/scheduler/TierManagerTest.java | 40 +++++++++++++++-- .../mesos/MesosTaskFactoryImplTest.java | 22 +++++----- .../preemptor/PreemptionVictimFilterTest.java | 32 +++++++------- .../scheduler/state/TaskAssignerImplTest.java | 15 +++---- .../apache/aurora/scheduler/tiers-example.json | 9 ++++ 15 files changed, 238 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/examples/vagrant/upstart/aurora-scheduler-kerberos.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf index 744b4a3..4f43892 100644 --- a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf +++ b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf @@ -54,4 +54,5 @@ exec bin/aurora-scheduler \ -http_authentication_mechanism=NEGOTIATE \ -kerberos_server_keytab=/home/vagrant/krb5-1.13.1/build/testdir/HTTP-aurora.local.keytab \ -kerberos_server_principal=HTTP/[email protected] \ - -enable_h2_console=true + -enable_h2_console=true \ + -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf index 789a3a0..e909451 100644 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ b/examples/vagrant/upstart/aurora-scheduler.conf @@ -44,4 +44,5 @@ exec bin/aurora-scheduler \ -http_authentication_mechanism=BASIC \ -use_beta_db_task_store=true \ -shiro_ini_path=etc/shiro.example.ini \ - -enable_h2_console=true + -enable_h2_console=true \ + -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/jmh/java/org/apache/aurora/benchmark/Offers.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/Offers.java b/src/jmh/java/org/apache/aurora/benchmark/Offers.java index 26bad1c..055a2ff 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/Offers.java +++ b/src/jmh/java/org/apache/aurora/benchmark/Offers.java @@ -21,11 +21,12 @@ import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Data; import org.apache.aurora.scheduler.HostOffer; import org.apache.aurora.scheduler.ResourceSlot; -import org.apache.aurora.scheduler.TierInfo; import org.apache.aurora.scheduler.offers.OfferManager; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.mesos.Protos; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; + /** * Offer factory. */ @@ -90,7 +91,7 @@ final class Offers { for (IHostAttributes attributes : hostAttributes) { Protos.Offer offer = Protos.Offer.newBuilder() .addAllResources(new ResourceSlot(cpu, ram, disk, ports) - .toResourceList(new TierInfo(false))) + .toResourceList(DEFAULT)) .setId(Protos.OfferID.newBuilder().setValue(String.format(OFFER_ID_FORMAT, id++))) .setFrameworkId(Protos.FrameworkID.newBuilder().setValue(FRAMEWORK_ID)) .setSlaveId(Protos.SlaveID.newBuilder().setValue(attributes.getSlaveId())) http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java index d4bc6b9..6d940d1 100644 --- a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java @@ -13,6 +13,9 @@ */ package org.apache.aurora.scheduler; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; @@ -20,12 +23,18 @@ import java.util.logging.Logger; import javax.inject.Singleton; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Throwables; +import com.google.common.io.Files; import com.google.inject.AbstractModule; import com.google.inject.PrivateModule; import com.google.inject.TypeLiteral; import org.apache.aurora.common.args.Arg; import org.apache.aurora.common.args.CmdLine; +import org.apache.aurora.common.args.constraints.CanRead; import org.apache.aurora.common.args.constraints.Positive; import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Time; @@ -35,8 +44,10 @@ import org.apache.aurora.scheduler.TierManager.TierManagerImpl; import org.apache.aurora.scheduler.base.AsyncUtil; import org.apache.aurora.scheduler.events.PubsubEventModule; import org.apache.mesos.Protos; +import org.codehaus.jackson.map.ObjectMapper; import static org.apache.aurora.scheduler.SchedulerServicesModule.addSchedulerActiveServiceBinding; +import static org.apache.aurora.scheduler.TierManager.TierManagerImpl.TierConfig; /** * Binding module for top-level scheduling logic. @@ -60,6 +71,11 @@ public class SchedulerModule extends AbstractModule { help = "The maximum number of status updates that can be processed in a batch.") private static final Arg<Integer> MAX_STATUS_UPDATE_BATCH_SIZE = Arg.create(1000); + @CanRead + @CmdLine(name = "tier_config", + help = "Configuration file defining supported task tiers, task traits and behaviors.") + private static final Arg<File> TIER_CONFIG_FILE = Arg.create(); + @Override protected void configure() { bind(TaskIdGenerator.class).to(TaskIdGeneratorImpl.class); @@ -94,8 +110,38 @@ public class SchedulerModule extends AbstractModule { bind(TaskStatusHandler.class).to(TaskStatusHandlerImpl.class); bind(TaskStatusHandlerImpl.class).in(Singleton.class); + bind(TierConfig.class).toInstance(parseTierConfig(readTierFile())); bind(TierManager.class).to(TierManagerImpl.class); bind(TierManagerImpl.class).in(Singleton.class); addSchedulerActiveServiceBinding(binder()).to(TaskStatusHandlerImpl.class); } + + private static Optional<String> readTierFile() { + if (TIER_CONFIG_FILE.hasAppliedValue()) { + try { + return Optional.of(Files.toString(TIER_CONFIG_FILE.get(), StandardCharsets.UTF_8)); + } catch (IOException e) { + LOG.severe("Error loading tier configuration file."); + throw Throwables.propagate(e); + } + } + + return Optional.<String>absent(); + } + + @VisibleForTesting + static TierConfig parseTierConfig(Optional<String> config) { + Optional<TierConfig> map = config.transform(new Function<String, TierConfig>() { + @Override + public TierConfig apply(String input) { + try { + return new ObjectMapper().readValue(input, TierConfig.class); + } catch (IOException e) { + LOG.severe("Error parsing tier configuration file."); + throw Throwables.propagate(e); + } + } + }); + return map.or(TierConfig.EMPTY); + } } http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/main/java/org/apache/aurora/scheduler/TierInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/TierInfo.java b/src/main/java/org/apache/aurora/scheduler/TierInfo.java index 72c2f9e..b4611b9 100644 --- a/src/main/java/org/apache/aurora/scheduler/TierInfo.java +++ b/src/main/java/org/apache/aurora/scheduler/TierInfo.java @@ -15,16 +15,20 @@ package org.apache.aurora.scheduler; import java.util.Objects; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; /** * Defines common task tier traits and behaviors. */ -public class TierInfo { +public final class TierInfo { + public static final TierInfo DEFAULT = new TierInfo(); + private final boolean revocable; - @VisibleForTesting + private TierInfo() { + this(false); + } + public TierInfo(boolean revocable) { this.revocable = revocable; } http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/main/java/org/apache/aurora/scheduler/TierManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/TierManager.java b/src/main/java/org/apache/aurora/scheduler/TierManager.java index 652afec..2866efd 100644 --- a/src/main/java/org/apache/aurora/scheduler/TierManager.java +++ b/src/main/java/org/apache/aurora/scheduler/TierManager.java @@ -13,11 +13,23 @@ */ package org.apache.aurora.scheduler; +import java.util.Map; + +import javax.inject.Inject; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; + import org.apache.aurora.scheduler.storage.entities.ITaskConfig; +import org.codehaus.jackson.annotate.JsonCreator; +import org.codehaus.jackson.annotate.JsonProperty; + +import static java.util.Objects.requireNonNull; + +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; /** * Translates job tier configuration into a set of task traits/attributes. - * TODO(maxim): Implement external configuration support: AURORA-1443. */ public interface TierManager { @@ -30,10 +42,39 @@ public interface TierManager { TierInfo getTier(ITaskConfig taskConfig); class TierManagerImpl implements TierManager { + private final TierConfig tierConfig; + + static class TierConfig { + static final TierConfig EMPTY = new TierConfig(ImmutableMap.of()); + + private final Map<String, TierInfo> tiers; + + @JsonCreator + TierConfig(@JsonProperty("tiers") Map<String, TierInfo> tiers) { + this.tiers = ImmutableMap.copyOf(tiers); + } + + @VisibleForTesting + public Map<String, TierInfo> getTiers() { + return tiers; + } + } + + @Inject + TierManagerImpl(TierConfig tierConfig) { + this.tierConfig = requireNonNull(tierConfig); + } @Override public TierInfo getTier(ITaskConfig taskConfig) { - return new TierInfo(false); + if (taskConfig.isSetTier()) { + // The default behavior in case of tier config file absence or tier name mismatch is to use + // non-revocable resources. This is subject to change once the feature is ready. + // Tracked by AURORA-1443. + return tierConfig.tiers.getOrDefault(taskConfig.getTier(), DEFAULT); + } + + return DEFAULT; } } } http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java index d7f3c60..2cdb2f2 100644 --- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java +++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java @@ -32,6 +32,7 @@ import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.TaskConstraint; import org.apache.aurora.gen.TaskEvent; import org.apache.aurora.gen.ValueConstraint; +import org.apache.aurora.scheduler.TierInfo; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; @@ -45,6 +46,7 @@ import org.apache.aurora.scheduler.storage.entities.ITaskConfig; public final class TaskTestUtil { public static final IJobKey JOB = JobKeys.from("role", "env", "job"); + public static final TierInfo REVOCABLE_TIER = new TierInfo(true); private TaskTestUtil() { // Utility class. http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java b/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java index 399f58d..6fad334 100644 --- a/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java +++ b/src/test/java/org/apache/aurora/scheduler/ResourceSlotTest.java @@ -39,6 +39,8 @@ import static org.apache.aurora.scheduler.ResourceType.CPUS; import static org.apache.aurora.scheduler.ResourceType.DISK_MB; import static org.apache.aurora.scheduler.ResourceType.PORTS; import static org.apache.aurora.scheduler.ResourceType.RAM_MB; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; +import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -110,7 +112,7 @@ public class ResourceSlotTest { makeMesosResource(RAM_MB, TASK.getRamMb(), false), makeMesosResource(DISK_MB, TASK.getDiskMb(), false), makeMesosRangeResource(PORTS, ports)), - ImmutableSet.copyOf(resources.toResourceList(ports, new TierInfo(false)))); + ImmutableSet.copyOf(resources.toResourceList(ports, DEFAULT))); } @Test @@ -123,7 +125,7 @@ public class ResourceSlotTest { makeMesosResource(RAM_MB, TASK.getRamMb(), false), makeMesosResource(DISK_MB, TASK.getDiskMb(), false), makeMesosRangeResource(PORTS, ports)), - ImmutableSet.copyOf(resources.toResourceList(ports, new TierInfo(true)))); + ImmutableSet.copyOf(resources.toResourceList(ports, REVOCABLE_TIER))); } @Test @@ -134,7 +136,7 @@ public class ResourceSlotTest { makeMesosResource(CPUS, TASK.getNumCpus(), true), makeMesosResource(RAM_MB, TASK.getRamMb(), false), makeMesosResource(DISK_MB, TASK.getDiskMb(), false)), - ImmutableSet.copyOf(resources.toResourceList(new TierInfo(true)))); + ImmutableSet.copyOf(resources.toResourceList(REVOCABLE_TIER))); } @Test http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/ResourcesTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/ResourcesTest.java b/src/test/java/org/apache/aurora/scheduler/ResourcesTest.java index eb1baed..430e278 100644 --- a/src/test/java/org/apache/aurora/scheduler/ResourcesTest.java +++ b/src/test/java/org/apache/aurora/scheduler/ResourcesTest.java @@ -34,6 +34,8 @@ import static org.apache.aurora.scheduler.ResourceType.CPUS; import static org.apache.aurora.scheduler.ResourceType.DISK_MB; import static org.apache.aurora.scheduler.ResourceType.PORTS; import static org.apache.aurora.scheduler.ResourceType.RAM_MB; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; +import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.apache.mesos.Protos.Value.Type.RANGES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -131,11 +133,11 @@ public class ResourcesTest { assertEquals( new ResourceSlot(8.0, Amount.of(1024L, MB), Amount.of(0L, MB), 0), - Resources.from(createOffer(resources)).filter(new TierInfo(true)).slot()); + Resources.from(createOffer(resources)).filter(REVOCABLE_TIER).slot()); assertEquals( new ResourceSlot(8.0, Amount.of(1024L, MB), Amount.of(0L, MB), 0), - Resources.from(createOffer(resources)).filter(new TierInfo(false)).slot()); + Resources.from(createOffer(resources)).filter(DEFAULT).slot()); } private Resource createPortRange(Pair<Integer, Integer> range) { http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java b/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java new file mode 100644 index 0000000..9dd0569 --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java @@ -0,0 +1,44 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import org.junit.Test; + +import static org.apache.aurora.scheduler.SchedulerModule.parseTierConfig; +import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; +import static org.junit.Assert.assertEquals; + +public class SchedulerModuleTest { + @Test + public void testNoTierConfigFile() { + assertEquals(ImmutableMap.of(), parseTierConfig(Optional.absent()).getTiers()); + } + + @Test + public void testTierConfigFile() { + assertEquals( + ImmutableMap.of("revocable", REVOCABLE_TIER), + parseTierConfig(Optional.of("{\"tiers\":{\"revocable\": {\"revocable\": true}}}")) + .getTiers()); + } + + @Test(expected = RuntimeException.class) + public void testTierConfigExtraKeysNotAllowed() { + parseTierConfig( + Optional.of("{\"tiers\":{\"revocable\": {\"revocable\": true, \"foo\": false}}}")); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java index eb02071..350b90b 100644 --- a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java @@ -13,18 +13,52 @@ */ package org.apache.aurora.scheduler; +import com.google.common.base.Optional; + import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.TierManager.TierManagerImpl; +import org.apache.aurora.scheduler.TierManager.TierManagerImpl.TierConfig; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.junit.Test; +import static org.apache.aurora.scheduler.SchedulerModule.parseTierConfig; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; +import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.junit.Assert.assertEquals; public class TierManagerTest { @Test - public void testIsNotRevocable() { - TierInfo expected = new TierInfo(false); - assertEquals(expected, new TierManagerImpl().getTier(ITaskConfig.build(new TaskConfig()))); + public void testRevocable() { + TierManager manager = new TierManagerImpl( + parseTierConfig(Optional.of("{\"tiers\":{\"revocable\": {\"revocable\": true}}}"))); + assertEquals( + REVOCABLE_TIER, + manager.getTier(ITaskConfig.build(new TaskConfig().setTier("revocable")))); + } + + @Test + public void testNameMismatch() { + TierManager manager = new TierManagerImpl( + parseTierConfig(Optional.of("{\"tiers\":{\"revocable\": {\"revocable\": true}}}"))); + assertEquals( + DEFAULT, + manager.getTier(ITaskConfig.build(new TaskConfig().setTier("Revocable")))); + } + + @Test + public void testDefaultTier() { + TierManager manager = new TierManagerImpl(TierConfig.EMPTY); + assertEquals( + DEFAULT, + manager.getTier(ITaskConfig.build(new TaskConfig().setTier("revocable")))); + } + + @Test + public void testNoTier() { + TierManager manager = new TierManagerImpl(TierConfig.EMPTY); + assertEquals( + DEFAULT, + manager.getTier(ITaskConfig.build(new TaskConfig()))); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java index adcc7a8..dddf795 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java @@ -31,7 +31,6 @@ import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.Volume; import org.apache.aurora.scheduler.ResourceSlot; import org.apache.aurora.scheduler.Resources; -import org.apache.aurora.scheduler.TierInfo; import org.apache.aurora.scheduler.TierManager; import org.apache.aurora.scheduler.mesos.MesosTaskFactory.MesosTaskFactoryImpl; import org.apache.aurora.scheduler.storage.entities.IAssignedTask; @@ -49,6 +48,8 @@ import org.junit.Before; import org.junit.Test; import static org.apache.aurora.scheduler.ResourceSlot.MIN_THERMOS_RESOURCES; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; +import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.apache.aurora.scheduler.mesos.MesosTaskFactory.MesosTaskFactoryImpl.RESOURCES_EPSILON; import static org.apache.aurora.scheduler.mesos.TaskExecutors.NO_OVERHEAD_EXECUTOR; import static org.apache.aurora.scheduler.mesos.TaskExecutors.SOME_OVERHEAD_EXECUTOR; @@ -69,7 +70,6 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { .setNumCpus(5) .setContainer(Container.mesos(new MesosContainer())) .setRequestedPorts(ImmutableSet.of("http"))); - private static final TierInfo DEFAULT_TIER = new TierInfo(false); private static final IAssignedTask TASK = IAssignedTask.build(new AssignedTask() .setInstanceId(2) .setTaskId("task-id") @@ -97,7 +97,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { .setExecutorId(MesosTaskFactoryImpl.getExecutorId(TASK.getTaskId())) .setName(MesosTaskFactoryImpl.EXECUTOR_NAME) .setSource(MesosTaskFactoryImpl.getInstanceSourceName(TASK.getTask(), TASK.getInstanceId())) - .addAllResources(RESOURCES_EPSILON.toResourceList(DEFAULT_TIER)) + .addAllResources(RESOURCES_EPSILON.toResourceList(DEFAULT)) .setCommand(CommandInfo.newBuilder() .setValue("./executor.pex") .addUris(URI.newBuilder().setValue(NO_OVERHEAD_EXECUTOR.getExecutorPath()) @@ -121,7 +121,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { @Test public void testExecutorInfoUnchanged() { - expect(tierManager.getTier(TASK_CONFIG)).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(TASK_CONFIG)).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -133,7 +133,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { @Test public void testTaskInfoRevocable() { - expect(tierManager.getTier(TASK_CONFIG)).andReturn(new TierInfo(true)).times(2); + expect(tierManager.getTier(TASK_CONFIG)).andReturn(REVOCABLE_TIER).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -150,7 +150,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { builder.getTask().unsetRequestedPorts(); builder.unsetAssignedPorts(); IAssignedTask assignedTask = IAssignedTask.build(builder); - expect(tierManager.getTier(assignedTask.getTask())).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(assignedTask.getTask())).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -164,7 +164,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { // Here the ram required for the executor is greater than the sum of task resources // + executor overhead. We need to ensure we allocate a non-zero amount of ram in this case. config = NO_OVERHEAD_EXECUTOR; - expect(tierManager.getTier(TASK_CONFIG)).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(TASK_CONFIG)).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -192,7 +192,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { .setRequestedPorts(ImmutableSet.of()); IAssignedTask assignedTask = IAssignedTask.build(builder.setAssignedPorts(ImmutableMap.of())); - expect(tierManager.getTier(assignedTask.getTask())).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(assignedTask.getTask())).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -214,7 +214,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { private TaskInfo getDockerTaskInfo(IAssignedTask task) { config = TaskExecutors.SOME_OVERHEAD_EXECUTOR; - expect(tierManager.getTier(task.getTask())).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(task.getTask())).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -255,7 +255,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { .setThermosObserverRoot("/var/run/thermos") .setExecutorOverhead(SOME_OVERHEAD_EXECUTOR.getExecutorOverhead()) .build(); - expect(tierManager.getTier(TASK_CONFIG)).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(TASK_CONFIG)).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); @@ -273,7 +273,7 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { .setExecutorOverhead(SOME_OVERHEAD_EXECUTOR.getExecutorOverhead()) .setGlobalContainerMounts(ImmutableList.of(new Volume("/container", "/host", Mode.RO))) .build(); - expect(tierManager.getTier(TASK_WITH_DOCKER.getTask())).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(TASK_WITH_DOCKER.getTask())).andReturn(DEFAULT).times(2); taskFactory = new MesosTaskFactoryImpl(config, tierManager); control.replay(); http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java b/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java index 819b51e..8ce8da8 100644 --- a/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java +++ b/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java @@ -38,7 +38,6 @@ import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.TaskEvent; import org.apache.aurora.scheduler.HostOffer; import org.apache.aurora.scheduler.ResourceSlot; -import org.apache.aurora.scheduler.TierInfo; import org.apache.aurora.scheduler.TierManager; import org.apache.aurora.scheduler.filter.SchedulingFilter; import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto; @@ -60,6 +59,8 @@ import org.junit.Test; import static org.apache.aurora.gen.MaintenanceMode.NONE; import static org.apache.aurora.gen.ScheduleStatus.RUNNING; import static org.apache.aurora.scheduler.ResourceType.CPUS; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; +import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.apache.aurora.scheduler.filter.AttributeAggregate.EMPTY; import static org.apache.aurora.scheduler.preemptor.PreemptorMetrics.MISSING_ATTRIBUTES_NAME; import static org.apache.mesos.Protos.Offer; @@ -85,7 +86,6 @@ public class PreemptionVictimFilterTest extends EasyMockTest { private static final String HOST_ATTRIBUTE = "host"; private static final String OFFER = "offer"; private static final Optional<HostOffer> NO_OFFER = Optional.absent(); - private static final TierInfo DEFAULT_TIER = new TierInfo(false); private StorageTestUtil storageUtil; private SchedulingFilter schedulingFilter; @@ -127,7 +127,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { setUpHost(); schedulingFilter = createMock(SchedulingFilter.class); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); ScheduledTask lowPriority = makeTask(USER_A, JOB_A, TASK_ID_A); assignToHost(lowPriority); @@ -144,7 +144,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { setUpHost(); schedulingFilter = createMock(SchedulingFilter.class); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); ScheduledTask lowPriority = makeTask(USER_A, JOB_A, TASK_ID_A, 10); assignToHost(lowPriority); @@ -164,7 +164,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { setUpHost(); schedulingFilter = createMock(SchedulingFilter.class); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); ScheduledTask highPriority = makeTask(USER_A, JOB_A, TASK_ID_A, 100); assignToHost(highPriority); @@ -201,7 +201,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { setUpHost(); schedulingFilter = createMock(SchedulingFilter.class); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); // Use a very low priority for the production task to show that priority is irrelevant. ScheduledTask p1 = makeProductionTask(USER_A, JOB_A, TASK_ID_A + "_p1", -1000); ScheduledTask a1 = makeTask(USER_A, JOB_A, TASK_ID_B + "_a1", 100); @@ -218,7 +218,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { setUpHost(); schedulingFilter = createMock(SchedulingFilter.class); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); // Use a very low priority for the production task to show that priority is irrelevant. ScheduledTask p1 = makeProductionTask(USER_A, JOB_A, TASK_ID_A + "_p1", -1000); ScheduledTask a1 = makeTask(USER_B, JOB_A, TASK_ID_B + "_a1", 100); @@ -245,7 +245,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testProductionPreemptingManyNonProduction() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER).times(5); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT).times(5); ScheduledTask a1 = makeTask(USER_A, JOB_A, TASK_ID_A + "_a1"); a1.getAssignedTask().getTask().setNumCpus(1).setRamMb(512); @@ -268,7 +268,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testMinimalSetPreempted() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER).times(9); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT).times(9); ScheduledTask a1 = makeTask(USER_A, JOB_A, TASK_ID_A + "_a1"); a1.getAssignedTask().getTask().setNumCpus(4).setRamMb(4096); @@ -313,7 +313,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testPreemptWithOfferAndTask() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); setUpHost(); @@ -337,7 +337,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testRevocableOfferFiltered() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); setUpHost(); @@ -359,7 +359,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testRevocableVictimsFiltered() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(new TierInfo(true)); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(REVOCABLE_TIER); setUpHost(); @@ -381,7 +381,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testRevocableVictimRamUsed() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(new TierInfo(true)); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(REVOCABLE_TIER); setUpHost(); @@ -405,7 +405,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testPreemptWithOfferAndMultipleTasks() throws Exception { schedulingFilter = new SchedulingFilterImpl(TaskExecutors.NO_OVERHEAD_EXECUTOR); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER).times(5); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT).times(5); setUpHost(); @@ -457,7 +457,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { @Test public void testAllVictimsVetoed() { schedulingFilter = createMock(SchedulingFilter.class); - expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(EasyMock.anyObject())).andReturn(DEFAULT); ScheduledTask task = makeProductionTask(USER_A, JOB_A, TASK_ID_A); assignToHost(task); @@ -504,7 +504,7 @@ public class PreemptionVictimFilterTest extends EasyMockTest { boolean revocable) { List<Resource> resources = - new ResourceSlot(cpu, ram, disk, numPorts).toResourceList(new TierInfo(false)); + new ResourceSlot(cpu, ram, disk, numPorts).toResourceList(DEFAULT); if (revocable) { resources = ImmutableList.<Resource>builder() .addAll(FluentIterable.from(resources) http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/java/org/apache/aurora/scheduler/state/TaskAssignerImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/state/TaskAssignerImplTest.java b/src/test/java/org/apache/aurora/scheduler/state/TaskAssignerImplTest.java index d8a524d..3cbe9ac 100644 --- a/src/test/java/org/apache/aurora/scheduler/state/TaskAssignerImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/state/TaskAssignerImplTest.java @@ -28,7 +28,6 @@ import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.HostOffer; import org.apache.aurora.scheduler.Resources; -import org.apache.aurora.scheduler.TierInfo; import org.apache.aurora.scheduler.TierManager; import org.apache.aurora.scheduler.base.TaskGroupKey; import org.apache.aurora.scheduler.base.Tasks; @@ -56,6 +55,7 @@ import org.junit.Test; import static org.apache.aurora.gen.ScheduleStatus.LOST; import static org.apache.aurora.gen.ScheduleStatus.PENDING; +import static org.apache.aurora.scheduler.TierInfo.DEFAULT; import static org.apache.aurora.scheduler.filter.AttributeAggregate.EMPTY; import static org.apache.aurora.scheduler.state.TaskAssigner.TaskAssignerImpl.LAUNCH_FAILED_MSG; import static org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider; @@ -102,7 +102,6 @@ public class TaskAssignerImplTest extends EasyMockTest { new UnusedResource(Resources.from(MESOS_OFFER).slot(), OFFER.getAttributes()); private static final ResourceRequest RESOURCE_REQUEST = new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY); - private static final TierInfo DEFAULT_TIER = new TierInfo(false); private MutableStoreProvider storeProvider; private StateManager stateManager; @@ -127,7 +126,7 @@ public class TaskAssignerImplTest extends EasyMockTest { public void testAssignNoVetoes() throws Exception { expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER)); offerManager.launchTask(MESOS_OFFER.getId(), TASK_INFO); - expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT); expect(filter.filter(UNUSED, RESOURCE_REQUEST)).andReturn(ImmutableSet.of()); expect(stateManager.assignTask( storeProvider, @@ -153,7 +152,7 @@ public class TaskAssignerImplTest extends EasyMockTest { public void testAssignVetoesWithStaticBan() throws Exception { expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER)); offerManager.banOffer(MESOS_OFFER.getId(), GROUP_KEY); - expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT); expect(filter.filter(UNUSED, RESOURCE_REQUEST)) .andReturn(ImmutableSet.of(Veto.constraintMismatch("denied"))); @@ -170,7 +169,7 @@ public class TaskAssignerImplTest extends EasyMockTest { @Test public void testAssignVetoesWithNoStaticBan() throws Exception { expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER)); - expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT); expect(filter.filter(UNUSED, RESOURCE_REQUEST)) .andReturn(ImmutableSet.of(Veto.unsatisfiedLimit("limit"))); @@ -189,7 +188,7 @@ public class TaskAssignerImplTest extends EasyMockTest { expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER)); offerManager.launchTask(MESOS_OFFER.getId(), TASK_INFO); expectLastCall().andThrow(new OfferManager.LaunchException("expected")); - expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT); expect(filter.filter(UNUSED, RESOURCE_REQUEST)).andReturn(ImmutableSet.of()); expect(stateManager.assignTask( storeProvider, @@ -253,7 +252,7 @@ public class TaskAssignerImplTest extends EasyMockTest { IHostAttributes.build(new HostAttributes())); expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(offer, OFFER)); - expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT_TIER); + expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT); expect(filter.filter(UNUSED, RESOURCE_REQUEST)).andReturn(ImmutableSet.of()); expect(stateManager.assignTask( storeProvider, @@ -294,7 +293,7 @@ public class TaskAssignerImplTest extends EasyMockTest { IHostAttributes.build(new HostAttributes())); expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(mismatched, OFFER)); - expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT_TIER).times(2); + expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT).times(2); expect(filter.filter( new UnusedResource( Resources.from(mismatched.getOffer()).slot(), http://git-wip-us.apache.org/repos/asf/aurora/blob/89da936f/src/test/resources/org/apache/aurora/scheduler/tiers-example.json ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/tiers-example.json b/src/test/resources/org/apache/aurora/scheduler/tiers-example.json new file mode 100644 index 0000000..2140773 --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/tiers-example.json @@ -0,0 +1,9 @@ +{ + "tiers": + { + "revocable": + { + "revocable": true + } + } +}
