Repository: aurora Updated Branches: refs/heads/master 7e30ebe34 -> e817eb11d
Introduce command line option to control the offer filter duration Aurora is declining Mesos offers implicitly when launching a task and explicitly when compacting multiple offers of a slave into a single one. The filter duration instructs Mesos to return the declined resources to us only after a timeout of X seconds, even if there is no other framework that wants them. If no filter is supplied, the hardcoded default of 5 seconds would be used. By making this value configurable, Aurora can be tuned for either single or multi-framework deployment. Bugs closed: AURORA-1658 Reviewed at https://reviews.apache.org/r/46603/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/e817eb11 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/e817eb11 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/e817eb11 Branch: refs/heads/master Commit: e817eb11d0708a5217e144c1b05a80f14759e337 Parents: 7e30ebe Author: Stephan Erb <s...@apache.org> Authored: Thu Apr 28 00:23:20 2016 +0200 Committer: Stephan Erb <step...@dev.static-void.de> Committed: Thu Apr 28 00:23:20 2016 +0200 ---------------------------------------------------------------------- RELEASE-NOTES.md | 4 ++ .../aurora/benchmark/SchedulingBenchmarks.java | 4 +- .../aurora/benchmark/fakes/FakeDriver.java | 4 +- .../apache/aurora/scheduler/mesos/Driver.java | 7 ++- .../scheduler/mesos/SchedulerDriverService.java | 8 +-- .../aurora/scheduler/offers/OfferManager.java | 28 +++++----- .../aurora/scheduler/offers/OfferSettings.java | 55 ++++++++++++++++++++ .../aurora/scheduler/offers/OffersModule.java | 19 +++++-- .../offers/RandomJitterReturnDelay.java | 7 +-- .../scheduler/offers/OfferManagerImplTest.java | 36 +++++++------ 10 files changed, 124 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 3f2c54c..7a37d0d 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -8,6 +8,10 @@ - Adding --nosetuid-health-checks flag to control whether the executor runs health checks as the job's role's user. - Upgraded Mesos to 0.27.2 +- New scheduler command line argument `-offer_filter_duration` to control the time after which we + expect Mesos to re-offer unused resources. A short duration improves scheduling performance in + smaller clusters, but might lead to resource starvation for other frameworks if you run multiple + ones in your cluster. Uses the Mesos default of 5s. ### Deprecations and removals: http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java index 1d725c0..1aa484c 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java +++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java @@ -46,6 +46,7 @@ import org.apache.aurora.scheduler.filter.SchedulingFilterImpl; import org.apache.aurora.scheduler.mesos.Driver; import org.apache.aurora.scheduler.mesos.TestExecutorSettings; import org.apache.aurora.scheduler.offers.OfferManager; +import org.apache.aurora.scheduler.offers.OfferSettings; import org.apache.aurora.scheduler.preemptor.BiCache; import org.apache.aurora.scheduler.preemptor.ClusterStateImpl; import org.apache.aurora.scheduler.preemptor.PendingTaskProcessor; @@ -130,7 +131,8 @@ public class SchedulingBenchmarks { }); bind(OfferManager.class).to(OfferManager.OfferManagerImpl.class); bind(OfferManager.OfferManagerImpl.class).in(Singleton.class); - bind(OfferManager.OfferReturnDelay.class).toInstance(() -> DELAY_FOREVER); + bind(OfferSettings.class).toInstance( + new OfferSettings(NO_DELAY, () -> DELAY_FOREVER)); bind(BiCache.BiCacheSettings.class).toInstance( new BiCache.BiCacheSettings(DELAY_FOREVER, "")); bind(TaskScheduler.class).to(TaskScheduler.TaskSchedulerImpl.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/jmh/java/org/apache/aurora/benchmark/fakes/FakeDriver.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/fakes/FakeDriver.java b/src/jmh/java/org/apache/aurora/benchmark/fakes/FakeDriver.java index d1bb8f2..1bfd8db 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/fakes/FakeDriver.java +++ b/src/jmh/java/org/apache/aurora/benchmark/fakes/FakeDriver.java @@ -27,12 +27,12 @@ public class FakeDriver extends AbstractIdleService implements Driver { } @Override - public void launchTask(Protos.OfferID offerId, Protos.TaskInfo task) { + public void launchTask(Protos.OfferID offerId, Protos.TaskInfo task, Protos.Filters filter) { // no-op } @Override - public void declineOffer(Protos.OfferID offerId) { + public void declineOffer(Protos.OfferID offerId, Protos.Filters filters) { // no-op } http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/main/java/org/apache/aurora/scheduler/mesos/Driver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/Driver.java b/src/main/java/org/apache/aurora/scheduler/mesos/Driver.java index 013c50c..1448146 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/Driver.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/Driver.java @@ -17,6 +17,7 @@ import java.util.Collection; import com.google.common.util.concurrent.Service; +import org.apache.mesos.Protos.Filters; import org.apache.mesos.Protos.OfferID; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.Protos.TaskStatus; @@ -36,15 +37,17 @@ public interface Driver extends Service { * * @param offerId ID of the resource offer to accept with the task. * @param task Task to launch. + * @param filter offer filter applied to unused resources in this offer. */ - void launchTask(OfferID offerId, TaskInfo task); + void launchTask(OfferID offerId, TaskInfo task, Filters filter); /** * Declines a resource offer. * * @param offerId ID of the offer to decline. + * @param filter offer filter applied to this declined offer. */ - void declineOffer(OfferID offerId); + void declineOffer(OfferID offerId, Filters filter); /** * Sends a kill task request for the given {@code taskId} to the mesos master. http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java b/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java index 7ff3e3e..41b9aab 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java @@ -116,16 +116,16 @@ class SchedulerDriverService extends AbstractIdleService implements Driver { } @Override - public void launchTask(Protos.OfferID offerId, Protos.TaskInfo task) { + public void launchTask(Protos.OfferID offerId, Protos.TaskInfo task, Protos.Filters filter) { ensureRunning(); Futures.getUnchecked(driverFuture) - .launchTasks(ImmutableList.of(offerId), ImmutableList.of(task)); + .launchTasks(ImmutableList.of(offerId), ImmutableList.of(task), filter); } @Override - public void declineOffer(Protos.OfferID offerId) { + public void declineOffer(Protos.OfferID offerId, Protos.Filters filter) { ensureRunning(); - Futures.getUnchecked(driverFuture).declineOffer(offerId); + Futures.getUnchecked(driverFuture).declineOffer(offerId, filter); } @Override http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/main/java/org/apache/aurora/scheduler/offers/OfferManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/offers/OfferManager.java b/src/main/java/org/apache/aurora/scheduler/offers/OfferManager.java index 452451f..a1ae7c7 100644 --- a/src/main/java/org/apache/aurora/scheduler/offers/OfferManager.java +++ b/src/main/java/org/apache/aurora/scheduler/offers/OfferManager.java @@ -24,7 +24,6 @@ import javax.inject.Inject; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Optional; -import com.google.common.base.Supplier; import com.google.common.collect.FluentIterable; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; @@ -35,7 +34,6 @@ import com.google.common.collect.Ordering; import com.google.common.eventbus.Subscribe; import org.apache.aurora.common.inject.TimedInterceptor.Timed; -import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Time; import org.apache.aurora.common.stats.Stats; import org.apache.aurora.gen.MaintenanceMode; @@ -130,14 +128,6 @@ public interface OfferManager extends EventSubscriber { Optional<HostOffer> getOffer(SlaveID slaveId); /** - * Calculates the amount of time before an offer should be 'returned' by declining it. - * The delay is calculated for each offer that is received, so the return delay may be - * fixed or variable. - */ - interface OfferReturnDelay extends Supplier<Amount<Long, Time>> { - } - - /** * Thrown when there was an unexpected failure trying to launch a task. */ class LaunchException extends Exception { @@ -159,18 +149,18 @@ public interface OfferManager extends EventSubscriber { private final AtomicLong offerRaces = Stats.exportLong("offer_accept_races"); private final Driver driver; - private final OfferReturnDelay returnDelay; + private final OfferSettings offerSettings; private final DelayExecutor executor; @Inject @VisibleForTesting public OfferManagerImpl( Driver driver, - OfferReturnDelay returnDelay, + OfferSettings offerSettings, @AsyncExecutor DelayExecutor executor) { this.driver = requireNonNull(driver); - this.returnDelay = requireNonNull(returnDelay); + this.offerSettings = requireNonNull(offerSettings); this.executor = requireNonNull(executor); } @@ -193,7 +183,7 @@ public interface OfferManager extends EventSubscriber { hostOffers.add(offer); executor.execute( () -> removeAndDecline(offer.getOffer().getId()), - returnDelay.get()); + offerSettings.getOfferReturnDelay()); } } @@ -205,7 +195,13 @@ public interface OfferManager extends EventSubscriber { void decline(OfferID id) { LOG.debug("Declining offer " + id); - driver.declineOffer(id); + driver.declineOffer(id, getOfferFilter()); + } + + private Protos.Filters getOfferFilter() { + return Protos.Filters.newBuilder() + .setRefuseSeconds(offerSettings.getOfferFilterDuration().as(Time.SECONDS)) + .build(); } @Override @@ -361,7 +357,7 @@ public interface OfferManager extends EventSubscriber { // which is a feature of ConcurrentSkipListSet. if (hostOffers.remove(offerId)) { try { - driver.launchTask(offerId, task); + driver.launchTask(offerId, task, getOfferFilter()); } catch (IllegalStateException e) { // TODO(William Farner): Catch only the checked exception produced by Driver // once it changes from throwing IllegalStateException when the driver is not yet http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/main/java/org/apache/aurora/scheduler/offers/OfferSettings.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/offers/OfferSettings.java b/src/main/java/org/apache/aurora/scheduler/offers/OfferSettings.java new file mode 100644 index 0000000..e16e36e --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/offers/OfferSettings.java @@ -0,0 +1,55 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.offers; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; + +import org.apache.aurora.common.quantity.Amount; +import org.apache.aurora.common.quantity.Time; + +import static java.util.Objects.requireNonNull; + +/** + * Settings required to create an OfferManager. + */ +@VisibleForTesting +public class OfferSettings { + + private final Amount<Long, Time> offerFilterDuration; + private final Supplier<Amount<Long, Time>> returnDelaySupplier; + + public OfferSettings( + Amount<Long, Time> offerFilterDuration, + Supplier<Amount<Long, Time>> returnDelaySupplier) { + + this.offerFilterDuration = requireNonNull(offerFilterDuration); + this.returnDelaySupplier = requireNonNull(returnDelaySupplier); + } + + /** + * Duration after which we want Mesos to re-offer unused or declined resources. + */ + public Amount<Long, Time> getOfferFilterDuration() { + return offerFilterDuration; + } + + /** + * The amount of time after which an unused offer should be 'returned' to Mesos by declining it. + * The delay is calculated for each offer using a random duration within a fixed window. + */ + public Amount<Long, Time> getOfferReturnDelay() { + return returnDelaySupplier.get(); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java b/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java index 90f8abf..202cae9 100644 --- a/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java +++ b/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java @@ -43,16 +43,25 @@ public class OffersModule extends AbstractModule { private static final Arg<Amount<Integer, Time>> OFFER_HOLD_JITTER_WINDOW = Arg.create(Amount.of(1, Time.MINUTES)); + @CmdLine(name = "offer_filter_duration", + help = "Duration after which we expect Mesos to re-offer unused resources. A short duration " + + "improves scheduling performance in smaller clusters, but might lead to resource " + + "starvation for other frameworks if you run many frameworks in your cluster.") + private static final Arg<Amount<Long, Time>> OFFER_FILTER_DURATION = + Arg.create(Amount.of(5L, Time.SECONDS)); + @Override protected void configure() { install(new PrivateModule() { @Override protected void configure() { - bind(OfferManager.OfferReturnDelay.class).toInstance( - new RandomJitterReturnDelay( - MIN_OFFER_HOLD_TIME.get().as(Time.MILLISECONDS), - OFFER_HOLD_JITTER_WINDOW.get().as(Time.MILLISECONDS), - Random.Util.newDefaultRandom())); + bind(OfferSettings.class).toInstance( + new OfferSettings( + OFFER_FILTER_DURATION.get(), + new RandomJitterReturnDelay( + MIN_OFFER_HOLD_TIME.get().as(Time.MILLISECONDS), + OFFER_HOLD_JITTER_WINDOW.get().as(Time.MILLISECONDS), + Random.Util.newDefaultRandom()))); bind(OfferManager.class).to(OfferManager.OfferManagerImpl.class); bind(OfferManager.OfferManagerImpl.class).in(Singleton.class); expose(OfferManager.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java b/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java index a52fd4e..4b308dc 100644 --- a/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java +++ b/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java @@ -16,19 +16,20 @@ package org.apache.aurora.scheduler.offers; import java.util.Objects; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Time; import org.apache.aurora.common.util.Random; -import org.apache.aurora.scheduler.offers.OfferManager.OfferReturnDelay; import static com.google.common.base.Preconditions.checkArgument; /** - * Returns offers after a random duration within a fixed window. + * Calculates the amount of time before an offer should be 'returned' by declining it. + * Uses a random duration within a fixed window. */ @VisibleForTesting -class RandomJitterReturnDelay implements OfferReturnDelay { +class RandomJitterReturnDelay implements Supplier<Amount<Long, Time>> { private final int minHoldTimeMs; private final int maxJitterWindowMs; private final Random random; http://git-wip-us.apache.org/repos/asf/aurora/blob/e817eb11/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java b/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java index 76da6d8..e7534c4 100644 --- a/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java @@ -31,11 +31,11 @@ import org.apache.aurora.scheduler.events.PubsubEvent.DriverDisconnected; import org.apache.aurora.scheduler.events.PubsubEvent.HostAttributesChanged; import org.apache.aurora.scheduler.mesos.Driver; import org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl; -import org.apache.aurora.scheduler.offers.OfferManager.OfferReturnDelay; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.apache.aurora.scheduler.testing.FakeScheduledExecutor; import org.apache.mesos.Protos; +import org.apache.mesos.Protos.Filters; import org.apache.mesos.Protos.TaskInfo; import org.junit.Before; import org.junit.Test; @@ -67,6 +67,10 @@ public class OfferManagerImplTest extends EasyMockTest { private static final TaskGroupKey GROUP_KEY = TaskGroupKey.from( ITaskConfig.build(new TaskConfig().setJob(new JobKey("role", "env", "name")))); private static final TaskInfo TASK_INFO = TaskInfo.getDefaultInstance(); + private static final long OFFER_FILTER_SECONDS = 0L; + private static final Filters OFFER_FILTER = Filters.newBuilder() + .setRefuseSeconds(OFFER_FILTER_SECONDS) + .build(); private Driver driver; private FakeScheduledExecutor clock; @@ -78,8 +82,10 @@ public class OfferManagerImplTest extends EasyMockTest { DelayExecutor executorMock = createMock(DelayExecutor.class); clock = FakeScheduledExecutor.fromDelayExecutor(executorMock); addTearDown(clock::assertEmpty); - OfferReturnDelay returnDelay = () -> RETURN_DELAY; - offerManager = new OfferManagerImpl(driver, returnDelay, executorMock); + OfferSettings offerSettings = new OfferSettings( + Amount.of(OFFER_FILTER_SECONDS, Time.SECONDS), + () -> RETURN_DELAY); + offerManager = new OfferManagerImpl(driver, offerSettings, executorMock); } @Test @@ -89,10 +95,10 @@ public class OfferManagerImplTest extends EasyMockTest { HostOffer offerA = setMode(OFFER_A, DRAINING); HostOffer offerC = setMode(OFFER_C, DRAINING); - driver.launchTask(OFFER_B.getOffer().getId(), TASK_INFO); + driver.launchTask(OFFER_B.getOffer().getId(), TASK_INFO, OFFER_FILTER); - driver.declineOffer(OFFER_A_ID); - driver.declineOffer(offerC.getOffer().getId()); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); + driver.declineOffer(offerC.getOffer().getId(), OFFER_FILTER); control.replay(); @@ -108,8 +114,8 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void hostAttributeChangeUpdatesOfferSorting() throws Exception { - driver.declineOffer(OFFER_A_ID); - driver.declineOffer(OFFER_B.getOffer().getId()); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); + driver.declineOffer(OFFER_B.getOffer().getId(), OFFER_FILTER); control.replay(); @@ -134,7 +140,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void testAddSameSlaveOffer() { - driver.declineOffer(OFFER_A_ID); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); expectLastCall().times(2); control.replay(); @@ -160,7 +166,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void testOfferFilteringDueToStaticBan() throws Exception { - driver.declineOffer(OFFER_A_ID); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); control.replay(); @@ -181,7 +187,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void testStaticBanIsClearedOnOfferReturn() throws Exception { - driver.declineOffer(OFFER_A_ID); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); expectLastCall().times(2); control.replay(); @@ -201,7 +207,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void testStaticBanIsClearedOnDriverDisconnect() throws Exception { - driver.declineOffer(OFFER_A_ID); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); control.replay(); @@ -220,7 +226,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void getOffer() { - driver.declineOffer(OFFER_A_ID); + driver.declineOffer(OFFER_A_ID, OFFER_FILTER); control.replay(); @@ -231,7 +237,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test(expected = OfferManager.LaunchException.class) public void testLaunchTaskDriverThrows() throws OfferManager.LaunchException { - driver.launchTask(OFFER_A_ID, TASK_INFO); + driver.launchTask(OFFER_A_ID, TASK_INFO, OFFER_FILTER); expectLastCall().andThrow(new IllegalStateException()); control.replay(); @@ -263,7 +269,7 @@ public class OfferManagerImplTest extends EasyMockTest { @Test public void testDeclineOffer() throws Exception { - driver.declineOffer(OFFER_A.getOffer().getId()); + driver.declineOffer(OFFER_A.getOffer().getId(), OFFER_FILTER); control.replay();
