Repository: aurora Updated Branches: refs/heads/master 27a602d2c -> 5587bce74
Ignore maxWaitToInstanceRunningMs in the scheduler updater. Bugs closed: AURORA-1240 Reviewed at https://reviews.apache.org/r/32840/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/5587bce7 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/5587bce7 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/5587bce7 Branch: refs/heads/master Commit: 5587bce74c59d720f0cb6ba9772e4e9ea656b157 Parents: 27a602d Author: Bill Farner <[email protected]> Authored: Mon Apr 6 11:31:03 2015 -0700 Committer: Bill Farner <[email protected]> Committed: Mon Apr 6 11:31:03 2015 -0700 ---------------------------------------------------------------------- .../thrift/SchedulerThriftInterface.java | 15 +++++- .../scheduler/updater/InstanceUpdater.java | 38 ++----------- .../scheduler/updater/OneWayJobUpdater.java | 1 - .../scheduler/updater/StateEvaluator.java | 5 +- .../aurora/scheduler/updater/UpdateFactory.java | 1 - .../thrift/SchedulerThriftInterfaceTest.java | 41 +++++++++++--- .../scheduler/updater/InstanceUpdaterTest.java | 57 +++++++++----------- .../aurora/scheduler/updater/JobUpdaterIT.java | 24 +++------ 8 files changed, 85 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java index 5588d17..73c4605 100644 --- a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java +++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java @@ -1100,6 +1100,10 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin { return builder.build(); } + @VisibleForTesting + static final String MAX_WAIT_TO_INSTANCE_FIELD_WARNING = + "Warning: the maxWaitToInstanceRunningMs setting is deprecated and will be removed in 0.9.0."; + @Override public Response startJobUpdate( JobUpdateRequest mutableRequest, @@ -1132,6 +1136,13 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin { return invalidRequest(INVALID_MAX_FAILED_INSTANCES); } + final Response response = ok(); + if (settings.isSetMaxWaitToInstanceRunningMs()) { + addMessage( + response, + MAX_WAIT_TO_INSTANCE_FIELD_WARNING); + } + if (settings.getMaxWaitToInstanceRunningMs() < 0) { return invalidRequest(INVALID_MAX_WAIT_TO_RUNNING); } @@ -1212,10 +1223,10 @@ class SchedulerThriftInterface implements AnnotatedAuroraAdmin { jobUpdateController.start( update, new AuditData(context.getIdentity(), Optional.fromNullable(message))); - return ok(Result.startJobUpdateResult( + return response.setResult(Result.startJobUpdateResult( new StartJobUpdateResult(update.getSummary().getKey().newBuilder()))); } catch (UpdateStateException | TaskValidationException e) { - return error(INVALID_REQUEST, e); + return addMessage(response, INVALID_REQUEST, e); } } }); http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/main/java/org/apache/aurora/scheduler/updater/InstanceUpdater.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/updater/InstanceUpdater.java b/src/main/java/org/apache/aurora/scheduler/updater/InstanceUpdater.java index 14753cf..722dba4 100644 --- a/src/main/java/org/apache/aurora/scheduler/updater/InstanceUpdater.java +++ b/src/main/java/org/apache/aurora/scheduler/updater/InstanceUpdater.java @@ -19,7 +19,6 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import com.twitter.common.quantity.Amount; import com.twitter.common.quantity.Time; import com.twitter.common.util.Clock; @@ -28,7 +27,6 @@ import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.scheduler.base.Tasks; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; -import org.apache.aurora.scheduler.storage.entities.ITaskEvent; import static java.util.Objects.requireNonNull; @@ -36,7 +34,6 @@ import static org.apache.aurora.gen.ScheduleStatus.KILLING; import static org.apache.aurora.gen.ScheduleStatus.RUNNING; import static org.apache.aurora.scheduler.updater.StateEvaluator.Result.EVALUATE_AFTER_MIN_RUNNING_MS; import static org.apache.aurora.scheduler.updater.StateEvaluator.Result.EVALUATE_ON_STATE_CHANGE; -import static org.apache.aurora.scheduler.updater.StateEvaluator.Result.FAILED_STUCK; import static org.apache.aurora.scheduler.updater.StateEvaluator.Result.FAILED_TERMINATED; import static org.apache.aurora.scheduler.updater.StateEvaluator.Result.KILL_TASK_AND_EVALUATE_ON_STATE_CHANGE; import static org.apache.aurora.scheduler.updater.StateEvaluator.Result.REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE; @@ -53,7 +50,6 @@ class InstanceUpdater implements StateEvaluator<Optional<IScheduledTask>> { private final Optional<ITaskConfig> desiredState; private final int toleratedFailures; private final Amount<Long, Time> minRunningTime; - private final Amount<Long, Time> maxNonRunningTime; private final Clock clock; private int observedFailures = 0; @@ -62,36 +58,17 @@ class InstanceUpdater implements StateEvaluator<Optional<IScheduledTask>> { Optional<ITaskConfig> desiredState, int toleratedFailures, Amount<Long, Time> minRunningTime, - Amount<Long, Time> maxNonRunningTime, Clock clock) { this.desiredState = requireNonNull(desiredState); this.toleratedFailures = toleratedFailures; this.minRunningTime = requireNonNull(minRunningTime); - this.maxNonRunningTime = requireNonNull(maxNonRunningTime); this.clock = requireNonNull(clock); } - private long millisSince(ITaskEvent event) { - return clock.nowMillis() - event.getTimestamp(); - } - private boolean appearsStable(IScheduledTask task) { - return millisSince(Tasks.getLatestEvent(task)) >= minRunningTime.as(Time.MILLISECONDS); - } - - private boolean appearsStuck(IScheduledTask task) { - // Walk task events backwards to find the first event, or first non-running event. - ITaskEvent earliestNonRunningEvent = task.getTaskEvents().get(0); - for (ITaskEvent event : Lists.reverse(task.getTaskEvents())) { - if (event.getStatus() == RUNNING) { - break; - } else { - earliestNonRunningEvent = event; - } - } - - return millisSince(earliestNonRunningEvent) >= maxNonRunningTime.as(Time.MILLISECONDS); + return (clock.nowMillis() - Tasks.getLatestEvent(task).getTimestamp()) + >= minRunningTime.as(Time.MILLISECONDS); } private static boolean isPermanentlyKilled(IScheduledTask task) { @@ -158,16 +135,9 @@ class InstanceUpdater implements StateEvaluator<Optional<IScheduledTask>> { // The desired task has terminated, this is a failure. LOG.info("Task is in terminal state " + status); return addFailureAndCheckIfFailed() ? FAILED_TERMINATED : EVALUATE_ON_STATE_CHANGE; - } else if (appearsStuck(actualState)) { - LOG.info("Task appears stuck."); - // The task is not running, but not terminated, and appears to have been in this state - // long enough that we should intervene. - return addFailureAndCheckIfFailed() - ? FAILED_STUCK - : KILL_TASK_AND_EVALUATE_ON_STATE_CHANGE; } else { - // The task is in a transient state on the way into or out of running, check back later. - return EVALUATE_AFTER_MIN_RUNNING_MS; + // The task is in the process of being restarted, check back later. + return EVALUATE_ON_STATE_CHANGE; } } else { // This is not the configuration that we would like to run. http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/main/java/org/apache/aurora/scheduler/updater/OneWayJobUpdater.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/updater/OneWayJobUpdater.java b/src/main/java/org/apache/aurora/scheduler/updater/OneWayJobUpdater.java index 0f4913a..fc88adb 100644 --- a/src/main/java/org/apache/aurora/scheduler/updater/OneWayJobUpdater.java +++ b/src/main/java/org/apache/aurora/scheduler/updater/OneWayJobUpdater.java @@ -262,7 +262,6 @@ class OneWayJobUpdater<K, T> { private static final Map<Result, SideEffect.InstanceUpdateStatus> TERMINAL_RESULT_TO_STATUS = ImmutableMap.of( Result.SUCCEEDED, SUCCEEDED, - Result.FAILED_STUCK, FAILED, Result.FAILED_TERMINATED, FAILED ); http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java b/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java index 1590e12..6815f67 100644 --- a/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java +++ b/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java @@ -23,8 +23,7 @@ import static java.util.Objects.requireNonNull; * A state evaluator is expected to be used multiple times over the course of changing an active * task's configuration. This should be invoked every time the state of an instance changes to * determine what action to take next. It's expected that it will eventually converge by - * {@link Result#SUCCEEDED succeeding}, or failing with {@link Result#FAILED_STUCK} or - * {@link Result#FAILED_TERMINATED}. + * {@link Result#SUCCEEDED succeeding}, or failing with {@link Result#FAILED_TERMINATED}. * * @param <T> Instance state type. */ @@ -56,7 +55,6 @@ interface StateEvaluator<T> { KILL_TASK_AND_EVALUATE_ON_STATE_CHANGE(Optional.of(InstanceAction.KILL_TASK), NO_FAILURE), EVALUATE_AFTER_MIN_RUNNING_MS(Optional.of(InstanceAction.WATCH_TASK), NO_FAILURE), SUCCEEDED(Optional.<InstanceAction>absent(), NO_FAILURE), - FAILED_STUCK(Optional.<InstanceAction>absent(), Optional.of(Failure.STUCK)), FAILED_TERMINATED(Optional.<InstanceAction>absent(), Optional.of(Failure.EXITED)); private final Optional<InstanceAction> action; @@ -77,7 +75,6 @@ interface StateEvaluator<T> { } enum Failure { - STUCK("took too long to transition from a transient state."), EXITED("exited."); private final String reason; http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java b/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java index b87ae4e..927a7d1 100644 --- a/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java +++ b/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java @@ -118,7 +118,6 @@ interface UpdateFactory { desiredStateConfig, settings.getMaxPerInstanceFailures(), Amount.of((long) settings.getMinWaitInInstanceRunningMs(), Time.MILLISECONDS), - Amount.of((long) settings.getMaxWaitToInstanceRunningMs(), Time.MILLISECONDS), clock)); } http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java index f5326af..9ae8f1a 100644 --- a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java +++ b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java @@ -125,7 +125,6 @@ import static org.apache.aurora.gen.MaintenanceMode.DRAINING; import static org.apache.aurora.gen.MaintenanceMode.NONE; import static org.apache.aurora.gen.MaintenanceMode.SCHEDULED; import static org.apache.aurora.gen.ResponseCode.AUTH_FAILED; -import static org.apache.aurora.gen.ResponseCode.ERROR; import static org.apache.aurora.gen.ResponseCode.INVALID_REQUEST; import static org.apache.aurora.gen.ResponseCode.LOCK_ERROR; import static org.apache.aurora.gen.ResponseCode.OK; @@ -1512,11 +1511,6 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest { return response(OK, Optional.<Result>absent()); } - private static Response errorResponse(String message) { - return response(ERROR, Optional.<Result>absent()) - .setDetails(ImmutableList.of(new ResponseDetail().setMessage(message))); - } - private static Response invalidResponse(String message) { return Responses.empty() .setResponseCode(INVALID_REQUEST) @@ -2360,6 +2354,41 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest { assertResponse(AUTH_FAILED, thrift.pulseJobUpdate(UPDATE_KEY.newBuilder(), SESSION)); } + @Test + public void testMaxWaitToInstanceRunningMsDeprecationWarning() throws Exception { + expectAuth(ROLE, true); + expectNoCronJob(); + + ITaskConfig newTask = buildTaskForJobUpdate(0).getAssignedTask().getTask(); + expect(taskIdGenerator.generate(newTask, 1)).andReturn(TASK_ID); + + IJobUpdate update = buildJobUpdate( + 1, + newTask, + ImmutableMap.<ITaskConfig, ImmutableSet<Range>>of()); + + expect(quotaManager.checkJobUpdate( + update, + storageUtil.mutableStoreProvider)).andReturn(ENOUGH_QUOTA); + + expect(uuidGenerator.createNew()).andReturn(UU_ID); + storageUtil.expectTaskFetch(Query.unscoped().byJob(JOB_KEY).active()); + + jobUpdateController.start(update, AUDIT); + + control.replay(); + + // Validate key is populated during sanitizing. + JobUpdateRequest request = buildJobUpdateRequest(update); + request.getTaskConfig().unsetJob(); + + Response response = assertOkResponse(thrift.startJobUpdate(request, AUDIT_MESSAGE, SESSION)); + assertEquals( + ImmutableList.of( + new ResponseDetail(SchedulerThriftInterface.MAX_WAIT_TO_INSTANCE_FIELD_WARNING)), + response.getDetails()); + } + private static final String AUTH_DENIED_MESSAGE = "Denied!"; private IExpectationSetters<?> expectAuth(Set<String> roles, boolean allowed) http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/test/java/org/apache/aurora/scheduler/updater/InstanceUpdaterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/updater/InstanceUpdaterTest.java b/src/test/java/org/apache/aurora/scheduler/updater/InstanceUpdaterTest.java index 09c147e..3f1f72b 100644 --- a/src/test/java/org/apache/aurora/scheduler/updater/InstanceUpdaterTest.java +++ b/src/test/java/org/apache/aurora/scheduler/updater/InstanceUpdaterTest.java @@ -49,7 +49,7 @@ public class InstanceUpdaterTest { private static final ITaskConfig NEW = ITaskConfig.build(new TaskConfig().setNumCpus(2.0)); private static final Amount<Long, Time> MIN_RUNNING_TIME = Amount.of(1L, Time.MINUTES); - private static final Amount<Long, Time> MAX_NON_RUNNING_TIME = Amount.of(5L, Time.MINUTES); + private static final Amount<Long, Time> A_LONG_TIME = Amount.of(1L, Time.DAYS); private static class TestFixture { private final FakeClock clock; @@ -59,12 +59,7 @@ public class InstanceUpdaterTest { TestFixture(Optional<ITaskConfig> newConfig, int maxToleratedFailures) { this.clock = new FakeClock(); - this.updater = new InstanceUpdater( - newConfig, - maxToleratedFailures, - MIN_RUNNING_TIME, - MAX_NON_RUNNING_TIME, - clock); + this.updater = new InstanceUpdater(newConfig, maxToleratedFailures, MIN_RUNNING_TIME, clock); this.taskUtil = new TaskUtil(clock); } @@ -120,7 +115,8 @@ public class InstanceUpdaterTest { f.evaluate(EVALUATE_ON_STATE_CHANGE, KILLING); f.evaluate(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE, FINISHED); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING, ASSIGNED, STARTING, RUNNING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING, ASSIGNED, STARTING); + f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); f.advanceTime(MIN_RUNNING_TIME); f.evaluateCurrentState(SUCCEEDED); } @@ -133,9 +129,10 @@ public class InstanceUpdaterTest { f.evaluate(EVALUATE_ON_STATE_CHANGE, KILLING); f.evaluate(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE, FINISHED); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING, ASSIGNED, STARTING, RUNNING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING, ASSIGNED, STARTING); + f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); f.evaluate(EVALUATE_ON_STATE_CHANGE, FAILED); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING, ASSIGNED, STARTING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING, ASSIGNED, STARTING); f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); f.advanceTime(MIN_RUNNING_TIME); f.evaluateCurrentState(SUCCEEDED); @@ -149,7 +146,8 @@ public class InstanceUpdaterTest { f.evaluate(EVALUATE_ON_STATE_CHANGE, KILLING); f.evaluate(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE, FINISHED); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING, ASSIGNED, STARTING, RUNNING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING, ASSIGNED, STARTING); + f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); f.evaluate(Result.FAILED_TERMINATED, FAILED); } @@ -175,7 +173,8 @@ public class InstanceUpdaterTest { f.setActualStateAbsent(); f.evaluateCurrentState(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING, ASSIGNED, STARTING, RUNNING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING, ASSIGNED, STARTING); + f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); f.advanceTime(MIN_RUNNING_TIME); f.evaluateCurrentState(SUCCEEDED); } @@ -198,36 +197,31 @@ public class InstanceUpdaterTest { f.evaluate(EVALUATE_ON_STATE_CHANGE, KILLING); f.evaluate(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE, FINISHED); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING); - f.advanceTime(MAX_NON_RUNNING_TIME); - f.evaluateCurrentState(KILL_TASK_AND_EVALUATE_ON_STATE_CHANGE); - f.setActualStateAbsent(); - f.evaluateCurrentState(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE); - f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING); - f.advanceTime(MAX_NON_RUNNING_TIME); - f.evaluateCurrentState(Result.FAILED_STUCK); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING); + f.advanceTime(A_LONG_TIME); + f.evaluateCurrentState(EVALUATE_ON_STATE_CHANGE); } @Test - public void testStuckInKilling() { + public void testSlowToKill() { TestFixture f = new TestFixture(NEW, 1); f.setActualState(OLD); f.evaluate(KILL_TASK_AND_EVALUATE_ON_STATE_CHANGE, RUNNING); f.evaluate(EVALUATE_ON_STATE_CHANGE, KILLING); f.evaluate(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE, FINISHED); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING); - f.advanceTime(MAX_NON_RUNNING_TIME); - f.evaluateCurrentState(KILL_TASK_AND_EVALUATE_ON_STATE_CHANGE); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING); + f.advanceTime(A_LONG_TIME); + f.evaluateCurrentState(EVALUATE_ON_STATE_CHANGE); f.setActualStateAbsent(); f.evaluateCurrentState(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, ASSIGNED, STARTING, RUNNING); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, KILLING); - f.advanceTime(MAX_NON_RUNNING_TIME); - f.evaluateCurrentState(Result.FAILED_STUCK); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, ASSIGNED, STARTING); + f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, KILLING); + f.advanceTime(A_LONG_TIME); + f.evaluateCurrentState(EVALUATE_ON_STATE_CHANGE); } @Test(expected = IllegalArgumentException.class) @@ -264,7 +258,8 @@ public class InstanceUpdaterTest { f.setActualStateAbsent(); f.evaluateCurrentState(REPLACE_TASK_AND_EVALUATE_ON_STATE_CHANGE); f.setActualState(NEW); - f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, PENDING, ASSIGNED, STARTING, RUNNING); + f.evaluate(EVALUATE_ON_STATE_CHANGE, PENDING, ASSIGNED, STARTING); + f.evaluate(EVALUATE_AFTER_MIN_RUNNING_MS, RUNNING); f.advanceTime(MIN_RUNNING_TIME); f.evaluateCurrentState(SUCCEEDED); } http://git-wip-us.apache.org/repos/asf/aurora/blob/5587bce7/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java index 312fd2b..dd4c940 100644 --- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java +++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java @@ -1115,8 +1115,8 @@ public class JobUpdaterIT extends EasyMockTest { } @Test - public void testStuckTask() throws Exception { - expectTaskKilled().times(3); + public void testSlowToScheduleTask() throws Exception { + expectTaskKilled().times(2); control.replay(); @@ -1134,28 +1134,16 @@ public class JobUpdaterIT extends EasyMockTest { changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING); clock.advance(WATCH_TIMEOUT); - // Instance 1 is stuck in PENDING. + // Instance 1 is not advancing past PENDING. changeState(JOB, 1, KILLED); actions.putAll(0, INSTANCE_UPDATED) .putAll(1, INSTANCE_UPDATING); assertState(ROLLING_FORWARD, actions.build()); clock.advance(RUNNING_TIMEOUT); - actions.putAll(1, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK); - assertState(ROLLING_BACK, actions.build()); - assertLatestUpdateMessage(JobUpdateControllerImpl.failureMessage(1, Failure.STUCK)); - - // Instance 1 is reverted. - changeState(JOB, 1, ASSIGNED, STARTING, RUNNING); - clock.advance(WATCH_TIMEOUT); - - // Instance 0 is reverted. - changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING); - clock.advance(WATCH_TIMEOUT); + assertState(ROLLING_FORWARD, actions.build()); - actions.putAll(0, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK) - .putAll(1, INSTANCE_ROLLED_BACK); - assertState(ROLLED_BACK, actions.build()); - assertJobState(JOB, ImmutableMap.of(0, OLD_CONFIG, 1, OLD_CONFIG)); + updater.abort(update.getSummary().getKey(), AUDIT); + assertState(ABORTED, actions.build()); } @Test
