Repository: aurora Updated Branches: refs/heads/master 297c0eb48 -> e9d723d9c
Use explicit status update acknowledgements. Bugs closed: AURORA-1228 Reviewed at https://reviews.apache.org/r/33243/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/e9d723d9 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/e9d723d9 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/e9d723d9 Branch: refs/heads/master Commit: e9d723d9cc846aab58d389ad71c29b662d313ed1 Parents: 297c0eb Author: Ben Mahler <[email protected]> Authored: Mon Apr 27 16:49:34 2015 -0700 Committer: Bill Farner <[email protected]> Committed: Mon Apr 27 16:49:34 2015 -0700 ---------------------------------------------------------------------- .../aurora/scheduler/mesos/DriverFactoryImpl.java | 14 +++++++++++--- .../scheduler/mesos/MesosSchedulerImpl.java | 11 ++++++----- .../scheduler/mesos/MesosSchedulerImplTest.java | 18 +++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/e9d723d9/src/main/java/org/apache/aurora/scheduler/mesos/DriverFactoryImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/DriverFactoryImpl.java b/src/main/java/org/apache/aurora/scheduler/mesos/DriverFactoryImpl.java index db7aa74..a5f5e99 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/DriverFactoryImpl.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/DriverFactoryImpl.java @@ -34,11 +34,19 @@ class DriverFactoryImpl implements DriverFactory { Optional<Protos.Credential> credentials, FrameworkInfo frameworkInfo, String master) { - if (credentials.isPresent()) { - return new MesosSchedulerDriver(scheduler, frameworkInfo, master, credentials.get()); + return new MesosSchedulerDriver( + scheduler, + frameworkInfo, + master, + false, // Disable implicit acknowledgements. + credentials.get()); } else { - return new MesosSchedulerDriver(scheduler, frameworkInfo, master); + return new MesosSchedulerDriver( + scheduler, + frameworkInfo, + master, + false); // Disable implicit acknowledgements. } } } http://git-wip-us.apache.org/repos/asf/aurora/blob/e9d723d9/src/main/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImpl.java b/src/main/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImpl.java index f40746b..9b8ab7c 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImpl.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImpl.java @@ -228,21 +228,22 @@ class MesosSchedulerImpl implements Scheduler { try { for (TaskLauncher launcher : taskLaunchers) { if (launcher.statusUpdate(status)) { + driver.acknowledgeStatusUpdate(status); return; } } } catch (SchedulerException e) { log.log(Level.SEVERE, "Status update failed due to scheduler exception: " + e, e); - // We re-throw the exception here in an effort to NACK the status update and trigger an - // abort of the driver. Previously we directly issued driver.abort(), but the re-entrancy - // guarantees of that are uncertain (and we believe it was not working). However, this - // was difficult to discern since logging is unreliable during JVM shutdown and we would not - // see the above log message. + // We re-throw the exception here to trigger an abort of the driver. throw e; } log.warning("Unhandled status update " + status); counters.get("scheduler_status_updates").incrementAndGet(); + + // Even though we have not handled this update, we acknowledge it to prevent an unbounded + // growth of status update retries from Mesos. + driver.acknowledgeStatusUpdate(status); } @Override http://git-wip-us.apache.org/repos/asf/aurora/blob/e9d723d9/src/test/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImplTest.java index 4ef23ea..abdeee4 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosSchedulerImplTest.java @@ -44,6 +44,7 @@ import org.apache.aurora.scheduler.storage.Storage.StorageException; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.testing.StorageTestUtil; import org.apache.aurora.scheduler.testing.FakeStatsProvider; +import org.apache.mesos.Protos; import org.apache.mesos.Protos.ExecutorID; import org.apache.mesos.Protos.FrameworkID; import org.apache.mesos.Protos.MasterInfo; @@ -246,8 +247,9 @@ public class MesosSchedulerImplTest extends EasyMockTest { @Override void expectations() { eventSink.post(PUBSUB_EVENT); - expect(systemLauncher.statusUpdate(STATUS)).andReturn(false); - expect(userLauncher.statusUpdate(STATUS)).andReturn(false); + expect(systemLauncher.statusUpdate(status)).andReturn(false); + expect(userLauncher.statusUpdate(status)).andReturn(false); + expect(driver.acknowledgeStatusUpdate(status)).andReturn(Protos.Status.DRIVER_RUNNING); } }.run(); } @@ -260,12 +262,13 @@ public class MesosSchedulerImplTest extends EasyMockTest { @Override void expectations() { eventSink.post(new TaskStatusReceived( - STATUS.getState(), + status.getState(), Optional.fromNullable(status.getSource()), Optional.fromNullable(status.getReason()), Optional.of(1000000L) )); expect(systemLauncher.statusUpdate(status)).andReturn(true); + expect(driver.acknowledgeStatusUpdate(status)).andReturn(Protos.Status.DRIVER_RUNNING); } } @@ -290,8 +293,9 @@ public class MesosSchedulerImplTest extends EasyMockTest { @Override void expectations() { eventSink.post(PUBSUB_EVENT); - expect(systemLauncher.statusUpdate(STATUS)).andReturn(false); - expect(userLauncher.statusUpdate(STATUS)).andReturn(true); + expect(systemLauncher.statusUpdate(status)).andReturn(false); + expect(userLauncher.statusUpdate(status)).andReturn(true); + expect(driver.acknowledgeStatusUpdate(status)).andReturn(Protos.Status.DRIVER_RUNNING); } }.run(); } @@ -302,8 +306,8 @@ public class MesosSchedulerImplTest extends EasyMockTest { @Override void expectations() { eventSink.post(PUBSUB_EVENT); - expect(systemLauncher.statusUpdate(STATUS)).andReturn(false); - expect(userLauncher.statusUpdate(STATUS)).andThrow(new StorageException("Injected.")); + expect(systemLauncher.statusUpdate(status)).andReturn(false); + expect(userLauncher.statusUpdate(status)).andThrow(new StorageException("Injected.")); } }.run(); }
