Repository: incubator-myriad Updated Branches: refs/heads/master 832cf0589 -> 420cb78a6
MYRIAD-164 Remove Myriad state after framework shutdown New Myriad framework fails to start after an earlier one is shutdown. The Myriad state saves the framework id for the old framework. When the new framework starts, it tries to re-registers with the old framework id. This causes an error message similar to I1029 12:07:08.330493 15884 sched.cpp:819] Got error 'Completed framework attempted to re-register' I1029 12:07:08.330507 15884 sched.cpp:1625] Asked to abort the driver I1029 12:07:08.331507 15884 sched.cpp:861] Aborting framework '20151029-110656-1986333194-5050-10105-0001' We need to remove the Myriad state as part of framework shutdown. Added a method removeMyriadState to MyriadStateStore interface and added implementation in MyriadFileSystemRMStateStore. This closes: #31 Review: https://github.com/apache/incubator-myriad/pull/31 Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/420cb78a Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/420cb78a Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/420cb78a Branch: refs/heads/master Commit: 420cb78a6d24e1016647f1f81d4421c52dc49130 Parents: 832cf05 Author: Swapnil Daingade <sdaing...@maprtech.com> Authored: Wed Nov 4 16:52:48 2015 -0800 Committer: Santosh Marella <mare...@gmail.com> Committed: Wed Nov 4 16:52:48 2015 -0800 ---------------------------------------------------------------------- .../recovery/MyriadFileSystemRMStateStore.java | 11 ++++++++ .../java/org/apache/myriad/MyriadModule.java | 4 ++- .../myriad/scheduler/MyriadOperations.java | 28 ++++++++++++++------ .../apache/myriad/state/MyriadStateStore.java | 1 + 4 files changed, 35 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/420cb78a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java index 85fed08..dda9b28 100644 --- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java +++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java @@ -97,4 +97,15 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme LOGGER.error("State information for Myriad could not be stored at: " + myriadStatePath, e); } } + + @Override + public synchronized void removeMyriadState() throws Exception { + if (fs.exists(myriadPathRoot)) { + if (!fs.delete(myriadPathRoot, true)) { + LOGGER.error("Failed to delete MyriadStateStore path " + myriadPathRoot); + return; + } + LOGGER.info("Deleted Myriad state store path " + myriadPathRoot); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/420cb78a/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java b/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java index 41abb9a..5896e93 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/MyriadModule.java @@ -146,7 +146,9 @@ public class MyriadModule extends AbstractModule { return new SchedulerState(myriadStateStore); } - private MyriadStateStore providesMyriadStateStore() { + @Provides + @Singleton + MyriadStateStore providesMyriadStateStore() { // TODO (sdaingade) Read the implementation class from yml // once multiple implementations are available. if (rmContext.getStateStore() instanceof MyriadStateStore) { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/420cb78a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/MyriadOperations.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/MyriadOperations.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/MyriadOperations.java index 34367ef..c662902 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/MyriadOperations.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/MyriadOperations.java @@ -33,6 +33,7 @@ import org.apache.myriad.configuration.ServiceConfiguration; import org.apache.myriad.policy.NodeScaleDownPolicy; import org.apache.myriad.scheduler.constraints.Constraint; import org.apache.myriad.scheduler.constraints.LikeConstraint; +import org.apache.myriad.state.MyriadStateStore; import org.apache.myriad.state.NodeTask; import org.apache.myriad.state.SchedulerState; import org.apache.myriad.webapp.MyriadWebServer; @@ -50,15 +51,18 @@ public class MyriadOperations { private NodeScaleDownPolicy nodeScaleDownPolicy; private MyriadDriverManager driverManager; private MyriadWebServer myriadWebServer; + private MyriadStateStore myriadStateStore; @Inject - public MyriadOperations(MyriadConfiguration cfg, SchedulerState schedulerState, NodeScaleDownPolicy nodeScaleDownPolicy, - MyriadDriverManager driverManager, MyriadWebServer myriadWebServer) { + public MyriadOperations(MyriadConfiguration cfg, SchedulerState schedulerState, + NodeScaleDownPolicy nodeScaleDownPolicy, MyriadDriverManager driverManager, + MyriadWebServer myriadWebServer, MyriadStateStore myriadStateStore) { this.cfg = cfg; this.schedulerState = schedulerState; this.nodeScaleDownPolicy = nodeScaleDownPolicy; this.driverManager = driverManager; this.myriadWebServer = myriadWebServer; + this.myriadStateStore = myriadStateStore; } public void flexUpCluster(ServiceResourceProfile serviceResourceProfile, int instances, Constraint constraint) { @@ -255,6 +259,7 @@ public class MyriadOperations { * Shutdown framework means the Mesos driver is stopped taking down the executors and associated tasks */ public void shutdownFramework() { + LOGGER.info("Received request to shutdown Myriad Framework.."); Status driverStatus = driverManager.getDriverStatus(); if (Status.DRIVER_RUNNING != driverStatus) { @@ -263,13 +268,20 @@ public class MyriadOperations { // Stop the driver, tasks, and executor. driverStatus = driverManager.stopDriver(false); LOGGER.info("Myriad driver was shutdown with status " + driverStatus); - try { - myriadWebServer.stop(); - } catch (Exception e) { - LOGGER.info("Failed to shutdown Myriad webserver: " + e.getMessage()); - return; - } + } + + try { + myriadWebServer.stop(); LOGGER.info("Myriad webserver was shutdown successfully"); + } catch (Exception e) { + LOGGER.info("Failed to shutdown Myriad webserver: " + e.getMessage()); + } + + try { + myriadStateStore.removeMyriadState(); + LOGGER.info("Myriad State store was removed successfully."); + } catch (Exception e) { + LOGGER.info("Failed to remove Myriad state store: " + e.getMessage()); } } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/420cb78a/myriad-scheduler/src/main/java/org/apache/myriad/state/MyriadStateStore.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/state/MyriadStateStore.java b/myriad-scheduler/src/main/java/org/apache/myriad/state/MyriadStateStore.java index 69b0238..73a0c8e 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/state/MyriadStateStore.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/state/MyriadStateStore.java @@ -30,4 +30,5 @@ public interface MyriadStateStore { void storeMyriadState(StoreContext storeContext) throws Exception; + void removeMyriadState() throws Exception; }
