SLIDER-572 add startup delay for the chaos monkey
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/517042f2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/517042f2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/517042f2 Branch: refs/heads/develop Commit: 517042f2e6878445610884eaa1b47fa3a349ed17 Parents: 25bbfeb Author: Steve Loughran <[email protected]> Authored: Wed Oct 29 20:46:32 2014 +0000 Committer: Steve Loughran <[email protected]> Committed: Fri Oct 31 11:07:49 2014 +0000 ---------------------------------------------------------------------- .../org/apache/slider/api/InternalKeys.java | 11 ++++++++- .../apache/slider/core/conf/MapOperations.java | 2 +- .../server/appmaster/SliderAppMaster.java | 24 ++++++++++---------- .../appmaster/monkey/ChaosMonkeyService.java | 20 +++++++++------- .../model/monkey/TestMockMonkey.groovy | 4 ++-- 5 files changed, 37 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index 5f150e6..074644d 100644 --- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -121,6 +121,14 @@ public interface InternalKeys { int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0; int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0; + String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay"; + String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days"; + String CHAOS_MONKEY_DELAY_HOURS = CHAOS_MONKEY_DELAY + ".hours"; + String CHAOS_MONKEY_DELAY_MINUTES = CHAOS_MONKEY_DELAY + ".minutes"; + String CHAOS_MONKEY_DELAY_SECONDS = CHAOS_MONKEY_DELAY + ".seconds"; + + int DEFAULT_CHAOS_MONKEY_STARTUP_DELAY = 0; + /** * Prefix for all chaos monkey probabilities */ @@ -133,7 +141,8 @@ public interface InternalKeys { /** * Probability of a monkey check killing the AM: {@value} */ - String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = CHAOS_MONKEY_PROBABILITY +".amfailure"; + String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = + CHAOS_MONKEY_PROBABILITY + ".amfailure"; /** * Default probability of a monkey check killing the AM: {@value} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java index 6503c9b..de8fc2c 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java @@ -282,7 +282,7 @@ public class MapOperations implements Map<String, String> { * @param defHours * @param defMins * @param defSecs - * @return + * @return the aggregate time range in seconds */ public long getTimeRange(String basekey, int defDays, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 39a2572..855ecd7 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1101,14 +1101,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } -/* - - @Override - protected RegistryOperationsService createRegistryOperationsInstance() { - return new ResourceManagerRegistryService("YarnRegistry"); - } -*/ - /** * TODO: purge this once RM is doing the work * @throws IOException @@ -1133,7 +1125,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService if (instance == null) { return false; } - // this is where component registrations will go + // this is where component registrations go log.info("Registering component {}", id); String cid = RegistryPathUtils.encodeYarnID(id.toString()); ServiceRecord container = new ServiceRecord(); @@ -2185,8 +2177,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService "Chaos monkey not configured with a time interval...not enabling"); return false; } - log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)", - monkeyInterval, monkeyInterval/(60*60)); + + long monkeyDelay = internals.getTimeRange( + InternalKeys.CHAOS_MONKEY_DELAY, + 0, + 0, + 0, + (int)monkeyInterval); + + log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours -delay {}", + monkeyInterval, monkeyInterval/(60*60), monkeyDelay); monkey = new ChaosMonkeyService(metrics, actionQueues); initAndAddService(monkey); @@ -2204,7 +2204,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService containerKillProbability); // and schedule it - if (monkey.schedule(monkeyInterval, TimeUnit.SECONDS)) { + if (monkey.schedule(monkeyDelay, monkeyInterval, TimeUnit.SECONDS)) { log.info("Chaos Monkey is running"); return true; } else { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java index f7b1eb7..80f981c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java @@ -92,13 +92,14 @@ public class ChaosMonkeyService extends AbstractService { /** * Schedule the monkey - * @param time interval + * + * @param delay initial delay * @param timeUnit time unit * @return true if it was scheduled (i.e. 1+ action) and interval > 0 */ - public boolean schedule(long time, TimeUnit timeUnit) { - if (time > 0 && !chaosEntries.isEmpty()) { - queues.schedule(getChaosAction(time, timeUnit)); + public boolean schedule(long delay, long interval, TimeUnit timeUnit) { + if (interval > 0 && !chaosEntries.isEmpty()) { + queues.schedule(getChaosAction(delay, interval, timeUnit)); return true; } else { return false; @@ -107,15 +108,18 @@ public class ChaosMonkeyService extends AbstractService { /** * Get the chaos action - * @param time interval + * + * @param delay * @param timeUnit time unit * @return the action to schedule */ - public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) { + public RenewingAction<MonkeyPlayAction> getChaosAction(long delay, + long interval, + TimeUnit timeUnit) { RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>( new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS), - time, - time, + delay, + interval, timeUnit, 0 ); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy index 26d07ee..d31c9f6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy @@ -78,7 +78,7 @@ class TestMockMonkey extends BaseMockAppStateTest { assert 0 == monkey.targetCount; monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100) assert 1 == monkey.targetCount; - assert monkey.schedule(1, TimeUnit.SECONDS) + assert monkey.schedule(0, 1, TimeUnit.SECONDS) assert 1 == queues.scheduledActions.size() } @@ -96,7 +96,7 @@ class TestMockMonkey extends BaseMockAppStateTest { public void testMonkeyScheduleProb0Actions() throws Throwable { ChaosCounter counter = new ChaosCounter() monkey.addTarget("target", counter, 0) - assert !monkey.schedule(1, TimeUnit.SECONDS) + assert !monkey.schedule(0, 1, TimeUnit.SECONDS) assert 0 == queues.scheduledActions.size() }
