Shim interfaces to preface args system overhaul. Reviewed at https://reviews.apache.org/r/41804/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/fe13e4ed Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/fe13e4ed Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/fe13e4ed Branch: refs/heads/master Commit: fe13e4ed52d4dc0a35f9e50b5e49c6e705f64579 Parents: 317aa4d Author: Bill Farner <[email protected]> Authored: Tue Jan 19 14:05:48 2016 -0800 Committer: Bill Farner <[email protected]> Committed: Tue Jan 19 14:05:48 2016 -0800 ---------------------------------------------------------------------- config/legacy_untested_classes.txt | 9 +- .../aurora/benchmark/SchedulingBenchmarks.java | 13 ++- .../aurora/scheduler/SchedulerModule.java | 52 +++++++-- .../apache/aurora/scheduler/app/AppModule.java | 42 +++++++- .../aurora/scheduler/app/SchedulerMain.java | 48 +++++++-- .../aurora/scheduler/async/AsyncModule.java | 16 ++- .../configuration/executor/ExecutorModule.java | 101 ++++++++++++++---- .../scheduler/cron/quartz/CronModule.java | 50 +++++++-- .../aurora/scheduler/http/H2ConsoleModule.java | 15 ++- .../scheduler/http/JettyServerModule.java | 49 ++++++--- .../aurora/scheduler/http/api/ApiModule.java | 21 +++- .../http/api/security/HttpSecurityModule.java | 41 +++++-- .../http/api/security/IniShiroRealmModule.java | 15 ++- .../api/security/Kerberos5ShiroRealmModule.java | 33 +++++- .../log/mesos/MesosLogStreamModule.java | 106 ++++++++++++++----- .../mesos/CommandLineDriverSettingsModule.java | 87 ++++++++++++--- .../aurora/scheduler/offers/OffersModule.java | 25 ++++- .../scheduler/preemptor/PreemptorModule.java | 65 ++++++++---- .../aurora/scheduler/pruning/PruningModule.java | 63 +++++++++-- .../reconciliation/ReconciliationModule.java | 64 +++++++++-- .../scheduler/scheduling/SchedulingModule.java | 92 ++++++++++++++-- .../apache/aurora/scheduler/sla/SlaModule.java | 45 +++++--- .../scheduler/stats/AsyncStatsModule.java | 41 +++++-- .../aurora/scheduler/stats/StatsModule.java | 26 ++++- .../scheduler/storage/backup/BackupModule.java | 42 ++++++-- .../aurora/scheduler/storage/db/DbModule.java | 52 +++++++-- .../scheduler/storage/log/LogStorageModule.java | 33 +++++- .../storage/mem/InMemStoresModule.java | 13 ++- .../client/flagged/FlaggedClientConfig.java | 56 ++++++++-- .../aurora/scheduler/app/SchedulerIT.java | 14 ++- .../scheduler/app/local/LocalSchedulerMain.java | 14 ++- .../scheduler/http/AbstractJettyTest.java | 2 +- .../http/api/security/HttpSecurityIT.java | 4 +- .../preemptor/PreemptorModuleTest.java | 24 ++++- .../aurora/scheduler/sla/SlaModuleTest.java | 21 +++- 35 files changed, 1168 insertions(+), 226 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/config/legacy_untested_classes.txt ---------------------------------------------------------------------- diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt index 6b71fd2..30d4b13 100644 --- a/config/legacy_untested_classes.txt +++ b/config/legacy_untested_classes.txt @@ -9,6 +9,7 @@ org/apache/aurora/scheduler/app/SchedulerMain$2 org/apache/aurora/scheduler/app/SchedulerMain$2$1 org/apache/aurora/scheduler/app/SchedulerMain$3 org/apache/aurora/scheduler/app/SchedulerMain$4 +org/apache/aurora/scheduler/app/SchedulerMain$Params org/apache/aurora/scheduler/async/OfferQueue$OfferQueueImpl$2 org/apache/aurora/scheduler/base/Conversions$2 org/apache/aurora/scheduler/base/Conversions$3 @@ -55,16 +56,21 @@ org/apache/aurora/scheduler/http/Utilization$5 org/apache/aurora/scheduler/http/Utilization$Display org/apache/aurora/scheduler/http/Utilization$DisplayMetric org/apache/aurora/scheduler/http/api/security/FieldGetter$IdentityFieldGetter +org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule$1 org/apache/aurora/scheduler/http/api/security/Kerberos5Realm +org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule$1 org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule -org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$3 +org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$1 org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$4 org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$5 +org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule$1 org/apache/aurora/scheduler/mesos/DriverFactoryImpl org/apache/aurora/scheduler/mesos/LibMesosLoadingModule +org/apache/aurora/scheduler/preemptor/PreemptorModule$Params org/apache/aurora/scheduler/stats/AsyncStatsModule$OfferAdapter$1 org/apache/aurora/scheduler/stats/TaskStatCalculator org/apache/aurora/scheduler/storage/CallOrderEnforcingStorage$State +org/apache/aurora/scheduler/storage/backup/BackupModule$1 org/apache/aurora/scheduler/storage/backup/BackupModule$LifecycleHook org/apache/aurora/scheduler/storage/mem/MemTaskStore$Task org/apache/aurora/scheduler/storage/mem/Util @@ -75,3 +81,4 @@ org/apache/aurora/scheduler/zookeeper/guice/client/ZooKeeperClientModule$1 org/apache/aurora/scheduler/zookeeper/guice/client/ZooKeeperClientModule$LocalClientProvider org/apache/aurora/scheduler/zookeeper/guice/client/ZooKeeperClientModule$TestServerService org/apache/aurora/scheduler/zookeeper/guice/client/flagged/FlaggedClientConfig +org/apache/aurora/scheduler/zookeeper/guice/client/flagged/FlaggedClientConfig$1 http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java index 45ab76b..3ce266a 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java +++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java @@ -108,7 +108,18 @@ public class SchedulingBenchmarks { // TODO(maxim): Find a way to DRY it and reuse existing modules instead. Injector injector = Guice.createInjector( new StateModule(), - new PreemptorModule(true, NO_DELAY, NO_DELAY), + new PreemptorModule( + new PreemptorModule.Params() { + @Override + public Amount<Long, Time> preemptionDelay() { + return NO_DELAY; + } + + @Override + public Amount<Long, Time> preemptionSlotSearchInterval() { + return NO_DELAY; + } + }), new PrivateModule() { @Override protected void configure() { http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java index ddc0d05..83e9060 100644 --- a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java @@ -74,7 +74,43 @@ public class SchedulerModule extends AbstractModule { @CanRead @CmdLine(name = "tier_config", help = "Configuration file defining supported task tiers, task traits and behaviors.") - private static final Arg<File> TIER_CONFIG_FILE = Arg.create(); + private static final Arg<File> TIER_CONFIG_FILE = Arg.create(null); + + interface Params { + Amount<Long, Time> maxRegistrationDelay(); + + Amount<Long, Time> maxLeadingDuration(); + + int maxStatusUpdateBatchSize(); + + Optional<File> tierConfig(); + } + + private final Params params; + + public SchedulerModule() { + this.params = new Params() { + @Override + public Amount<Long, Time> maxRegistrationDelay() { + return MAX_REGISTRATION_DELAY.get(); + } + + @Override + public Amount<Long, Time> maxLeadingDuration() { + return MAX_LEADING_DURATION.get(); + } + + @Override + public int maxStatusUpdateBatchSize() { + return MAX_STATUS_UPDATE_BATCH_SIZE.get(); + } + + @Override + public Optional<File> tierConfig() { + return Optional.fromNullable(TIER_CONFIG_FILE.get()); + } + }; + } @Override protected void configure() { @@ -84,7 +120,7 @@ public class SchedulerModule extends AbstractModule { @Override protected void configure() { bind(LeadingOptions.class).toInstance( - new LeadingOptions(MAX_REGISTRATION_DELAY.get(), MAX_LEADING_DURATION.get())); + new LeadingOptions(params.maxRegistrationDelay(), params.maxLeadingDuration())); final ScheduledExecutorService executor = AsyncUtil.singleThreadLoggingScheduledExecutor("Lifecycle-%d", LOG); @@ -105,7 +141,7 @@ public class SchedulerModule extends AbstractModule { .toInstance(new LinkedBlockingQueue<>()); bind(new TypeLiteral<Integer>() { }) .annotatedWith(TaskStatusHandlerImpl.MaxBatchSize.class) - .toInstance(MAX_STATUS_UPDATE_BATCH_SIZE.get()); + .toInstance(params.maxStatusUpdateBatchSize()); bind(TaskStatusHandler.class).to(TaskStatusHandlerImpl.class); bind(TaskStatusHandlerImpl.class).in(Singleton.class); @@ -116,17 +152,15 @@ public class SchedulerModule extends AbstractModule { addSchedulerActiveServiceBinding(binder()).to(TaskStatusHandlerImpl.class); } - private static Optional<String> readTierFile() { - if (TIER_CONFIG_FILE.hasAppliedValue()) { + private Optional<String> readTierFile() { + return params.tierConfig().transform(file -> { try { - return Optional.of(Files.toString(TIER_CONFIG_FILE.get(), StandardCharsets.UTF_8)); + return Files.toString(file, StandardCharsets.UTF_8); } catch (IOException e) { LOG.error("Error loading tier configuration file."); throw Throwables.propagate(e); } - } - - return Optional.<String>absent(); + }); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/app/AppModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java index a25fa41..58026e1 100644 --- a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java +++ b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java @@ -81,14 +81,50 @@ public class AppModule extends AbstractModule { help = "Allow to pass docker container parameters in the job.") private static final Arg<Boolean> ENABLE_DOCKER_PARAMETERS = Arg.create(false); + public interface Params { + int maxTasksPerJob(); + + int maxUpdateInstanceFailures(); + + Set<_Fields> allowedContainerTypes(); + + boolean enableDockerParameters(); + } + + private final Params params; + + public AppModule() { + this.params = new Params() { + @Override + public int maxTasksPerJob() { + return MAX_TASKS_PER_JOB.get(); + } + + @Override + public int maxUpdateInstanceFailures() { + return MAX_UPDATE_INSTANCE_FAILURES.get(); + } + + @Override + public Set<_Fields> allowedContainerTypes() { + return ALLOWED_CONTAINER_TYPES.get(); + } + + @Override + public boolean enableDockerParameters() { + return ENABLE_DOCKER_PARAMETERS.get(); + } + }; + } + @Override protected void configure() { bind(ConfigurationManager.class).toInstance( new ConfigurationManager( - ImmutableSet.copyOf(ALLOWED_CONTAINER_TYPES.get()), - ENABLE_DOCKER_PARAMETERS.get())); + ImmutableSet.copyOf(params.allowedContainerTypes()), + params.enableDockerParameters())); bind(Thresholds.class) - .toInstance(new Thresholds(MAX_TASKS_PER_JOB.get(), MAX_UPDATE_INSTANCE_FAILURES.get())); + .toInstance(new Thresholds(params.maxTasksPerJob(), params.maxUpdateInstanceFailures())); // Enable intercepted method timings and context classloader repair. TimedInterceptor.bind(binder()); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java index 0659c35..0861c4e 100644 --- a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java +++ b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java @@ -92,6 +92,20 @@ public class SchedulerMain { @CmdLine(name = "viz_job_url_prefix", help = "URL prefix for job container stats.") private static final Arg<String> STATS_URL_PREFIX = Arg.create(""); + public interface Params { + String clusterName(); + + String serversetPath(); + + default List<Class<? extends Module>> extraModules() { + return ImmutableList.of(); + } + + default String statsUrlPrefix() { + return ""; + } + } + @Inject private SingletonService schedulerService; @Inject private HttpService httpService; @Inject private SchedulerLifecycle schedulerLifecycle; @@ -155,7 +169,7 @@ public class SchedulerMain { * @param appEnvironmentModule Additional modules based on the execution environment. */ @VisibleForTesting - public static void flagConfiguredMain(Module appEnvironmentModule) { + public static void main(Params params, Module appEnvironmentModule) { AtomicLong uncaughtExceptions = Stats.exportLong("uncaught_exceptions"); Thread.setDefaultUncaughtExceptionHandler((t, e) -> { uncaughtExceptions.incrementAndGet(); @@ -167,7 +181,7 @@ public class SchedulerMain { appEnvironmentModule, getUniversalModule(), new ZooKeeperClientModule(zkClientConfig), - new ServiceDiscoveryModule(SERVERSET_PATH.get(), zkClientConfig.credentials), + new ServiceDiscoveryModule(params.serversetPath(), zkClientConfig.credentials), new BackupModule(SnapshotStoreImpl.class), new ExecutorModule(), new AbstractModule() { @@ -176,8 +190,8 @@ public class SchedulerMain { bind(IServerInfo.class).toInstance( IServerInfo.build( new ServerInfo() - .setClusterName(CLUSTER_NAME.get()) - .setStatsUrlPrefix(STATS_URL_PREFIX.get()))); + .setClusterName(params.clusterName()) + .setStatsUrlPrefix(params.statsUrlPrefix()))); } }); @@ -202,15 +216,37 @@ public class SchedulerMain { public static void main(String... args) { applyStaticArgumentValues(args); + Params params = new Params() { + @Override + public String clusterName() { + return CLUSTER_NAME.get(); + } + + @Override + public String serversetPath() { + return SERVERSET_PATH.get(); + } + + @Override + public List<Class<? extends Module>> extraModules() { + return EXTRA_MODULES.get(); + } + + @Override + public String statsUrlPrefix() { + return STATS_URL_PREFIX.get(); + } + }; + List<Module> modules = ImmutableList.<Module>builder() .add( new CommandLineDriverSettingsModule(), new LibMesosLoadingModule(), new MesosLogStreamModule(FlaggedClientConfig.create()), new LogStorageModule()) - .addAll(Iterables.transform(EXTRA_MODULES.get(), MoreModules::getModule)) + .addAll(Iterables.transform(params.extraModules(), MoreModules::getModule)) .build(); - flagConfiguredMain(Modules.combine(modules)); + main(params, Modules.combine(modules)); } private static void exit(String message, Exception error) { http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java b/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java index da07df6..d307861 100644 --- a/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java +++ b/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java @@ -53,14 +53,28 @@ public class AsyncModule extends AbstractModule { private static final Arg<Integer> ASYNC_WORKER_THREADS = Arg.create(8); private final ScheduledThreadPoolExecutor afterTransaction; + interface Params { + int asyncWorkerThreads(); + } + @Qualifier @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME) public @interface AsyncExecutor { } public AsyncModule() { + this(new Params() { + @Override + public int asyncWorkerThreads() { + return ASYNC_WORKER_THREADS.get(); + } + }); + } + + private AsyncModule(Params params) { // Don't worry about clean shutdown, these can be daemon and cleanup-free. // TODO(wfarner): Should we use a bounded caching thread pool executor instead? - this(AsyncUtil.loggingScheduledExecutor(ASYNC_WORKER_THREADS.get(), "AsyncProcessor-%d", LOG)); + this( + AsyncUtil.loggingScheduledExecutor(params.asyncWorkerThreads(), "AsyncProcessor-%d", LOG)); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java index 949c299..8c58c5b 100644 --- a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java @@ -15,7 +15,6 @@ package org.apache.aurora.scheduler.configuration.executor; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.List; import java.util.Optional; @@ -96,32 +95,96 @@ public class ExecutorModule extends AbstractModule { + "into all (non-mesos) containers.") private static final Arg<List<Volume>> GLOBAL_CONTAINER_MOUNTS = Arg.create(ImmutableList.of()); - private static CommandInfo makeExecutorCommand() { + public interface Params { + Optional<File> customExecutorConfig(); + + String thermosExecutorPath(); + + List<String> thermosExecutorResources(); + + Optional<String> thermosExecutorFlags(); + + String thermosObserverRoot(); + + double executorOverheadCpus(); + + Amount<Long, Data> executorOverheadRam(); + + List<Volume> globalContainerMounts(); + } + + private final Params params; + + public ExecutorModule() { + this.params = new Params() { + @Override + public Optional<File> customExecutorConfig() { + return Optional.ofNullable(CUSTOM_EXECUTOR_CONFIG.get()); + } + + @Override + public String thermosExecutorPath() { + return THERMOS_EXECUTOR_PATH.get(); + } + + @Override + public List<String> thermosExecutorResources() { + return THERMOS_EXECUTOR_RESOURCES.get(); + } + + @Override + public Optional<String> thermosExecutorFlags() { + return Optional.ofNullable(THERMOS_EXECUTOR_FLAGS.get()); + } + + @Override + public String thermosObserverRoot() { + return THERMOS_OBSERVER_ROOT.get(); + } + + @Override + public double executorOverheadCpus() { + return EXECUTOR_OVERHEAD_CPUS.get(); + } + + @Override + public Amount<Long, Data> executorOverheadRam() { + return EXECUTOR_OVERHEAD_RAM.get(); + } + + @Override + public List<Volume> globalContainerMounts() { + return GLOBAL_CONTAINER_MOUNTS.get(); + } + }; + } + + private CommandInfo makeExecutorCommand() { Stream<String> resourcesToFetch = Stream.concat( - ImmutableList.of(THERMOS_EXECUTOR_PATH.get()).stream(), - THERMOS_EXECUTOR_RESOURCES.get().stream()); + ImmutableList.of(params.thermosExecutorPath()).stream(), + params.thermosExecutorResources().stream()); return CommandInfo.newBuilder() // Default to the value of $MESOS_SANDBOX if present. This is necessary for docker tasks, // in which case the mesos agent is responsible for setting $MESOS_SANDBOX. - .setValue("${MESOS_SANDBOX=.}/" + uriBasename(THERMOS_EXECUTOR_PATH.get()) - + " " + Optional.ofNullable(THERMOS_EXECUTOR_FLAGS.get()).orElse("")) + .setValue("${MESOS_SANDBOX=.}/" + uriBasename(params.thermosExecutorPath()) + + " " + params.thermosExecutorFlags().orElse("")) .addAllUris(resourcesToFetch .map(r -> URI.newBuilder().setValue(r).setExecutable(true).build()) .collect(GuavaUtils.toImmutableList())) .build(); } - private static ExecutorSettings makeThermosExecutorSettings() { + private ExecutorSettings makeThermosExecutorSettings() { List<Protos.Volume> volumeMounts = ImmutableList.<Protos.Volume>builder() .add(Protos.Volume.newBuilder() - .setHostPath(THERMOS_OBSERVER_ROOT.get()) - .setContainerPath(THERMOS_OBSERVER_ROOT.get()) + .setHostPath(params.thermosObserverRoot()) + .setContainerPath(params.thermosObserverRoot()) .setMode(Protos.Volume.Mode.RW) .build()) .addAll(Iterables.transform( - GLOBAL_CONTAINER_MOUNTS.get(), + params.globalContainerMounts(), v -> Protos.Volume.newBuilder() .setHostPath(v.getHostPath()) .setContainerPath(v.getContainerPath()) @@ -136,20 +199,16 @@ public class ExecutorModule extends AbstractModule { // Necessary as executorId is a required field. .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID) .setCommand(makeExecutorCommand()) - .addResources(makeResource(CPUS, EXECUTOR_OVERHEAD_CPUS.get())) - .addResources(makeResource(RAM_MB, EXECUTOR_OVERHEAD_RAM.get().as(Data.MB))) + .addResources(makeResource(CPUS, params.executorOverheadCpus())) + .addResources(makeResource(RAM_MB, params.executorOverheadRam().as(Data.MB))) .build(), volumeMounts)); } - private static ExecutorSettings makeCustomExecutorSettings() { + private ExecutorSettings makeCustomExecutorSettings(File configFile) { try { - return - new ExecutorSettings( - ExecutorSettingsLoader.read( - Files.newBufferedReader( - CUSTOM_EXECUTOR_CONFIG.get().toPath(), - StandardCharsets.UTF_8))); + return new ExecutorSettings( + ExecutorSettingsLoader.read(Files.newBufferedReader(configFile.toPath()))); } catch (ExecutorSettingsLoader.ExecutorConfigException | IOException e) { throw new IllegalArgumentException("Failed to read executor settings: " + e, e); } @@ -157,8 +216,8 @@ public class ExecutorModule extends AbstractModule { @Override protected void configure() { - bind(ExecutorSettings.class).toInstance(CUSTOM_EXECUTOR_CONFIG.hasAppliedValue() - ? makeCustomExecutorSettings() + bind(ExecutorSettings.class).toInstance(params.customExecutorConfig().isPresent() + ? makeCustomExecutorSettings(params.customExecutorConfig().get()) : makeThermosExecutorSettings()); } http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/cron/quartz/CronModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/cron/quartz/CronModule.java b/src/main/java/org/apache/aurora/scheduler/cron/quartz/CronModule.java index 155d702..c165317 100644 --- a/src/main/java/org/apache/aurora/scheduler/cron/quartz/CronModule.java +++ b/src/main/java/org/apache/aurora/scheduler/cron/quartz/CronModule.java @@ -70,8 +70,41 @@ public class CronModule extends AbstractModule { public static final Arg<Amount<Long, Time>> CRON_START_MAX_BACKOFF = Arg.create(Amount.of(1L, Time.MINUTES)); - // Global per-JVM ID number generator for the provided Quartz Scheduler. - private static final AtomicLong ID_GENERATOR = new AtomicLong(); + interface Params { + int cronSchedulerNumThreads(); + + String cronTimezone(); + + Amount<Long, Time> cronStartInitialBackoff(); + + Amount<Long, Time> cronStartMaxBackoff(); + } + + private final Params params; + + public CronModule() { + this.params = new Params() { + @Override + public int cronSchedulerNumThreads() { + return NUM_THREADS.get(); + } + + @Override + public String cronTimezone() { + return CRON_TIMEZONE.get(); + } + + @Override + public Amount<Long, Time> cronStartInitialBackoff() { + return CRON_START_INITIAL_BACKOFF.get(); + } + + @Override + public Amount<Long, Time> cronStartMaxBackoff() { + return CRON_START_MAX_BACKOFF.get(); + } + }; + } @Override protected void configure() { @@ -88,7 +121,7 @@ public class CronModule extends AbstractModule { bind(AuroraCronJob.class).in(Singleton.class); bind(AuroraCronJob.Config.class).toInstance(new AuroraCronJob.Config( - new BackoffHelper(CRON_START_INITIAL_BACKOFF.get(), CRON_START_MAX_BACKOFF.get()))); + new BackoffHelper(params.cronStartInitialBackoff(), params.cronStartMaxBackoff()))); bind(CronLifecycle.class).in(Singleton.class); SchedulerServicesModule.addSchedulerActiveServiceBinding(binder()).to(CronLifecycle.class); @@ -96,7 +129,7 @@ public class CronModule extends AbstractModule { @Provides TimeZone provideTimeZone() { - TimeZone timeZone = TimeZone.getTimeZone(CRON_TIMEZONE.get()); + TimeZone timeZone = TimeZone.getTimeZone(params.cronTimezone()); TimeZone systemTimeZone = TimeZone.getDefault(); if (!timeZone.equals(systemTimeZone)) { LOG.warn("Cron schedules are configured to fire according to timezone " @@ -107,9 +140,12 @@ public class CronModule extends AbstractModule { return timeZone; } + // Global per-JVM ID number generator for the provided Quartz Scheduler. + private static final AtomicLong ID_GENERATOR = new AtomicLong(); + @Provides @Singleton - static Scheduler provideScheduler(AuroraCronJobFactory jobFactory) throws SchedulerException { + Scheduler provideScheduler(AuroraCronJobFactory jobFactory) throws SchedulerException { // There are several ways to create a quartz Scheduler instance. This path was chosen as the // simplest to create a Scheduler that uses a *daemon* QuartzSchedulerThread instance. Properties props = new Properties(); @@ -117,7 +153,9 @@ public class CronModule extends AbstractModule { props.setProperty(PROP_SCHED_NAME, name); props.setProperty(PROP_SCHED_INSTANCE_ID, name); props.setProperty(PROP_THREAD_POOL_CLASS, SimpleThreadPool.class.getCanonicalName()); - props.setProperty(PROP_THREAD_POOL_PREFIX + ".threadCount", NUM_THREADS.get().toString()); + props.setProperty( + PROP_THREAD_POOL_PREFIX + ".threadCount", + String.valueOf(params.cronSchedulerNumThreads())); props.setProperty(PROP_THREAD_POOL_PREFIX + ".makeThreadsDaemons", Boolean.TRUE.toString()); props.setProperty(PROP_SCHED_MAKE_SCHEDULER_THREAD_DAEMON, Boolean.TRUE.toString()); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/http/H2ConsoleModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/H2ConsoleModule.java b/src/main/java/org/apache/aurora/scheduler/http/H2ConsoleModule.java index 01d6b5d..f965161 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/H2ConsoleModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/H2ConsoleModule.java @@ -33,10 +33,23 @@ public class H2ConsoleModule extends ServletModule { @CmdLine(name = "enable_h2_console", help = "Enable H2 DB management console.") private static final Arg<Boolean> ENABLE_H2_CONSOLE = Arg.create(false); + interface Params { + boolean enableH2Console(); + } + private final boolean enabled; public H2ConsoleModule() { - this(ENABLE_H2_CONSOLE.get()); + this(new Params() { + @Override + public boolean enableH2Console() { + return ENABLE_H2_CONSOLE.get(); + } + }); + } + + public H2ConsoleModule(Params params) { + this(params.enableH2Console()); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java index df649ff..33e3cd8 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java @@ -114,6 +114,16 @@ public class JettyServerModule extends AbstractModule { help = "The port to start an HTTP server on. Default value will choose a random port.") protected static final Arg<Integer> HTTP_PORT = Arg.create(0); + public interface Params { + default Optional<String> hostname() { + return Optional.absent(); + } + + default int httpPort() { + return 0; + } + } + public static final Map<String, String> GUICE_CONTAINER_PARAMS = ImmutableMap.of( FEATURE_POJO_MAPPING, Boolean.TRUE.toString(), PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName(), @@ -124,14 +134,30 @@ public class JettyServerModule extends AbstractModule { .toString() .replace("assets/index.html", ""); + private final Params params; private final boolean production; public JettyServerModule() { - this(true); + this(new Params() { + @Override + public Optional<String> hostname() { + return Optional.fromNullable(HOSTNAME_OVERRIDE.get()); + } + + @Override + public int httpPort() { + return HTTP_PORT.get(); + } + }); + } + + public JettyServerModule(Params params) { + this(params, true); } @VisibleForTesting - JettyServerModule(boolean production) { + JettyServerModule(Params params, boolean production) { + this.params = requireNonNull(params); this.production = production; } @@ -148,10 +174,9 @@ public class JettyServerModule extends AbstractModule { .annotatedWith(Names.named(HealthHandler.HEALTH_CHECKER_KEY)) .toInstance(Suppliers.ofInstance(true)); - final Optional<String> hostnameOverride = Optional.fromNullable(HOSTNAME_OVERRIDE.get()); - if (hostnameOverride.isPresent()) { + if (params.hostname().isPresent()) { try { - InetAddress.getByName(hostnameOverride.get()); + InetAddress.getByName(params.hostname().get()); } catch (UnknownHostException e) { /* Possible misconfiguration, so warn the user. */ LOG.warn("Unable to resolve name specified in -hostname. " @@ -161,7 +186,7 @@ public class JettyServerModule extends AbstractModule { install(new PrivateModule() { @Override protected void configure() { - bind(new TypeLiteral<Optional<String>>() { }).toInstance(hostnameOverride); + bind(Params.class).toInstance(params); bind(HttpService.class).to(HttpServerLauncher.class); bind(HttpServerLauncher.class).in(Singleton.class); expose(HttpServerLauncher.class); @@ -301,18 +326,18 @@ public class JettyServerModule extends AbstractModule { } public static final class HttpServerLauncher extends AbstractIdleService implements HttpService { + private final Params params; private final ServletContextListener servletContextListener; - private final Optional<String> advertisedHostOverride; private volatile Server server; private volatile HostAndPort serverAddress = null; @Inject HttpServerLauncher( - ServletContextListener servletContextListener, - Optional<String> advertisedHostOverride) { + Params params, + ServletContextListener servletContextListener) { + this.params = requireNonNull(params); this.servletContextListener = requireNonNull(servletContextListener); - this.advertisedHostOverride = requireNonNull(advertisedHostOverride); } private static final Map<String, String> REGEX_REWRITE_RULES = @@ -352,7 +377,7 @@ public class JettyServerModule extends AbstractModule { public HostAndPort getAddress() { Preconditions.checkState(state() == State.RUNNING); return HostAndPort.fromParts( - advertisedHostOverride.or(serverAddress.getHostText()), + params.hostname().or(serverAddress.getHostText()), serverAddress.getPort()); } @@ -375,7 +400,7 @@ public class JettyServerModule extends AbstractModule { rootHandler.addHandler(servletHandler); ServerConnector connector = new ServerConnector(server); - connector.setPort(HTTP_PORT.get()); + connector.setPort(params.httpPort()); server.addConnector(connector); server.setHandler(getGzipHandler(getRewriteHandler(rootHandler))); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java index cd5adf9..b37278b 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java @@ -13,6 +13,8 @@ */ package org.apache.aurora.scheduler.http.api; +import java.util.Optional; + import javax.inject.Singleton; import com.google.common.collect.ImmutableMap; @@ -47,10 +49,25 @@ public class ApiModule extends ServletModule { .newClassPathResource("org/apache/aurora/scheduler/gen/client") .toString(); + interface Params { + Optional<String> enableCorsFor(); + } + + private final Params params; + + public ApiModule() { + this.params = new Params() { + @Override + public Optional<String> enableCorsFor() { + return Optional.ofNullable(ENABLE_CORS_FOR.get()); + } + }; + } + @Override protected void configureServlets() { - if (ENABLE_CORS_FOR.get() != null) { - filter(API_PATH).through(new CorsFilter(ENABLE_CORS_FOR.get())); + if (params.enableCorsFor().isPresent()) { + filter(API_PATH).through(new CorsFilter(params.enableCorsFor().get())); } serve(API_PATH).with(TServlet.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java index e328620..d305112 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java @@ -81,7 +81,7 @@ public class HttpSecurityModule extends ServletModule { @CmdLine(name = "shiro_after_auth_filter", help = "Fully qualified class name of the servlet filter to be applied after the" + " shiro auth filters are applied.") - private static final Arg<Class<? extends Filter>> SHIRO_AFTER_AUTH_FILTER = Arg.create(); + private static final Arg<Class<? extends Filter>> SHIRO_AFTER_AUTH_FILTER = Arg.create(null); @VisibleForTesting static final Matcher<Method> AURORA_SCHEDULER_MANAGER_SERVICE = @@ -112,21 +112,48 @@ public class HttpSecurityModule extends ServletModule { private static final Arg<HttpAuthenticationMechanism> HTTP_AUTHENTICATION_MECHANISM = Arg.create(HttpAuthenticationMechanism.NONE); + interface Params { + Set<Module> shiroRealmModule(); + + Optional<Class<? extends Filter>> shiroAfterAuthFilter(); + + HttpAuthenticationMechanism httpAuthenticationMechanism(); + } + private final HttpAuthenticationMechanism mechanism; private final Set<Module> shiroConfigurationModules; private final Optional<Key<? extends Filter>> shiroAfterAuthFilterKey; public HttpSecurityModule() { + this(new Params() { + @Override + public Set<Module> shiroRealmModule() { + return SHIRO_REALM_MODULE.get(); + } + + @Override + public Optional<Class<? extends Filter>> shiroAfterAuthFilter() { + return Optional.ofNullable(SHIRO_AFTER_AUTH_FILTER.get()); + } + + @Override + public HttpAuthenticationMechanism httpAuthenticationMechanism() { + return HTTP_AUTHENTICATION_MECHANISM.get(); + } + }); + } + + public HttpSecurityModule(Params params) { this( - HTTP_AUTHENTICATION_MECHANISM.get(), - SHIRO_REALM_MODULE.get(), - SHIRO_AFTER_AUTH_FILTER.hasAppliedValue() ? Key.get(SHIRO_AFTER_AUTH_FILTER.get()) : null); + params.httpAuthenticationMechanism(), + params.shiroRealmModule(), + params.shiroAfterAuthFilter().map(Key::get)); } @VisibleForTesting HttpSecurityModule( Module shiroConfigurationModule, - Key<? extends Filter> shiroAfterAuthFilterKey) { + Optional<Key<? extends Filter>> shiroAfterAuthFilterKey) { this(HttpAuthenticationMechanism.BASIC, ImmutableSet.of(shiroConfigurationModule), @@ -136,11 +163,11 @@ public class HttpSecurityModule extends ServletModule { private HttpSecurityModule( HttpAuthenticationMechanism mechanism, Set<Module> shiroConfigurationModules, - Key<? extends Filter> shiroAfterAuthFilterKey) { + Optional<Key<? extends Filter>> shiroAfterAuthFilterKey) { this.mechanism = requireNonNull(mechanism); this.shiroConfigurationModules = requireNonNull(shiroConfigurationModules); - this.shiroAfterAuthFilterKey = Optional.ofNullable(shiroAfterAuthFilterKey); + this.shiroAfterAuthFilterKey = requireNonNull(shiroAfterAuthFilterKey); } @Override http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java index 43c38dc..209a714 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java @@ -39,10 +39,23 @@ public class IniShiroRealmModule extends AbstractModule { help = "Path to shiro.ini for authentication and authorization configuration.") private static final Arg<Ini> SHIRO_INI_PATH = Arg.create(null); + interface Params { + Optional<Ini> shiroIniPath(); + } + private final Optional<Ini> ini; public IniShiroRealmModule() { - this(Optional.fromNullable(SHIRO_INI_PATH.get())); + this(new Params() { + @Override + public Optional<Ini> shiroIniPath() { + return Optional.fromNullable(SHIRO_INI_PATH.get()); + } + }); + } + + public IniShiroRealmModule(Params params) { + this(params.shiroIniPath()); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java index 0f8bdbb..0090567 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java @@ -81,17 +81,44 @@ public class Kerberos5ShiroRealmModule extends AbstractModule { @CmdLine(name = "kerberos_debug", help = "Produce additional Kerberos debugging output.") private static final Arg<Boolean> DEBUG = Arg.create(false); + interface Params { + Optional<File> kerberosServerKeytab(); + + Optional<KerberosPrincipal> kerberosServerPrincipal(); + + boolean kerberosDebug(); + } + private final Optional<File> serverKeyTab; private final Optional<KerberosPrincipal> serverPrincipal; private final GSSManager gssManager; private final boolean kerberosDebugEnabled; public Kerberos5ShiroRealmModule() { + this(new Params() { + @Override + public Optional<File> kerberosServerKeytab() { + return Optional.fromNullable(SERVER_KEYTAB.get()); + } + + @Override + public Optional<KerberosPrincipal> kerberosServerPrincipal() { + return Optional.fromNullable(SERVER_PRINCIPAL.get()); + } + + @Override + public boolean kerberosDebug() { + return DEBUG.get(); + } + }); + } + + public Kerberos5ShiroRealmModule(Params params) { this( - Optional.fromNullable(SERVER_KEYTAB.get()), - Optional.fromNullable(SERVER_PRINCIPAL.get()), + params.kerberosServerKeytab(), + params.kerberosServerPrincipal(), GSSManager.getInstance(), - DEBUG.get()); + params.kerberosDebug()); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java index 5daafa9..9482d74 100644 --- a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java +++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java @@ -15,14 +15,14 @@ package org.apache.aurora.scheduler.log.mesos; import java.io.File; import java.util.List; -import java.util.Objects; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import javax.inject.Singleton; import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; +import com.google.common.collect.FluentIterable; import com.google.inject.PrivateModule; import com.google.inject.Provides; import com.google.inject.TypeLiteral; @@ -40,6 +40,8 @@ import org.apache.aurora.scheduler.zookeeper.guice.client.ZooKeeperClientModule. import org.apache.mesos.Log; import org.apache.zookeeper.common.PathUtils; +import static java.util.Objects.requireNonNull; + /** * Binds a native mesos Log implementation. * @@ -93,38 +95,87 @@ public class MesosLogStreamModule extends PrivateModule { private static final Arg<Amount<Long, Time>> WRITE_TIMEOUT = Arg.create(Amount.of(3L, Time.SECONDS)); - private static <T> T getRequiredArg(Arg<T> arg, String name) { - if (!arg.hasAppliedValue()) { + private static void requireParam(Optional<?> arg, String name) { + if (!arg.isPresent()) { throw new IllegalStateException( String.format("A value for the -%s flag must be supplied", name)); } - return arg.get(); } + interface Params { + int nativeLogQuorumSize(); + + Optional<File> nativeLogFilePath(); + + Optional<String> nativeLogZkGroupPath(); + + Amount<Long, Time> nativeLogElectionTimeout(); + + int nativeLogElectionRetries(); + + Amount<Long, Time> nativeLogReadTimeout(); + + Amount<Long, Time> nativeLogWriteTimeout(); + } + + private final Params params; private final ClientConfig zkClientConfig; - private final File logPath; - private final String zkLogGroupPath; public MesosLogStreamModule(ClientConfig zkClientConfig) { - this(zkClientConfig, - getRequiredArg(LOG_PATH, "native_log_file_path"), - getRequiredArg(ZK_LOG_GROUP_PATH, "native_log_zk_group_path")); - } + this( + new Params() { + @Override + public int nativeLogQuorumSize() { + return QUORUM_SIZE.get(); + } + + @Override + public Optional<File> nativeLogFilePath() { + return Optional.ofNullable(LOG_PATH.get()); + } + + @Override + public Optional<String> nativeLogZkGroupPath() { + return Optional.ofNullable(ZK_LOG_GROUP_PATH.get()); + } - public MesosLogStreamModule(ClientConfig zkClientConfig, File logPath, String zkLogGroupPath) { - this.zkClientConfig = Objects.requireNonNull(zkClientConfig); - this.logPath = Objects.requireNonNull(logPath); + @Override + public Amount<Long, Time> nativeLogElectionTimeout() { + return COORDINATOR_ELECTION_TIMEOUT.get(); + } + + @Override + public int nativeLogElectionRetries() { + return COORDINATOR_ELECTION_RETRIES.get(); + } + + @Override + public Amount<Long, Time> nativeLogReadTimeout() { + return READ_TIMEOUT.get(); + } + + @Override + public Amount<Long, Time> nativeLogWriteTimeout() { + return WRITE_TIMEOUT.get(); + } + }, + zkClientConfig); + } - PathUtils.validatePath(zkLogGroupPath); // This checks for null. - this.zkLogGroupPath = zkLogGroupPath; + public MesosLogStreamModule(Params params, ClientConfig zkClientConfig) { + requireParam(params.nativeLogFilePath(), "native_log_file_path"); + requireParam(params.nativeLogZkGroupPath(), "native_log_zk_group_path"); + PathUtils.validatePath(params.nativeLogZkGroupPath().get()); + this.params = requireNonNull(params); + this.zkClientConfig = requireNonNull(zkClientConfig); } @Override protected void configure() { bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.ReadTimeout.class) - .toInstance(READ_TIMEOUT.get()); + .toInstance(params.nativeLogReadTimeout()); bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.WriteTimeout.class) - .toInstance(WRITE_TIMEOUT.get()); + .toInstance(params.nativeLogWriteTimeout()); bind(org.apache.aurora.scheduler.log.Log.class).to(MesosLog.class); bind(MesosLog.class).in(Singleton.class); @@ -134,21 +185,23 @@ public class MesosLogStreamModule extends PrivateModule { @Provides @Singleton Log provideLog() { + File logPath = params.nativeLogFilePath().get(); File parentDir = logPath.getParentFile(); if (!parentDir.exists() && !parentDir.mkdirs()) { addError("Failed to create parent directory to store native log at: %s", parentDir); } - String zkConnectString = Joiner.on(',').join( - Iterables.transform(zkClientConfig.servers, InetSocketAddressHelper::toString)); + String zkConnectString = FluentIterable.from(zkClientConfig.servers) + .transform(InetSocketAddressHelper::toString) + .join(Joiner.on(',')); return new Log( - QUORUM_SIZE.get(), + params.nativeLogQuorumSize(), logPath.getAbsolutePath(), zkConnectString, zkClientConfig.sessionTimeout.getValue(), zkClientConfig.sessionTimeout.getUnit().getTimeUnit(), - zkLogGroupPath, + params.nativeLogZkGroupPath().get(), zkClientConfig.credentials.scheme(), zkClientConfig.credentials.authToken()); } @@ -160,9 +213,12 @@ public class MesosLogStreamModule extends PrivateModule { @Provides Log.Writer provideWriter(Log log) { - Amount<Long, Time> electionTimeout = COORDINATOR_ELECTION_TIMEOUT.get(); - return new Log.Writer(log, electionTimeout.getValue(), electionTimeout.getUnit().getTimeUnit(), - COORDINATOR_ELECTION_RETRIES.get()); + Amount<Long, Time> electionTimeout = params.nativeLogElectionTimeout(); + return new Log.Writer( + log, + electionTimeout.getValue(), + electionTimeout.getUnit().getTimeUnit(), + params.nativeLogElectionRetries()); } @Provides http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java b/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java index 7de8f4c..4c41576 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java @@ -62,7 +62,7 @@ public class CommandLineDriverSettingsModule extends AbstractModule { help = "Properties file which contains framework credentials to authenticate with Mesos" + "master. Must contain the properties '" + PRINCIPAL_KEY + "' and " + "'" + SECRET_KEY + "'.") - private static final Arg<File> FRAMEWORK_AUTHENTICATION_FILE = Arg.create(); + private static final Arg<File> FRAMEWORK_AUTHENTICATION_FILE = Arg.create(null); @CmdLine(name = "framework_failover_timeout", help = "Time after which a framework is considered deleted. SHOULD BE VERY HIGH.") @@ -93,37 +93,92 @@ public class CommandLineDriverSettingsModule extends AbstractModule { help = "The Mesos role this framework will register as. The default is to left this empty, " + "and the framework will register without any role and only receive unreserved " + "resources in offer.") - private static final Arg<String> MESOS_ROLE = Arg.create(); + private static final Arg<String> MESOS_ROLE = Arg.create(null); + + interface Params { + String mesosMasterAddress(); + + Optional<File> frameworkAuthenticationFile(); + + Amount<Long, Time> frameworkFailoverTimeout(); + + boolean frameworkAnnouncePrincipal(); + + String executorUser(); + + boolean receiveRevocableResources(); + + Optional<String> mesosRole(); + } // TODO(wfarner): Figure out a way to change this without risk of fallout (MESOS-703). private static final String TWITTER_FRAMEWORK_NAME = "TwitterScheduler"; + private final Params params; + + public CommandLineDriverSettingsModule() { + this.params = new Params() { + @Override + public String mesosMasterAddress() { + return MESOS_MASTER_ADDRESS.get(); + } + + @Override + public Optional<File> frameworkAuthenticationFile() { + return Optional.fromNullable(FRAMEWORK_AUTHENTICATION_FILE.get()); + } + + @Override + public Amount<Long, Time> frameworkFailoverTimeout() { + return FRAMEWORK_FAILOVER_TIMEOUT.get(); + } + + @Override + public boolean frameworkAnnouncePrincipal() { + return FRAMEWORK_ANNOUNCE_PRINCIPAL.get(); + } + + @Override + public String executorUser() { + return EXECUTOR_USER.get(); + } + + @Override + public boolean receiveRevocableResources() { + return RECEIVE_REVOCABLE_RESOURCES.get(); + } + + @Override + public Optional<String> mesosRole() { + return Optional.fromNullable(MESOS_ROLE.get()); + } + }; + } + @Override protected void configure() { Optional<Protos.Credential> credentials = getCredentials(); - Optional<String> principal = Optional.absent(); - if (FRAMEWORK_ANNOUNCE_PRINCIPAL.get() && credentials.isPresent()) { - principal = Optional.of(credentials.get().getPrincipal()); - } - Optional<String> role = - MESOS_ROLE.hasAppliedValue() ? Optional.of(MESOS_ROLE.get()) : Optional.absent(); + Optional<String> principal = params.frameworkAnnouncePrincipal() && credentials.isPresent() + ? Optional.of(credentials.get().getPrincipal()) + : Optional.absent(); DriverSettings settings = new DriverSettings( - MESOS_MASTER_ADDRESS.get(), + params.mesosMasterAddress(), credentials, buildFrameworkInfo( - EXECUTOR_USER.get(), + params.executorUser(), principal, - FRAMEWORK_FAILOVER_TIMEOUT.get(), - RECEIVE_REVOCABLE_RESOURCES.get(), - role)); + params.frameworkFailoverTimeout(), + params.receiveRevocableResources(), + params.mesosRole())); bind(DriverSettings.class).toInstance(settings); } - private static Optional<Protos.Credential> getCredentials() { - if (FRAMEWORK_AUTHENTICATION_FILE.hasAppliedValue()) { + private Optional<Protos.Credential> getCredentials() { + if (params.frameworkAuthenticationFile().isPresent()) { Properties properties; try { - properties = parseCredentials(new FileInputStream(FRAMEWORK_AUTHENTICATION_FILE.get())); + properties = + parseCredentials(new FileInputStream(params.frameworkAuthenticationFile().get())); } catch (FileNotFoundException e) { LOG.error("Authentication File not Found"); throw Throwables.propagate(e); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java b/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java index 90f8abf..39b4ab1 100644 --- a/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java +++ b/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java @@ -43,6 +43,27 @@ public class OffersModule extends AbstractModule { private static final Arg<Amount<Integer, Time>> OFFER_HOLD_JITTER_WINDOW = Arg.create(Amount.of(1, Time.MINUTES)); + interface Params { + Amount<Integer, Time> minOfferHoldTime(); + + Amount<Integer, Time> offerHoldJitterWindow(); + } + private final Params params; + + public OffersModule() { + params = new Params() { + @Override + public Amount<Integer, Time> minOfferHoldTime() { + return MIN_OFFER_HOLD_TIME.get(); + } + + @Override + public Amount<Integer, Time> offerHoldJitterWindow() { + return OFFER_HOLD_JITTER_WINDOW.get(); + } + }; + } + @Override protected void configure() { install(new PrivateModule() { @@ -50,8 +71,8 @@ public class OffersModule extends AbstractModule { protected void configure() { bind(OfferManager.OfferReturnDelay.class).toInstance( new RandomJitterReturnDelay( - MIN_OFFER_HOLD_TIME.get().as(Time.MILLISECONDS), - OFFER_HOLD_JITTER_WINDOW.get().as(Time.MILLISECONDS), + params.minOfferHoldTime().as(Time.MILLISECONDS), + params.offerHoldJitterWindow().as(Time.MILLISECONDS), Random.Util.newDefaultRandom())); bind(OfferManager.class).to(OfferManager.OfferManagerImpl.class); bind(OfferManager.OfferManagerImpl.class).in(Singleton.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java b/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java index 23d1c12..e58688a 100644 --- a/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java +++ b/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java @@ -59,23 +59,49 @@ public class PreemptorModule extends AbstractModule { private static final Arg<Amount<Long, Time>> PREEMPTION_SLOT_SEARCH_INTERVAL = Arg.create(Amount.of(1L, Time.MINUTES)); - private final boolean enablePreemptor; - private final Amount<Long, Time> preemptionDelay; - private final Amount<Long, Time> slotSearchInterval; + public interface Params { + default boolean enablePreemptor() { + return true; + } + + Amount<Long, Time> preemptionDelay(); + + default Amount<Long, Time> preemptionSlotHoldTime() { + return Amount.of(5L, Time.MINUTES); + } + + Amount<Long, Time> preemptionSlotSearchInterval(); + } + + private final Params params; @VisibleForTesting - public PreemptorModule( - boolean enablePreemptor, - Amount<Long, Time> preemptionDelay, - Amount<Long, Time> slotSearchInterval) { - - this.enablePreemptor = enablePreemptor; - this.preemptionDelay = requireNonNull(preemptionDelay); - this.slotSearchInterval = requireNonNull(slotSearchInterval); + public PreemptorModule(Params params) { + this.params = requireNonNull(params); } public PreemptorModule() { - this(ENABLE_PREEMPTOR.get(), PREEMPTION_DELAY.get(), PREEMPTION_SLOT_SEARCH_INTERVAL.get()); + this(new Params() { + @Override + public boolean enablePreemptor() { + return ENABLE_PREEMPTOR.get(); + } + + @Override + public Amount<Long, Time> preemptionDelay() { + return PREEMPTION_DELAY.get(); + } + + @Override + public Amount<Long, Time> preemptionSlotHoldTime() { + return PREEMPTION_SLOT_HOLD_TIME.get(); + } + + @Override + public Amount<Long, Time> preemptionSlotSearchInterval() { + return PREEMPTION_SLOT_SEARCH_INTERVAL.get(); + } + }); } @Override @@ -83,7 +109,7 @@ public class PreemptorModule extends AbstractModule { install(new PrivateModule() { @Override protected void configure() { - if (enablePreemptor) { + if (params.enablePreemptor()) { LOG.info("Preemptor Enabled."); bind(PreemptorMetrics.class).in(Singleton.class); bind(PreemptionVictimFilter.class) @@ -93,9 +119,9 @@ public class PreemptorModule extends AbstractModule { bind(Preemptor.PreemptorImpl.class).in(Singleton.class); bind(new TypeLiteral<Amount<Long, Time>>() { }) .annotatedWith(PendingTaskProcessor.PreemptionDelay.class) - .toInstance(preemptionDelay); + .toInstance(params.preemptionDelay()); bind(BiCacheSettings.class).toInstance( - new BiCacheSettings(PREEMPTION_SLOT_HOLD_TIME.get(), "preemption_slot_cache_size")); + new BiCacheSettings(params.preemptionSlotHoldTime(), "preemption_slot_cache_size")); bind(new TypeLiteral<BiCache<PreemptionProposal, TaskGroupKey>>() { }) .in(Singleton.class); bind(PendingTaskProcessor.class).in(Singleton.class); @@ -107,8 +133,8 @@ public class PreemptorModule extends AbstractModule { bind(AbstractScheduledService.Scheduler.class).toInstance( AbstractScheduledService.Scheduler.newFixedRateSchedule( 0L, - slotSearchInterval.getValue(), - slotSearchInterval.getUnit().getTimeUnit())); + params.preemptionSlotSearchInterval().getValue(), + params.preemptionSlotSearchInterval().getUnit().getTimeUnit())); expose(PreemptorService.class); expose(PendingTaskProcessor.class); @@ -124,9 +150,8 @@ public class PreemptorModule extends AbstractModule { // and private modules due to multiple injectors. We accept the added complexity here to keep // the other bindings private. PubsubEventModule.bindSubscriber(binder(), ClusterStateImpl.class); - if (enablePreemptor) { - SchedulerServicesModule.addSchedulerActiveServiceBinding(binder()) - .to(PreemptorService.class); + if (params.enablePreemptor()) { + SchedulerServicesModule.addSchedulerActiveServiceBinding(binder()).to(PreemptorService.class); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java b/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java index 735199a..be10bff 100644 --- a/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java +++ b/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java @@ -66,6 +66,56 @@ public class PruningModule extends AbstractModule { private static final Arg<Amount<Long, Time>> JOB_UPDATE_HISTORY_PRUNING_THRESHOLD = Arg.create(Amount.of(30L, Time.DAYS)); + interface Params { + Amount<Long, Time> historyPruneThreshold(); + + int historyMaxPerJobThreshold(); + + Amount<Long, Time> historyMinRetentionThreshold(); + + int jobUpdateHistoryPerJobThreshold(); + + Amount<Long, Time> jobUpdateHistoryPruningInterval(); + + Amount<Long, Time> jobUpdateHistoryPruningThreshold(); + } + + private final Params params; + + public PruningModule() { + this.params = new Params() { + @Override + public Amount<Long, Time> historyPruneThreshold() { + return HISTORY_PRUNE_THRESHOLD.get(); + } + + @Override + public int historyMaxPerJobThreshold() { + return HISTORY_MAX_PER_JOB_THRESHOLD.get(); + } + + @Override + public Amount<Long, Time> historyMinRetentionThreshold() { + return HISTORY_MIN_RETENTION_THRESHOLD.get(); + } + + @Override + public int jobUpdateHistoryPerJobThreshold() { + return JOB_UPDATE_HISTORY_PER_JOB_THRESHOLD.get(); + } + + @Override + public Amount<Long, Time> jobUpdateHistoryPruningInterval() { + return JOB_UPDATE_HISTORY_PRUNING_INTERVAL.get(); + } + + @Override + public Amount<Long, Time> jobUpdateHistoryPruningThreshold() { + return JOB_UPDATE_HISTORY_PRUNING_THRESHOLD.get(); + } + }; + } + @Override protected void configure() { install(new PrivateModule() { @@ -74,10 +124,9 @@ public class PruningModule extends AbstractModule { // TODO(ksweeney): Create a configuration validator module so this can be injected. // TODO(William Farner): Revert this once large task counts is cheap ala hierarchichal store bind(HistoryPrunnerSettings.class).toInstance(new HistoryPrunnerSettings( - HISTORY_PRUNE_THRESHOLD.get(), - HISTORY_MIN_RETENTION_THRESHOLD.get(), - HISTORY_MAX_PER_JOB_THRESHOLD.get() - )); + params.historyPruneThreshold(), + params.historyMinRetentionThreshold(), + params.historyMaxPerJobThreshold())); bind(TaskHistoryPruner.class).in(Singleton.class); expose(TaskHistoryPruner.class); @@ -90,9 +139,9 @@ public class PruningModule extends AbstractModule { protected void configure() { bind(JobUpdateHistoryPruner.HistoryPrunerSettings.class).toInstance( new JobUpdateHistoryPruner.HistoryPrunerSettings( - JOB_UPDATE_HISTORY_PRUNING_INTERVAL.get(), - JOB_UPDATE_HISTORY_PRUNING_THRESHOLD.get(), - JOB_UPDATE_HISTORY_PER_JOB_THRESHOLD.get())); + params.jobUpdateHistoryPruningInterval(), + params.jobUpdateHistoryPruningThreshold(), + params.jobUpdateHistoryPerJobThreshold())); bind(ScheduledExecutorService.class).toInstance( AsyncUtil.singleThreadLoggingScheduledExecutor("JobUpdatePruner-%d", LOG)); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java b/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java index cccee08..b8e2c0a 100644 --- a/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java +++ b/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java @@ -88,6 +88,56 @@ public class ReconciliationModule extends AbstractModule { private static final Arg<Amount<Long, Time>> RECONCILIATION_SCHEDULE_SPREAD = Arg.create(Amount.of(30L, Time.MINUTES)); + interface Params { + Amount<Long, Time> transientTaskStateTimeout(); + + Amount<Long, Time> initialTaskKillRetryInterval(); + + Amount<Long, Time> reconciliationInitialDelay(); + + Amount<Long, Time> reconciliationExplicitInterval(); + + Amount<Long, Time> reconciliationImplicitInterval(); + + Amount<Long, Time> reconciliationScheduleSpread(); + } + + private final Params params; + + public ReconciliationModule() { + this.params = new Params() { + @Override + public Amount<Long, Time> transientTaskStateTimeout() { + return TRANSIENT_TASK_STATE_TIMEOUT.get(); + } + + @Override + public Amount<Long, Time> initialTaskKillRetryInterval() { + return INITIAL_TASK_KILL_RETRY_INTERVAL.get(); + } + + @Override + public Amount<Long, Time> reconciliationInitialDelay() { + return RECONCILIATION_INITIAL_DELAY.get(); + } + + @Override + public Amount<Long, Time> reconciliationExplicitInterval() { + return RECONCILIATION_EXPLICIT_INTERVAL.get(); + } + + @Override + public Amount<Long, Time> reconciliationImplicitInterval() { + return RECONCILIATION_IMPLICIT_INTERVAL.get(); + } + + @Override + public Amount<Long, Time> reconciliationScheduleSpread() { + return RECONCILIATION_SCHEDULE_SPREAD.get(); + } + }; + } + @Qualifier @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME) @interface BackgroundWorker { } @@ -98,7 +148,7 @@ public class ReconciliationModule extends AbstractModule { @Override protected void configure() { bind(new TypeLiteral<Amount<Long, Time>>() { }) - .toInstance(TRANSIENT_TASK_STATE_TIMEOUT.get()); + .toInstance(params.transientTaskStateTimeout()); bind(TaskTimeout.class).in(Singleton.class); expose(TaskTimeout.class); @@ -112,8 +162,8 @@ public class ReconciliationModule extends AbstractModule { protected void configure() { bind(BackoffStrategy.class).toInstance( new TruncatedBinaryBackoff( - INITIAL_TASK_KILL_RETRY_INTERVAL.get(), - TRANSIENT_TASK_STATE_TIMEOUT.get())); + params.initialTaskKillRetryInterval(), + params.transientTaskStateTimeout())); bind(KillRetry.class).in(Singleton.class); expose(KillRetry.class); } @@ -124,10 +174,10 @@ public class ReconciliationModule extends AbstractModule { @Override protected void configure() { bind(TaskReconcilerSettings.class).toInstance(new TaskReconcilerSettings( - RECONCILIATION_INITIAL_DELAY.get(), - RECONCILIATION_EXPLICIT_INTERVAL.get(), - RECONCILIATION_IMPLICIT_INTERVAL.get(), - RECONCILIATION_SCHEDULE_SPREAD.get())); + params.reconciliationInitialDelay(), + params.reconciliationExplicitInterval(), + params.reconciliationImplicitInterval(), + params.reconciliationScheduleSpread())); bind(ScheduledExecutorService.class).annotatedWith(BackgroundWorker.class) .toInstance(AsyncUtil.loggingScheduledExecutor(1, "TaskReconciler-%d", LOG)); bind(TaskReconciler.class).in(Singleton.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/scheduling/SchedulingModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/scheduling/SchedulingModule.java b/src/main/java/org/apache/aurora/scheduler/scheduling/SchedulingModule.java index 577edcb..150c654 100644 --- a/src/main/java/org/apache/aurora/scheduler/scheduling/SchedulingModule.java +++ b/src/main/java/org/apache/aurora/scheduler/scheduling/SchedulingModule.java @@ -29,6 +29,7 @@ import org.apache.aurora.common.util.TruncatedBinaryBackoff; import org.apache.aurora.scheduler.base.TaskGroupKey; import org.apache.aurora.scheduler.events.PubsubEventModule; import org.apache.aurora.scheduler.preemptor.BiCache; +import org.apache.aurora.scheduler.preemptor.BiCache.BiCacheSettings; import org.apache.aurora.scheduler.scheduling.RescheduleCalculator.RescheduleCalculatorImpl; /** @@ -83,23 +84,96 @@ public class SchedulingModule extends AbstractModule { private static final Arg<Amount<Long, Time>> RESERVATION_DURATION = Arg.create(Amount.of(3L, Time.MINUTES)); + interface Params { + double maxScheduleAttemptsPerSec(); + + Amount<Long, Time> flappingTaskThreshold(); + + Amount<Long, Time> initialFlappingTaskDelay(); + + Amount<Long, Time> maxFlappingTaskDelay(); + + Amount<Integer, Time> maxRescheduleTaskDelayOnStartup(); + + Amount<Long, Time> firstScheduleDelay(); + + Amount<Long, Time> initialSchedulePenalty(); + + Amount<Long, Time> maxSchedulePenalty(); + + Amount<Long, Time> offerReservationDuration(); + } + + private final Params params; + + public SchedulingModule() { + this.params = new Params() { + @Override + public double maxScheduleAttemptsPerSec() { + return MAX_SCHEDULE_ATTEMPTS_PER_SEC.get(); + } + + @Override + public Amount<Long, Time> flappingTaskThreshold() { + return FLAPPING_THRESHOLD.get(); + } + + @Override + public Amount<Long, Time> initialFlappingTaskDelay() { + return INITIAL_FLAPPING_DELAY.get(); + } + + @Override + public Amount<Long, Time> maxFlappingTaskDelay() { + return MAX_FLAPPING_DELAY.get(); + } + + @Override + public Amount<Integer, Time> maxRescheduleTaskDelayOnStartup() { + return MAX_RESCHEDULING_DELAY.get(); + } + + @Override + public Amount<Long, Time> firstScheduleDelay() { + return FIRST_SCHEDULE_DELAY.get(); + } + + @Override + public Amount<Long, Time> initialSchedulePenalty() { + return INITIAL_SCHEDULE_PENALTY.get(); + } + + @Override + public Amount<Long, Time> maxSchedulePenalty() { + return MAX_SCHEDULE_PENALTY.get(); + } + + @Override + public Amount<Long, Time> offerReservationDuration() { + return RESERVATION_DURATION.get(); + } + }; + } + @Override protected void configure() { install(new PrivateModule() { @Override protected void configure() { bind(TaskGroups.TaskGroupsSettings.class).toInstance(new TaskGroups.TaskGroupsSettings( - FIRST_SCHEDULE_DELAY.get(), + params.firstScheduleDelay(), new TruncatedBinaryBackoff( - INITIAL_SCHEDULE_PENALTY.get(), - MAX_SCHEDULE_PENALTY.get()), - RateLimiter.create(MAX_SCHEDULE_ATTEMPTS_PER_SEC.get()))); + params.initialSchedulePenalty(), + params.maxSchedulePenalty()), + RateLimiter.create(params.maxScheduleAttemptsPerSec()))); bind(RescheduleCalculatorImpl.RescheduleCalculatorSettings.class) .toInstance(new RescheduleCalculatorImpl.RescheduleCalculatorSettings( - new TruncatedBinaryBackoff(INITIAL_FLAPPING_DELAY.get(), MAX_FLAPPING_DELAY.get()), - FLAPPING_THRESHOLD.get(), - MAX_RESCHEDULING_DELAY.get())); + new TruncatedBinaryBackoff( + params.initialFlappingTaskDelay(), + params.maxFlappingTaskDelay()), + params.flappingTaskThreshold(), + params.maxRescheduleTaskDelayOnStartup())); bind(RescheduleCalculator.class).to(RescheduleCalculatorImpl.class).in(Singleton.class); expose(RescheduleCalculator.class); @@ -113,8 +187,8 @@ public class SchedulingModule extends AbstractModule { @Override protected void configure() { bind(new TypeLiteral<BiCache<String, TaskGroupKey>>() { }).in(Singleton.class); - bind(BiCache.BiCacheSettings.class).toInstance( - new BiCache.BiCacheSettings(RESERVATION_DURATION.get(), "reservation_cache_size")); + bind(BiCacheSettings.class).toInstance( + new BiCacheSettings(params.offerReservationDuration(), "reservation_cache_size")); bind(TaskScheduler.class).to(TaskScheduler.TaskSchedulerImpl.class); bind(TaskScheduler.TaskSchedulerImpl.class).in(Singleton.class); expose(TaskScheduler.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/sla/SlaModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/sla/SlaModule.java b/src/main/java/org/apache/aurora/scheduler/sla/SlaModule.java index d569241..159a884 100644 --- a/src/main/java/org/apache/aurora/scheduler/sla/SlaModule.java +++ b/src/main/java/org/apache/aurora/scheduler/sla/SlaModule.java @@ -72,37 +72,52 @@ public class SlaModule extends AbstractModule { private static final Arg<Set<MetricCategory>> SLA_NON_PROD_METRICS = Arg.<Set<MetricCategory>>create(ImmutableSet.of()); + public interface Params { + Amount<Long, Time> slaStatRefreshInterval(); + + Set<MetricCategory> slaProdMetrics(); + + Set<MetricCategory> slaNonProdMetrics(); + } + @VisibleForTesting @Qualifier @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME) @interface SlaExecutor { } - private final Amount<Long, Time> refreshInterval; - private final Set<MetricCategory> prodMetrics; - private final Set<MetricCategory> nonProdMetrics; + private final Params params; @VisibleForTesting - SlaModule( - Amount<Long, Time> refreshInterval, - Set<MetricCategory> prodMetrics, - Set<MetricCategory> nonProdMetrics) { - - this.refreshInterval = refreshInterval; - this.prodMetrics = prodMetrics; - this.nonProdMetrics = nonProdMetrics; + SlaModule(Params params) { + this.params = requireNonNull(params); } public SlaModule() { - this(SLA_REFRESH_INTERVAL.get(), SLA_PROD_METRICS.get(), SLA_NON_PROD_METRICS.get()); + this(new Params() { + @Override + public Amount<Long, Time> slaStatRefreshInterval() { + return SLA_REFRESH_INTERVAL.get(); + } + + @Override + public Set<MetricCategory> slaProdMetrics() { + return SLA_PROD_METRICS.get(); + } + + @Override + public Set<MetricCategory> slaNonProdMetrics() { + return SLA_NON_PROD_METRICS.get(); + } + }); } @Override protected void configure() { bind(MetricCalculatorSettings.class) .toInstance(new MetricCalculatorSettings( - refreshInterval.as(Time.MILLISECONDS), - prodMetrics, - nonProdMetrics)); + params.slaStatRefreshInterval().as(Time.MILLISECONDS), + params.slaProdMetrics(), + params.slaNonProdMetrics())); bind(MetricCalculator.class).in(Singleton.class); bind(ScheduledExecutorService.class) http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java b/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java index 08eb6d6..e8d1e14 100644 --- a/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java +++ b/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java @@ -58,6 +58,35 @@ public class AsyncStatsModule extends AbstractModule { private static final Arg<Amount<Long, Time>> SLOT_STAT_INTERVAL = Arg.create(Amount.of(1L, Time.MINUTES)); + interface Params { + Amount<Long, Time> asyncTaskStatUpdateInterval(); + + Amount<Long, Time> asyncSlotStatUpdateInterval(); + } + + private final Params params; + + public AsyncStatsModule() { + this.params = new Params() { + @Override + public Amount<Long, Time> asyncTaskStatUpdateInterval() { + return TASK_STAT_INTERVAL.get(); + } + + @Override + public Amount<Long, Time> asyncSlotStatUpdateInterval() { + return SLOT_STAT_INTERVAL.get(); + } + }; + } + + private static Scheduler fromDuration(Amount<Long, Time> duration) { + return Scheduler.newFixedDelaySchedule( + duration.getValue(), + duration.getValue(), + duration.getUnit().getTimeUnit()); + } + @Override protected void configure() { bind(TaskStatCalculator.class).in(Singleton.class); @@ -69,11 +98,7 @@ public class AsyncStatsModule extends AbstractModule { @Override protected void configure() { bind(TaskStatUpdaterService.class).in(Singleton.class); - bind(Scheduler.class).toInstance( - Scheduler.newFixedRateSchedule( - TASK_STAT_INTERVAL.get().getValue(), - TASK_STAT_INTERVAL.get().getValue(), - TASK_STAT_INTERVAL.get().getUnit().getTimeUnit())); + bind(Scheduler.class).toInstance(fromDuration(params.asyncTaskStatUpdateInterval())); expose(TaskStatUpdaterService.class); } }); @@ -84,11 +109,7 @@ public class AsyncStatsModule extends AbstractModule { @Override protected void configure() { bind(SlotSizeCounterService.class).in(Singleton.class); - bind(Scheduler.class).toInstance( - Scheduler.newFixedRateSchedule( - SLOT_STAT_INTERVAL.get().getValue(), - SLOT_STAT_INTERVAL.get().getValue(), - SLOT_STAT_INTERVAL.get().getUnit().getTimeUnit())); + bind(Scheduler.class).toInstance(fromDuration(params.asyncSlotStatUpdateInterval())); expose(SlotSizeCounterService.class); } }); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/stats/StatsModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/stats/StatsModule.java b/src/main/java/org/apache/aurora/scheduler/stats/StatsModule.java index 4767ef1..c56ff3c 100644 --- a/src/main/java/org/apache/aurora/scheduler/stats/StatsModule.java +++ b/src/main/java/org/apache/aurora/scheduler/stats/StatsModule.java @@ -45,6 +45,28 @@ public class StatsModule extends AbstractModule { private static final Arg<Amount<Long, Time>> RETENTION_PERIOD = Arg.create(Amount.of(1L, Time.HOURS)); + interface Params { + Amount<Long, Time> statSamplingInterval(); + + Amount<Long, Time> statRetentionPeriod(); + } + + private final Params params; + + public StatsModule() { + this.params = new Params() { + @Override + public Amount<Long, Time> statSamplingInterval() { + return SAMPLING_INTERVAL.get(); + } + + @Override + public Amount<Long, Time> statRetentionPeriod() { + return RETENTION_PERIOD.get(); + } + }; + } + @Override protected void configure() { requireBinding(ShutdownRegistry.class); @@ -53,10 +75,10 @@ public class StatsModule extends AbstractModule { bind(StatRegistry.class).toInstance(Stats.STAT_REGISTRY); bind(new TypeLiteral<Amount<Long, Time>>() { }) .annotatedWith(Names.named(TimeSeriesRepositoryImpl.SAMPLE_RETENTION_PERIOD)) - .toInstance(RETENTION_PERIOD.get()); + .toInstance(params.statRetentionPeriod()); bind(new TypeLiteral<Amount<Long, Time>>() { }) .annotatedWith(Names.named(TimeSeriesRepositoryImpl.SAMPLE_PERIOD)) - .toInstance(SAMPLING_INTERVAL.get()); + .toInstance(params.statSamplingInterval()); bind(TimeSeriesRepository.class).to(TimeSeriesRepositoryImpl.class); bind(TimeSeriesRepositoryImpl.class).in(Singleton.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java b/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java index cded40b..38bf1aa 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java @@ -64,8 +64,20 @@ public class BackupModule extends PrivateModule { help = "Directory to store backups under. Will be created if it does not exist.") private static final Arg<File> BACKUP_DIR = Arg.create(); + public interface Params { + default Amount<Long, Time> backupInterval() { + return Amount.of(1L, Time.HOURS); + } + + default int maxSavedBackups() { + return 48; + } + + File backupDir(); + } + + private final Params params; private final Class<? extends SnapshotStore<Snapshot>> snapshotStore; - private final File unvalidatedBackupDir; /** * Creates a new backup module. @@ -73,18 +85,35 @@ public class BackupModule extends PrivateModule { * @param snapshotStore Snapshot store implementation class. */ public BackupModule(Class<? extends SnapshotStore<Snapshot>> snapshotStore) { - this(BACKUP_DIR.get(), snapshotStore); + this( + new Params() { + @Override + public Amount<Long, Time> backupInterval() { + return BACKUP_INTERVAL.get(); + } + + @Override + public int maxSavedBackups() { + return MAX_SAVED_BACKUPS.get(); + } + + @Override + public File backupDir() { + return BACKUP_DIR.get(); + } + }, + snapshotStore); } /** * Creates a new backup module using a given backupDir instead of a flagged one. * - * @param backupDir Directory to write backups to. + * @param params Module configuration parameters. * @param snapshotStore Snapshot store implementation class. */ @VisibleForTesting - public BackupModule(File backupDir, Class<? extends SnapshotStore<Snapshot>> snapshotStore) { - this.unvalidatedBackupDir = requireNonNull(backupDir); + public BackupModule(Params params, Class<? extends SnapshotStore<Snapshot>> snapshotStore) { + this.params = requireNonNull(params); this.snapshotStore = requireNonNull(snapshotStore); } @@ -126,6 +155,7 @@ public class BackupModule extends PrivateModule { @Provides File provideBackupDir() { + File unvalidatedBackupDir = params.backupDir(); if (!unvalidatedBackupDir.exists()) { if (unvalidatedBackupDir.mkdirs()) { LOG.info("Created backup dir " + unvalidatedBackupDir.getPath() + "."); @@ -145,6 +175,6 @@ public class BackupModule extends PrivateModule { @Provides BackupConfig provideBackupConfig(File backupDir) { - return new BackupConfig(backupDir, MAX_SAVED_BACKUPS.get(), BACKUP_INTERVAL.get()); + return new BackupConfig(backupDir, params.maxSavedBackups(), params.backupInterval()); } } http://git-wip-us.apache.org/repos/asf/aurora/blob/fe13e4ed/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java index 2b3ee7b..b4deef0 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbModule.java @@ -75,10 +75,25 @@ public final class DbModule extends PrivateModule { private static final Arg<Amount<Long, Time>> SLOW_QUERY_LOG_THRESHOLD = Arg.create(Amount.of(25L, Time.MILLISECONDS)); - @CmdLine(name = "db_row_gc_interval", - help = "Interval on which to scan the database for unused row references.") - private static final Arg<Amount<Long, Time>> DB_ROW_GC_INTERVAL = - Arg.create(Amount.of(2L, Time.HOURS)); + interface Params { + boolean useBetaDbTaskStore(); + + Amount<Long, Time> slowQueryLogThreshold(); + } + + private static Params paramsFromCommandLine() { + return new Params() { + @Override + public boolean useBetaDbTaskStore() { + return USE_DB_TASK_STORE.get(); + } + + @Override + public Amount<Long, Time> slowQueryLogThreshold() { + return SLOW_QUERY_LOG_THRESHOLD.get(); + } + }; + } private static final Set<Class<?>> MAPPER_CLASSES = ImmutableSet.<Class<?>>builder() .add(AttributeMapper.class) @@ -192,7 +207,7 @@ public final class DbModule extends PrivateModule { } private static Module getTaskStoreModule(KeyFactory keyFactory) { - return USE_DB_TASK_STORE.get() + return paramsFromCommandLine().useBetaDbTaskStore() ? new TaskStoreModule(keyFactory) : new InMemStoresModule(keyFactory); } @@ -228,7 +243,8 @@ public final class DbModule extends PrivateModule { addTypeHandlersClasses(TypeHandlers.getAll()); - bind(new TypeLiteral<Amount<Long, Time>>() { }).toInstance(SLOW_QUERY_LOG_THRESHOLD.get()); + bind(new TypeLiteral<Amount<Long, Time>>() { }) + .toInstance(paramsFromCommandLine().slowQueryLogThreshold()); // Exposed for unit tests. bind(TaskConfigManager.class); @@ -295,6 +311,26 @@ public final class DbModule extends PrivateModule { * Module that sets up a periodic database garbage-collection routine. */ public static class GarbageCollectorModule extends AbstractModule { + @CmdLine(name = "db_row_gc_interval", + help = "Interval on which to scan the database for unused row references.") + private static final Arg<Amount<Long, Time>> DB_ROW_GC_INTERVAL = + Arg.create(Amount.of(2L, Time.HOURS)); + + interface Params { + Amount<Long, Time> dbRowGcInterval(); + } + + private final Params params; + + public GarbageCollectorModule() { + this.params = new Params() { + @Override + public Amount<Long, Time> dbRowGcInterval() { + return DB_ROW_GC_INTERVAL.get(); + } + }; + } + @Override protected void configure() { install(new PrivateModule() { @@ -304,8 +340,8 @@ public final class DbModule extends PrivateModule { bind(AbstractScheduledService.Scheduler.class).toInstance( AbstractScheduledService.Scheduler.newFixedRateSchedule( 0L, - DB_ROW_GC_INTERVAL.get().getValue(), - DB_ROW_GC_INTERVAL.get().getUnit().getTimeUnit())); + params.dbRowGcInterval().getValue(), + params.dbRowGcInterval().getUnit().getTimeUnit())); expose(RowGarbageCollector.class); } });
