Repository: aurora Updated Branches: refs/heads/master 0df94f6e7 -> 7e32c4381
AURORA-1616: [part 1] refactor tier management initialization to make it pluggable in tests. Also, renamed tiers-example.json to tiers.json. Reviewed at https://reviews.apache.org/r/43669/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/7e32c438 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/7e32c438 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/7e32c438 Branch: refs/heads/master Commit: 7e32c43814eff53b8cff974efc476b6557aeb7d5 Parents: 0df94f6 Author: Amol Deshmukh <[email protected]> Authored: Tue Feb 23 17:02:55 2016 -0800 Committer: Maxim Khutornenko <[email protected]> Committed: Tue Feb 23 17:02:55 2016 -0800 ---------------------------------------------------------------------- docs/deploying-aurora-scheduler.md | 2 +- .../upstart/aurora-scheduler-kerberos.conf | 2 +- examples/vagrant/upstart/aurora-scheduler.conf | 2 +- .../aurora/scheduler/SchedulerModule.java | 46 ---------- .../apache/aurora/scheduler/TierManager.java | 3 +- .../org/apache/aurora/scheduler/TierModule.java | 89 ++++++++++++++++++++ .../aurora/scheduler/app/SchedulerMain.java | 4 +- .../aurora/scheduler/SchedulerModuleTest.java | 2 +- .../aurora/scheduler/TierManagerTest.java | 2 +- .../aurora/scheduler/app/SchedulerIT.java | 3 + .../scheduler/app/local/LocalSchedulerMain.java | 2 + .../apache/aurora/scheduler/tiers-example.json | 9 -- .../org/apache/aurora/scheduler/tiers.json | 9 ++ 13 files changed, 113 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/docs/deploying-aurora-scheduler.md ---------------------------------------------------------------------- diff --git a/docs/deploying-aurora-scheduler.md b/docs/deploying-aurora-scheduler.md index 5f89ca9..883e0f3 100644 --- a/docs/deploying-aurora-scheduler.md +++ b/docs/deploying-aurora-scheduler.md @@ -250,7 +250,7 @@ Specify a tier configuration file path: -tier_config=path/to/tiers/config.json -Example [tier configuration file](../src/test/resources/org/apache/aurora/scheduler/tiers-example.json). +Example [tier configuration file](../src/test/resources/org/apache/aurora/scheduler/tiers.json). ### Maintaining an Aurora Installation http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/examples/vagrant/upstart/aurora-scheduler-kerberos.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf index 53db93c..d57ef37 100644 --- a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf +++ b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf @@ -53,4 +53,4 @@ exec bin/aurora-scheduler \ -kerberos_server_keytab=/home/vagrant/krb5-1.13.1/build/testdir/HTTP-aurora.local.keytab \ -kerberos_server_principal=HTTP/[email protected] \ -enable_h2_console=true \ - -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json + -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers.json http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf index 70e56b0..54fcde6 100644 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ b/examples/vagrant/upstart/aurora-scheduler.conf @@ -46,6 +46,6 @@ exec bin/aurora-scheduler \ -use_beta_db_task_store=true \ -shiro_ini_path=etc/shiro.example.ini \ -enable_h2_console=true \ - -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json \ + -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers.json \ -mesos_role=aurora-role \ -receive_revocable_resources=true http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java index ddc0d05..4a7ef0b 100644 --- a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java @@ -13,41 +13,29 @@ */ package org.apache.aurora.scheduler; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; - import javax.inject.Singleton; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; -import com.google.common.base.Throwables; -import com.google.common.io.Files; import com.google.inject.AbstractModule; import com.google.inject.PrivateModule; import com.google.inject.TypeLiteral; import org.apache.aurora.common.args.Arg; import org.apache.aurora.common.args.CmdLine; -import org.apache.aurora.common.args.constraints.CanRead; import org.apache.aurora.common.args.constraints.Positive; import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Time; import org.apache.aurora.scheduler.SchedulerLifecycle.LeadingOptions; import org.apache.aurora.scheduler.TaskIdGenerator.TaskIdGeneratorImpl; -import org.apache.aurora.scheduler.TierManager.TierManagerImpl; import org.apache.aurora.scheduler.base.AsyncUtil; import org.apache.aurora.scheduler.events.PubsubEventModule; import org.apache.mesos.Protos; -import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.aurora.scheduler.SchedulerServicesModule.addSchedulerActiveServiceBinding; -import static org.apache.aurora.scheduler.TierManager.TierManagerImpl.TierConfig; /** * Binding module for top-level scheduling logic. @@ -71,11 +59,6 @@ public class SchedulerModule extends AbstractModule { help = "The maximum number of status updates that can be processed in a batch.") private static final Arg<Integer> MAX_STATUS_UPDATE_BATCH_SIZE = Arg.create(1000); - @CanRead - @CmdLine(name = "tier_config", - help = "Configuration file defining supported task tiers, task traits and behaviors.") - private static final Arg<File> TIER_CONFIG_FILE = Arg.create(); - @Override protected void configure() { bind(TaskIdGenerator.class).to(TaskIdGeneratorImpl.class); @@ -109,36 +92,7 @@ public class SchedulerModule extends AbstractModule { bind(TaskStatusHandler.class).to(TaskStatusHandlerImpl.class); bind(TaskStatusHandlerImpl.class).in(Singleton.class); - - bind(TierConfig.class).toInstance(parseTierConfig(readTierFile())); - bind(TierManager.class).to(TierManagerImpl.class); - bind(TierManagerImpl.class).in(Singleton.class); addSchedulerActiveServiceBinding(binder()).to(TaskStatusHandlerImpl.class); } - private static Optional<String> readTierFile() { - if (TIER_CONFIG_FILE.hasAppliedValue()) { - try { - return Optional.of(Files.toString(TIER_CONFIG_FILE.get(), StandardCharsets.UTF_8)); - } catch (IOException e) { - LOG.error("Error loading tier configuration file."); - throw Throwables.propagate(e); - } - } - - return Optional.<String>absent(); - } - - @VisibleForTesting - static TierConfig parseTierConfig(Optional<String> config) { - Optional<TierConfig> map = config.transform(input -> { - try { - return new ObjectMapper().readValue(input, TierConfig.class); - } catch (IOException e) { - LOG.error("Error parsing tier configuration file."); - throw Throwables.propagate(e); - } - }); - return map.or(TierConfig.EMPTY); - } } http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/main/java/org/apache/aurora/scheduler/TierManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/TierManager.java b/src/main/java/org/apache/aurora/scheduler/TierManager.java index 7367df4..fce6e51 100644 --- a/src/main/java/org/apache/aurora/scheduler/TierManager.java +++ b/src/main/java/org/apache/aurora/scheduler/TierManager.java @@ -51,8 +51,9 @@ public interface TierManager { private final Map<String, TierInfo> tiers; + @VisibleForTesting @JsonCreator - TierConfig(@JsonProperty("tiers") Map<String, TierInfo> tiers) { + public TierConfig(@JsonProperty("tiers") Map<String, TierInfo> tiers) { this.tiers = ImmutableMap.copyOf(tiers); } http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/main/java/org/apache/aurora/scheduler/TierModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/TierModule.java b/src/main/java/org/apache/aurora/scheduler/TierModule.java new file mode 100644 index 0000000..b5f065e --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/TierModule.java @@ -0,0 +1,89 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Throwables; +import com.google.common.io.Files; +import com.google.inject.AbstractModule; + +import org.apache.aurora.common.args.Arg; +import org.apache.aurora.common.args.CmdLine; +import org.apache.aurora.common.args.constraints.CanRead; +import org.apache.aurora.scheduler.TierManager.TierManagerImpl; +import org.apache.aurora.scheduler.TierManager.TierManagerImpl.TierConfig; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Objects.requireNonNull; + +/** + * Binding module for tier management. + */ +public class TierModule extends AbstractModule { + private static final Logger LOG = LoggerFactory.getLogger(TierModule.class); + + @CanRead + @CmdLine(name = "tier_config", + help = "Configuration file defining supported task tiers, task traits and behaviors.") + private static final Arg<File> TIER_CONFIG_FILE = Arg.create(); + + private final TierConfig tierConfig; + + public TierModule() { + this(parseTierConfig(readTierFile())); + } + + @VisibleForTesting + public TierModule(TierConfig tierConfig) { + this.tierConfig = requireNonNull(tierConfig); + } + + @Override + protected void configure() { + bind(TierManager.class).toInstance(new TierManagerImpl(tierConfig)); + } + + static Optional<String> readTierFile() { + if (TIER_CONFIG_FILE.hasAppliedValue()) { + try { + return Optional.of(Files.toString(TIER_CONFIG_FILE.get(), StandardCharsets.UTF_8)); + } catch (IOException e) { + LOG.error("Error loading tier configuration file."); + throw Throwables.propagate(e); + } + } + + return Optional.<String>absent(); + } + + @VisibleForTesting + static TierConfig parseTierConfig(Optional<String> config) { + Optional<TierConfig> map = config.transform(input -> { + try { + return new ObjectMapper().readValue(input, TierConfig.class); + } catch (IOException e) { + LOG.error("Error parsing tier configuration file."); + throw Throwables.propagate(e); + } + }); + return map.or(TierConfig.EMPTY); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java index cd9cb09..60b66e8 100644 --- a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java +++ b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java @@ -48,6 +48,7 @@ import org.apache.aurora.common.zookeeper.SingletonService.LeadershipListener; import org.apache.aurora.gen.ServerInfo; import org.apache.aurora.scheduler.AppStartup; import org.apache.aurora.scheduler.SchedulerLifecycle; +import org.apache.aurora.scheduler.TierModule; import org.apache.aurora.scheduler.configuration.executor.ExecutorModule; import org.apache.aurora.scheduler.cron.quartz.CronModule; import org.apache.aurora.scheduler.http.HttpService; @@ -201,7 +202,8 @@ public class SchedulerMain { new CommandLineDriverSettingsModule(), new LibMesosLoadingModule(), new MesosLogStreamModule(FlaggedClientConfig.create()), - new LogStorageModule()) + new LogStorageModule(), + new TierModule()) .build(); flagConfiguredMain(Modules.combine(modules)); } http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java b/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java index 9dd0569..aa6e035 100644 --- a/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java +++ b/src/test/java/org/apache/aurora/scheduler/SchedulerModuleTest.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; -import static org.apache.aurora.scheduler.SchedulerModule.parseTierConfig; +import static org.apache.aurora.scheduler.TierModule.parseTierConfig; import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.junit.Assert.assertEquals; http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java index 350b90b..4da829f 100644 --- a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java @@ -21,8 +21,8 @@ import org.apache.aurora.scheduler.TierManager.TierManagerImpl.TierConfig; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.junit.Test; -import static org.apache.aurora.scheduler.SchedulerModule.parseTierConfig; import static org.apache.aurora.scheduler.TierInfo.DEFAULT; +import static org.apache.aurora.scheduler.TierModule.parseTierConfig; import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER; import static org.junit.Assert.assertEquals; http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java index 39916f5..7ee31fd 100644 --- a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java +++ b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java @@ -61,6 +61,8 @@ import org.apache.aurora.gen.storage.Transaction; import org.apache.aurora.gen.storage.storageConstants; import org.apache.aurora.scheduler.AppStartup; import org.apache.aurora.scheduler.ResourceSlot; +import org.apache.aurora.scheduler.TierManager.TierManagerImpl.TierConfig; +import org.apache.aurora.scheduler.TierModule; import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.log.Log; @@ -200,6 +202,7 @@ public class SchedulerIT extends BaseZooKeeperTest { Injector injector = Guice.createInjector( ImmutableList.<Module>builder() .add(SchedulerMain.getUniversalModule()) + .add(new TierModule(TierConfig.EMPTY)) .add(new LogStorageModule()) .add(new ZooKeeperClientModule(zkClientConfig)) .add(new ServiceDiscoveryModule(SERVERSET_PATH, credentials)) http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/test/java/org/apache/aurora/scheduler/app/local/LocalSchedulerMain.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/app/local/LocalSchedulerMain.java b/src/test/java/org/apache/aurora/scheduler/app/local/LocalSchedulerMain.java index 4716b66..fb27657 100644 --- a/src/test/java/org/apache/aurora/scheduler/app/local/LocalSchedulerMain.java +++ b/src/test/java/org/apache/aurora/scheduler/app/local/LocalSchedulerMain.java @@ -70,6 +70,8 @@ public final class LocalSchedulerMain { .add("-shiro_ini_path=" + ResourceUtils.CLASSPATH_PREFIX + "org/apache/aurora/scheduler/http/api/security/shiro-example.ini") + .add("-tier_config=" + + ResourceUtils.CLASSPATH_PREFIX + "org/apache/aurora/scheduler/tiers.json") .add("-enable_h2_console=true") .build(); SchedulerMain.applyStaticArgumentValues(arguments.toArray(new String[] {})); http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/test/resources/org/apache/aurora/scheduler/tiers-example.json ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/tiers-example.json b/src/test/resources/org/apache/aurora/scheduler/tiers-example.json deleted file mode 100644 index 2140773..0000000 --- a/src/test/resources/org/apache/aurora/scheduler/tiers-example.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "tiers": - { - "revocable": - { - "revocable": true - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/7e32c438/src/test/resources/org/apache/aurora/scheduler/tiers.json ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/tiers.json b/src/test/resources/org/apache/aurora/scheduler/tiers.json new file mode 100644 index 0000000..2140773 --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/tiers.json @@ -0,0 +1,9 @@ +{ + "tiers": + { + "revocable": + { + "revocable": true + } + } +}
