Repository: aurora Updated Branches: refs/heads/master 8c8661958 -> 86761e81e
Organize executor-related code into a package. Reviewed at https://reviews.apache.org/r/40320/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/86761e81 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/86761e81 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/86761e81 Branch: refs/heads/master Commit: 86761e81effef9c7c0eb1e9ff2c79a55a91beeb1 Parents: 8c86619 Author: Bill Farner <[email protected]> Authored: Mon Nov 16 11:46:05 2015 -0800 Committer: Bill Farner <[email protected]> Committed: Mon Nov 16 11:46:05 2015 -0800 ---------------------------------------------------------------------- config/legacy_untested_classes.txt | 2 + .../aurora/benchmark/SchedulingBenchmarks.java | 2 +- .../aurora/benchmark/StatusUpdateBenchmark.java | 2 +- .../aurora/scheduler/app/SchedulerMain.java | 130 +-------------- .../configuration/executor/ExecutorConfig.java | 70 +++++++++ .../configuration/executor/ExecutorModule.java | 157 +++++++++++++++++++ .../executor/ExecutorSettings.java | 71 +++++++++ .../configuration/executor/Executors.java | 29 ++++ .../scheduler/filter/SchedulingFilterImpl.java | 2 +- .../aurora/scheduler/mesos/ExecutorConfig.java | 70 --------- .../scheduler/mesos/ExecutorSettings.java | 71 --------- .../aurora/scheduler/mesos/Executors.java | 29 ---- .../scheduler/mesos/MesosTaskFactory.java | 1 + .../scheduler/mesos/TestExecutorSettings.java | 3 + .../preemptor/PreemptionVictimFilter.java | 2 +- .../aurora/scheduler/app/SchedulerIT.java | 2 +- .../mesos/MesosTaskFactoryImplTest.java | 2 + .../aurora/scheduler/mesos/TaskExecutors.java | 1 + 18 files changed, 343 insertions(+), 303 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/config/legacy_untested_classes.txt ---------------------------------------------------------------------- diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt index aac822b..07e49b1 100644 --- a/config/legacy_untested_classes.txt +++ b/config/legacy_untested_classes.txt @@ -13,6 +13,8 @@ org/apache/aurora/scheduler/async/OfferQueue$OfferQueueImpl$2 org/apache/aurora/scheduler/base/Conversions$1 org/apache/aurora/scheduler/base/Conversions$2 org/apache/aurora/scheduler/base/Conversions$3 +org/apache/aurora/scheduler/configuration/executor/ExecutorModule +org/apache/aurora/scheduler/configuration/executor/ExecutorModule$1 org/apache/aurora/scheduler/cron/quartz/CronSchedulerImpl org/apache/aurora/scheduler/cron/quartz/CronSchedulerImpl$1 org/apache/aurora/scheduler/http/AbortCallback http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java index b4ca01b..c13a3ba 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java +++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java @@ -38,12 +38,12 @@ import org.apache.aurora.scheduler.TaskIdGenerator; import org.apache.aurora.scheduler.TierManager; import org.apache.aurora.scheduler.async.AsyncModule; import org.apache.aurora.scheduler.async.DelayExecutor; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; import org.apache.aurora.scheduler.filter.SchedulingFilter; import org.apache.aurora.scheduler.filter.SchedulingFilterImpl; import org.apache.aurora.scheduler.mesos.Driver; -import org.apache.aurora.scheduler.mesos.ExecutorSettings; import org.apache.aurora.scheduler.mesos.TestExecutorSettings; import org.apache.aurora.scheduler.offers.OfferManager; import org.apache.aurora.scheduler.preemptor.BiCache; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/jmh/java/org/apache/aurora/benchmark/StatusUpdateBenchmark.java ---------------------------------------------------------------------- diff --git a/src/jmh/java/org/apache/aurora/benchmark/StatusUpdateBenchmark.java b/src/jmh/java/org/apache/aurora/benchmark/StatusUpdateBenchmark.java index 6236a53..892d6ce 100644 --- a/src/jmh/java/org/apache/aurora/benchmark/StatusUpdateBenchmark.java +++ b/src/jmh/java/org/apache/aurora/benchmark/StatusUpdateBenchmark.java @@ -51,6 +51,7 @@ import org.apache.aurora.scheduler.TaskStatusHandler; import org.apache.aurora.scheduler.TaskStatusHandlerImpl; import org.apache.aurora.scheduler.TierManager; import org.apache.aurora.scheduler.base.AsyncUtil; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; import org.apache.aurora.scheduler.filter.SchedulingFilter; @@ -58,7 +59,6 @@ import org.apache.aurora.scheduler.filter.SchedulingFilterImpl; import org.apache.aurora.scheduler.mesos.Driver; import org.apache.aurora.scheduler.mesos.DriverFactory; import org.apache.aurora.scheduler.mesos.DriverSettings; -import org.apache.aurora.scheduler.mesos.ExecutorSettings; import org.apache.aurora.scheduler.mesos.MesosSchedulerImpl; import org.apache.aurora.scheduler.mesos.TestExecutorSettings; import org.apache.aurora.scheduler.offers.OfferManager; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java index fb4f0a0..54814b2 100644 --- a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java +++ b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java @@ -17,18 +17,15 @@ import java.lang.Thread.UncaughtExceptionHandler; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Stream; import javax.inject.Inject; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -39,7 +36,6 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.util.Modules; -import org.apache.aurora.GuavaUtils; import org.apache.aurora.GuavaUtils.ServiceManagerIface; import org.apache.aurora.common.application.Lifecycle; import org.apache.aurora.common.args.Arg; @@ -48,27 +44,20 @@ import org.apache.aurora.common.args.ArgScanner.ArgScanException; import org.apache.aurora.common.args.CmdLine; import org.apache.aurora.common.args.constraints.NotEmpty; import org.apache.aurora.common.args.constraints.NotNull; -import org.apache.aurora.common.base.MorePreconditions; import org.apache.aurora.common.inject.Bindings; import org.apache.aurora.common.logging.RootLogConfig; -import org.apache.aurora.common.quantity.Amount; -import org.apache.aurora.common.quantity.Data; import org.apache.aurora.common.stats.Stats; import org.apache.aurora.common.zookeeper.Group; import org.apache.aurora.common.zookeeper.SingletonService; import org.apache.aurora.common.zookeeper.SingletonService.LeadershipListener; import org.apache.aurora.gen.ServerInfo; -import org.apache.aurora.gen.Volume; import org.apache.aurora.scheduler.AppStartup; -import org.apache.aurora.scheduler.ResourceType; import org.apache.aurora.scheduler.SchedulerLifecycle; +import org.apache.aurora.scheduler.configuration.executor.ExecutorModule; import org.apache.aurora.scheduler.cron.quartz.CronModule; import org.apache.aurora.scheduler.http.HttpService; import org.apache.aurora.scheduler.log.mesos.MesosLogStreamModule; import org.apache.aurora.scheduler.mesos.CommandLineDriverSettingsModule; -import org.apache.aurora.scheduler.mesos.ExecutorConfig; -import org.apache.aurora.scheduler.mesos.ExecutorSettings; -import org.apache.aurora.scheduler.mesos.Executors; import org.apache.aurora.scheduler.mesos.LibMesosLoadingModule; import org.apache.aurora.scheduler.stats.StatsModule; import org.apache.aurora.scheduler.storage.Storage; @@ -80,18 +69,9 @@ import org.apache.aurora.scheduler.storage.log.SnapshotStoreImpl; import org.apache.aurora.scheduler.zookeeper.guice.client.ZooKeeperClientModule; import org.apache.aurora.scheduler.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig; import org.apache.aurora.scheduler.zookeeper.guice.client.flagged.FlaggedClientConfig; -import org.apache.mesos.Protos; -import org.apache.mesos.Protos.CommandInfo; -import org.apache.mesos.Protos.CommandInfo.URI; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.Resource; -import org.apache.mesos.Protos.Value.Scalar; -import org.apache.mesos.Protos.Value.Type; import static org.apache.aurora.common.logging.RootLogConfig.Configuration; import static org.apache.aurora.gen.apiConstants.THRIFT_API_VERSION; -import static org.apache.aurora.scheduler.ResourceType.CPUS; -import static org.apache.aurora.scheduler.ResourceType.RAM_MB; /** * Launcher for the aurora scheduler. @@ -108,39 +88,6 @@ public class SchedulerMain { @CmdLine(name = "serverset_path", help = "ZooKeeper ServerSet path to register at.") private static final Arg<String> SERVERSET_PATH = Arg.create(); - @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor entry point.") - private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create(); - - @CmdLine(name = "thermos_executor_resources", - help = "A comma seperated list of additional resources to copy into the sandbox." - + "Note: if thermos_executor_path is not the thermos_executor.pex file itself, " - + "this must include it.") - private static final Arg<List<String>> THERMOS_EXECUTOR_RESOURCES = - Arg.create(ImmutableList.of()); - - @CmdLine(name = "thermos_executor_flags", - help = "Extra arguments to be passed to the thermos executor") - private static final Arg<String> THERMOS_EXECUTOR_FLAGS = Arg.create(null); - - @CmdLine(name = "thermos_observer_root", - help = "Path to the thermos observer root (by default /var/run/thermos.)") - private static final Arg<String> THERMOS_OBSERVER_ROOT = Arg.create("/var/run/thermos"); - - /** - * Extra CPU allocated for each executor. - */ - @CmdLine(name = "thermos_executor_cpu", - help = "The number of CPU cores to allocate for each instance of the executor.") - private static final Arg<Double> EXECUTOR_OVERHEAD_CPUS = Arg.create(0.25); - - /** - * Extra RAM allocated for the executor. - */ - @CmdLine(name = "thermos_executor_ram", - help = "The amount of RAM to allocate for each instance of the executor.") - private static final Arg<Amount<Long, Data>> EXECUTOR_OVERHEAD_RAM = - Arg.create(Amount.of(128L, Data.MB)); - @CmdLine(name = "extra_modules", help = "A list of modules that provide additional functionality.") private static final Arg<List<Class<? extends Module>>> EXTRA_MODULES = @@ -150,11 +97,6 @@ public class SchedulerMain { @CmdLine(name = "viz_job_url_prefix", help = "URL prefix for job container stats.") private static final Arg<String> STATS_URL_PREFIX = Arg.create(""); - @CmdLine(name = "global_container_mounts", - help = "A comma seperated list of mount points (in host:container form) to mount " - + "into all (non-mesos) containers.") - private static final Arg<List<Volume>> GLOBAL_CONTAINER_MOUNTS = Arg.create(ImmutableList.of()); - @Inject private SingletonService schedulerService; @Inject private HttpService httpService; @Inject private SchedulerLifecycle schedulerLifecycle; @@ -216,42 +158,6 @@ public class SchedulerMain { new DbModule.GarbageCollectorModule()); } - private static Resource makeResource(ResourceType type, double value) { - return Resource.newBuilder() - .setType(Type.SCALAR) - .setName(type.getName()) - .setScalar(Scalar.newBuilder().setValue(value)) - .build(); - } - - private static String uriBasename(String uri) { - int lastSlash = uri.lastIndexOf('/'); - if (lastSlash == -1) { - return uri; - } else { - String basename = uri.substring(lastSlash + 1); - MorePreconditions.checkNotBlank(basename, "URI must not end with a slash."); - - return basename; - } - } - - private static CommandInfo makeExecutorCommand() { - Stream<String> resourcesToFetch = Stream.concat( - ImmutableList.of(THERMOS_EXECUTOR_PATH.get()).stream(), - THERMOS_EXECUTOR_RESOURCES.get().stream()); - - return CommandInfo.newBuilder() - // Default to the value of $MESOS_SANDBOX if present. This is necessary for docker tasks, - // in which case the mesos agent is responsible for setting $MESOS_SANDBOX. - .setValue("${MESOS_SANDBOX=.}/" + uriBasename(THERMOS_EXECUTOR_PATH.get()) - + " " + Optional.ofNullable(THERMOS_EXECUTOR_FLAGS.get()).orElse("")) - .addAllUris(resourcesToFetch - .map(r -> URI.newBuilder().setValue(r).setExecutable(true).build()) - .collect(GuavaUtils.toImmutableList())) - .build(); - } - /** * Runs the scheduler by including modules configured from command line arguments in * addition to the provided environment-specific module. @@ -276,42 +182,10 @@ public class SchedulerMain { new ZooKeeperClientModule(zkClientConfig), new ServiceDiscoveryModule(SERVERSET_PATH.get(), zkClientConfig.credentials), new BackupModule(SnapshotStoreImpl.class), + new ExecutorModule(), new AbstractModule() { @Override protected void configure() { - List<Protos.Volume> volumeMounts = - ImmutableList.<Protos.Volume>builder() - .add(Protos.Volume.newBuilder() - .setHostPath(THERMOS_OBSERVER_ROOT.get()) - .setContainerPath(THERMOS_OBSERVER_ROOT.get()) - .setMode(Protos.Volume.Mode.RW) - .build()) - .addAll(Iterables.transform( - GLOBAL_CONTAINER_MOUNTS.get(), - new Function<Volume, Protos.Volume>() { - @Override - public Protos.Volume apply(Volume v) { - return Protos.Volume.newBuilder() - .setHostPath(v.getHostPath()) - .setContainerPath(v.getContainerPath()) - .setMode(Protos.Volume.Mode.valueOf(v.getMode().getValue())) - .build(); - } - })) - .build(); - - bind(ExecutorSettings.class).toInstance(new ExecutorSettings( - new ExecutorConfig( - ExecutorInfo.newBuilder() - .setName("aurora.task") - // Necessary as executorId is a required field. - .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID) - .setCommand(makeExecutorCommand()) - .addResources(makeResource(CPUS, EXECUTOR_OVERHEAD_CPUS.get())) - .addResources(makeResource(RAM_MB, EXECUTOR_OVERHEAD_RAM.get().as(Data.MB))) - .build(), - volumeMounts))); - bind(IServerInfo.class).toInstance( IServerInfo.build( new ServerInfo() http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorConfig.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorConfig.java new file mode 100644 index 0000000..24329a4 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorConfig.java @@ -0,0 +1,70 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.configuration.executor; + +import java.util.List; +import java.util.Objects; + +import com.google.common.base.MoreObjects; + +import org.apache.mesos.Protos.ExecutorInfo; +import org.apache.mesos.Protos.Volume; + +import static java.util.Objects.requireNonNull; + +/** + * Executor-related configuration used to populate task descriptions. + */ +public class ExecutorConfig { + + private final ExecutorInfo executor; + private final List<Volume> volumeMounts; + + public ExecutorConfig(ExecutorInfo executor, List<Volume> volumeMounts) { + this.executor = requireNonNull(executor); + this.volumeMounts = requireNonNull(volumeMounts); + } + + public ExecutorInfo getExecutor() { + return executor; + } + + public List<Volume> getVolumeMounts() { + return volumeMounts; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExecutorConfig)) { + return false; + } + + ExecutorConfig other = (ExecutorConfig) obj; + return Objects.equals(executor, other.executor) + && Objects.equals(volumeMounts, other.volumeMounts); + } + + @Override + public int hashCode() { + return Objects.hash(executor, volumeMounts); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("executor", executor) + .add("volumeMounts", volumeMounts) + .toString(); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java new file mode 100644 index 0000000..84e6be5 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java @@ -0,0 +1,157 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.configuration.executor; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.inject.AbstractModule; + +import org.apache.aurora.GuavaUtils; +import org.apache.aurora.common.args.Arg; +import org.apache.aurora.common.args.CmdLine; +import org.apache.aurora.common.base.MorePreconditions; +import org.apache.aurora.common.quantity.Amount; +import org.apache.aurora.common.quantity.Data; +import org.apache.aurora.gen.Volume; +import org.apache.aurora.scheduler.ResourceType; +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.CommandInfo; +import org.apache.mesos.Protos.CommandInfo.URI; +import org.apache.mesos.Protos.ExecutorInfo; +import org.apache.mesos.Protos.Resource; +import org.apache.mesos.Protos.Value.Scalar; +import org.apache.mesos.Protos.Value.Type; + +import static org.apache.aurora.scheduler.ResourceType.CPUS; +import static org.apache.aurora.scheduler.ResourceType.RAM_MB; + +/** + * Binding module for {@link ExecutorSettings}. + */ +public class ExecutorModule extends AbstractModule { + @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor entry point.") + private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create(); + + @CmdLine(name = "thermos_executor_resources", + help = "A comma seperated list of additional resources to copy into the sandbox." + + "Note: if thermos_executor_path is not the thermos_executor.pex file itself, " + + "this must include it.") + private static final Arg<List<String>> THERMOS_EXECUTOR_RESOURCES = + Arg.create(ImmutableList.of()); + + @CmdLine(name = "thermos_executor_flags", + help = "Extra arguments to be passed to the thermos executor") + private static final Arg<String> THERMOS_EXECUTOR_FLAGS = Arg.create(null); + + @CmdLine(name = "thermos_observer_root", + help = "Path to the thermos observer root (by default /var/run/thermos.)") + private static final Arg<String> THERMOS_OBSERVER_ROOT = Arg.create("/var/run/thermos"); + + /** + * Extra CPU allocated for each executor. + */ + @CmdLine(name = "thermos_executor_cpu", + help = "The number of CPU cores to allocate for each instance of the executor.") + private static final Arg<Double> EXECUTOR_OVERHEAD_CPUS = Arg.create(0.25); + + /** + * Extra RAM allocated for the executor. + */ + @CmdLine(name = "thermos_executor_ram", + help = "The amount of RAM to allocate for each instance of the executor.") + private static final Arg<Amount<Long, Data>> EXECUTOR_OVERHEAD_RAM = + Arg.create(Amount.of(128L, Data.MB)); + + @CmdLine(name = "global_container_mounts", + help = "A comma seperated list of mount points (in host:container form) to mount " + + "into all (non-mesos) containers.") + private static final Arg<List<Volume>> GLOBAL_CONTAINER_MOUNTS = Arg.create(ImmutableList.of()); + + private static CommandInfo makeExecutorCommand() { + Stream<String> resourcesToFetch = Stream.concat( + ImmutableList.of(THERMOS_EXECUTOR_PATH.get()).stream(), + THERMOS_EXECUTOR_RESOURCES.get().stream()); + + return CommandInfo.newBuilder() + // Default to the value of $MESOS_SANDBOX if present. This is necessary for docker tasks, + // in which case the mesos agent is responsible for setting $MESOS_SANDBOX. + .setValue("${MESOS_SANDBOX=.}/" + uriBasename(THERMOS_EXECUTOR_PATH.get()) + + " " + Optional.ofNullable(THERMOS_EXECUTOR_FLAGS.get()).orElse("")) + .addAllUris(resourcesToFetch + .map(r -> URI.newBuilder().setValue(r).setExecutable(true).build()) + .collect(GuavaUtils.toImmutableList())) + .build(); + } + + @Override + protected void configure() { + List<Protos.Volume> volumeMounts = + ImmutableList.<Protos.Volume>builder() + .add(Protos.Volume.newBuilder() + .setHostPath(THERMOS_OBSERVER_ROOT.get()) + .setContainerPath(THERMOS_OBSERVER_ROOT.get()) + .setMode(Protos.Volume.Mode.RW) + .build()) + .addAll(Iterables.transform( + GLOBAL_CONTAINER_MOUNTS.get(), + new Function<Volume, Protos.Volume>() { + @Override + public Protos.Volume apply(Volume v) { + return Protos.Volume.newBuilder() + .setHostPath(v.getHostPath()) + .setContainerPath(v.getContainerPath()) + .setMode(Protos.Volume.Mode.valueOf(v.getMode().getValue())) + .build(); + } + })) + .build(); + + bind(ExecutorSettings.class).toInstance(new ExecutorSettings( + new ExecutorConfig( + ExecutorInfo.newBuilder() + .setName("aurora.task") + // Necessary as executorId is a required field. + .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID) + .setCommand(makeExecutorCommand()) + .addResources(makeResource(CPUS, EXECUTOR_OVERHEAD_CPUS.get())) + .addResources(makeResource(RAM_MB, EXECUTOR_OVERHEAD_RAM.get().as(Data.MB))) + .build(), + volumeMounts))); + } + + private static Resource makeResource(ResourceType type, double value) { + return Resource.newBuilder() + .setType(Type.SCALAR) + .setName(type.getName()) + .setScalar(Scalar.newBuilder().setValue(value)) + .build(); + } + + private static String uriBasename(String uri) { + int lastSlash = uri.lastIndexOf('/'); + if (lastSlash == -1) { + return uri; + } else { + String basename = uri.substring(lastSlash + 1); + MorePreconditions.checkNotBlank(basename, "URI must not end with a slash."); + + return basename; + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettings.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettings.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettings.java new file mode 100644 index 0000000..7beea81 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettings.java @@ -0,0 +1,71 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.configuration.executor; + +import java.util.Objects; + +import org.apache.aurora.common.quantity.Amount; +import org.apache.aurora.common.quantity.Data; +import org.apache.aurora.scheduler.ResourceSlot; +import org.apache.aurora.scheduler.ResourceType; + +import static java.util.Objects.requireNonNull; + +/** + * Configuration for the executor to run, and resource overhead required for it. + */ +public class ExecutorSettings { + private final ExecutorConfig config; + + public ExecutorSettings(ExecutorConfig config) { + this.config = requireNonNull(config); + } + + public ExecutorConfig getExecutorConfig() { + // TODO(wfarner): Replace this with a generic name-based accessor once tasks can specify the + // executor they wish to use. + return config; + } + + private double getExecutorResourceValue(ResourceType resource) { + return config.getExecutor().getResourcesList().stream() + .filter(r -> r.getName().equals(resource.getName())) + .findFirst() + .map(r -> r.getScalar().getValue()) + .orElse(0D); + } + + public ResourceSlot getExecutorOverhead() { + return new ResourceSlot( + getExecutorResourceValue(ResourceType.CPUS), + Amount.of((long) getExecutorResourceValue(ResourceType.RAM_MB), Data.MB), + Amount.of((long) getExecutorResourceValue(ResourceType.DISK_MB), Data.MB), + 0); + } + + @Override + public int hashCode() { + return Objects.hash(config); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExecutorSettings)) { + return false; + } + + ExecutorSettings other = (ExecutorSettings) obj; + return Objects.equals(config, other.config); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/configuration/executor/Executors.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/Executors.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/Executors.java new file mode 100644 index 0000000..6ac2d00 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/Executors.java @@ -0,0 +1,29 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.configuration.executor; + +import org.apache.mesos.Protos.ExecutorID; + +/** + * Utility class for executor-related fields. + */ +public final class Executors { + private Executors() { + // Utility class. + } + + public static final ExecutorID PLACEHOLDER_EXECUTOR_ID = ExecutorID.newBuilder() + .setValue("PLACEHOLDER") + .build(); +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java index 52776c9..625adcb 100644 --- a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java +++ b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java @@ -32,7 +32,7 @@ import org.apache.aurora.gen.MaintenanceMode; import org.apache.aurora.gen.TaskConstraint; import org.apache.aurora.scheduler.ResourceSlot; import org.apache.aurora.scheduler.configuration.ConfigurationManager; -import org.apache.aurora.scheduler.mesos.ExecutorSettings; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.storage.entities.IAttribute; import org.apache.aurora.scheduler.storage.entities.IConstraint; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorConfig.java b/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorConfig.java deleted file mode 100644 index b6aa2e1..0000000 --- a/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorConfig.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.mesos; - -import java.util.List; -import java.util.Objects; - -import com.google.common.base.MoreObjects; - -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.Volume; - -import static java.util.Objects.requireNonNull; - -/** - * Executor-related configuration used to populate task descriptions. - */ -public class ExecutorConfig { - - private final ExecutorInfo executor; - private final List<Volume> volumeMounts; - - public ExecutorConfig(ExecutorInfo executor, List<Volume> volumeMounts) { - this.executor = requireNonNull(executor); - this.volumeMounts = requireNonNull(volumeMounts); - } - - public ExecutorInfo getExecutor() { - return executor; - } - - public List<Volume> getVolumeMounts() { - return volumeMounts; - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ExecutorConfig)) { - return false; - } - - ExecutorConfig other = (ExecutorConfig) obj; - return Objects.equals(executor, other.executor) - && Objects.equals(volumeMounts, other.volumeMounts); - } - - @Override - public int hashCode() { - return Objects.hash(executor, volumeMounts); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("executor", executor) - .add("volumeMounts", volumeMounts) - .toString(); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorSettings.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorSettings.java b/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorSettings.java deleted file mode 100644 index b7f3023..0000000 --- a/src/main/java/org/apache/aurora/scheduler/mesos/ExecutorSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.mesos; - -import java.util.Objects; - -import org.apache.aurora.common.quantity.Amount; -import org.apache.aurora.common.quantity.Data; -import org.apache.aurora.scheduler.ResourceSlot; -import org.apache.aurora.scheduler.ResourceType; - -import static java.util.Objects.requireNonNull; - -/** - * Configuration for the executor to run, and resource overhead required for it. - */ -public class ExecutorSettings { - private final ExecutorConfig config; - - public ExecutorSettings(ExecutorConfig config) { - this.config = requireNonNull(config); - } - - public ExecutorConfig getExecutorConfig() { - // TODO(wfarner): Replace this with a generic name-based accessor once tasks can specify the - // executor they wish to use. - return config; - } - - private double getExecutorResourceValue(ResourceType resource) { - return config.getExecutor().getResourcesList().stream() - .filter(r -> r.getName().equals(resource.getName())) - .findFirst() - .map(r -> r.getScalar().getValue()) - .orElse(0D); - } - - public ResourceSlot getExecutorOverhead() { - return new ResourceSlot( - getExecutorResourceValue(ResourceType.CPUS), - Amount.of((long) getExecutorResourceValue(ResourceType.RAM_MB), Data.MB), - Amount.of((long) getExecutorResourceValue(ResourceType.DISK_MB), Data.MB), - 0); - } - - @Override - public int hashCode() { - return Objects.hash(config); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ExecutorSettings)) { - return false; - } - - ExecutorSettings other = (ExecutorSettings) obj; - return Objects.equals(config, other.config); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/mesos/Executors.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/Executors.java b/src/main/java/org/apache/aurora/scheduler/mesos/Executors.java deleted file mode 100644 index 21152f5..0000000 --- a/src/main/java/org/apache/aurora/scheduler/mesos/Executors.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.mesos; - -import org.apache.mesos.Protos.ExecutorID; - -/** - * Utility class for executor-related fields. - */ -public final class Executors { - private Executors() { - // Utility class. - } - - public static final ExecutorID PLACEHOLDER_EXECUTOR_ID = ExecutorID.newBuilder() - .setValue("PLACEHOLDER") - .build(); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java index f1c2059..1b03f47 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java @@ -32,6 +32,7 @@ import org.apache.aurora.scheduler.TierManager; import org.apache.aurora.scheduler.base.JobKeys; import org.apache.aurora.scheduler.base.SchedulerException; import org.apache.aurora.scheduler.base.Tasks; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.storage.entities.IAssignedTask; import org.apache.aurora.scheduler.storage.entities.IDockerContainer; import org.apache.aurora.scheduler.storage.entities.IDockerParameter; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java b/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java index ad5927c..556a3f8 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java @@ -18,6 +18,9 @@ import com.google.common.collect.ImmutableList; import org.apache.aurora.scheduler.ResourceSlot; import org.apache.aurora.scheduler.ResourceType; import org.apache.aurora.scheduler.TierInfo; +import org.apache.aurora.scheduler.configuration.executor.ExecutorConfig; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; +import org.apache.aurora.scheduler.configuration.executor.Executors; import org.apache.mesos.Protos.CommandInfo; import org.apache.mesos.Protos.CommandInfo.URI; import org.apache.mesos.Protos.ExecutorInfo; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilter.java b/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilter.java index b14ab68..edfa202 100644 --- a/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilter.java +++ b/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilter.java @@ -30,12 +30,12 @@ import org.apache.aurora.scheduler.HostOffer; import org.apache.aurora.scheduler.ResourceSlot; import org.apache.aurora.scheduler.Resources; import org.apache.aurora.scheduler.TierManager; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.filter.AttributeAggregate; import org.apache.aurora.scheduler.filter.SchedulingFilter; import org.apache.aurora.scheduler.filter.SchedulingFilter.ResourceRequest; import org.apache.aurora.scheduler.filter.SchedulingFilter.UnusedResource; import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto; -import org.apache.aurora.scheduler.mesos.ExecutorSettings; import org.apache.aurora.scheduler.storage.Storage.StoreProvider; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java index da7d1e0..88253d0 100644 --- a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java +++ b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java @@ -72,13 +72,13 @@ import org.apache.aurora.gen.storage.Transaction; import org.apache.aurora.gen.storage.storageConstants; import org.apache.aurora.scheduler.AppStartup; import org.apache.aurora.scheduler.ResourceSlot; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.log.Log; import org.apache.aurora.scheduler.log.Log.Entry; import org.apache.aurora.scheduler.log.Log.Position; import org.apache.aurora.scheduler.log.Log.Stream; import org.apache.aurora.scheduler.mesos.DriverFactory; import org.apache.aurora.scheduler.mesos.DriverSettings; -import org.apache.aurora.scheduler.mesos.ExecutorSettings; import org.apache.aurora.scheduler.mesos.TestExecutorSettings; import org.apache.aurora.scheduler.storage.backup.BackupModule; import org.apache.aurora.scheduler.storage.entities.IServerInfo; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java index 50853cf..a5793bf 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java @@ -29,6 +29,8 @@ import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.ResourceSlot; import org.apache.aurora.scheduler.Resources; import org.apache.aurora.scheduler.TierManager; +import org.apache.aurora.scheduler.configuration.executor.ExecutorConfig; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; import org.apache.aurora.scheduler.mesos.MesosTaskFactory.MesosTaskFactoryImpl; import org.apache.aurora.scheduler.storage.entities.IAssignedTask; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; http://git-wip-us.apache.org/repos/asf/aurora/blob/86761e81/src/test/java/org/apache/aurora/scheduler/mesos/TaskExecutors.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/TaskExecutors.java b/src/test/java/org/apache/aurora/scheduler/mesos/TaskExecutors.java index a258d06..155ece4 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/TaskExecutors.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/TaskExecutors.java @@ -16,6 +16,7 @@ package org.apache.aurora.scheduler.mesos; import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Data; import org.apache.aurora.scheduler.ResourceSlot; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettings; /** * Utility class to contain constants related to setting up executor settings.
