Repository: aurora Updated Branches: refs/heads/master f28f41a70 -> bb16cade1
Adds the ability to set HOME to the sandbox before running the executor. In instances where the root filesystem is read-only, it is desirable to have the executor/runner extract themselves into the sandbox. Reviewed at https://reviews.apache.org/r/45396/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/bb16cade Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/bb16cade Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/bb16cade Branch: refs/heads/master Commit: bb16cade1b16ab57953532d66e56c7081d4551cb Parents: f28f41a Author: George Sirois <[email protected]> Authored: Tue Mar 29 09:23:54 2016 -0700 Committer: Joshua Cohen <[email protected]> Committed: Tue Mar 29 09:23:54 2016 -0700 ---------------------------------------------------------------------- RELEASE-NOTES.md | 5 ++ config/legacy_untested_classes.txt | 1 - docs/operations/configuration.md | 5 ++ .../configuration/executor/ExecutorModule.java | 46 ++++++++--- .../executor/ExecutorModuleTest.java | 80 ++++++++++++++++++++ 5 files changed, 126 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/bb16cade/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 34f28a1..450d363 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -11,6 +11,11 @@ - Added a new argument `--announcer-hostname` to thermos executor to override hostname in service registry endpoint. See [here](docs/configuration-reference.md#announcer-objects) for details. - Descheduling a cron job that was not actually scheduled will no longer return an error. +- Added a new argument `-thermos_home_in_sandbox` to the scheduler for optionally changing + HOME to the sandbox during thermos executor/runner execution. This is useful in cases + where the root filesystem inside of the container is read-only, as it moves PEX extraction into + the sandbox. See [here](docs/operations/configuration.md#docker-containers) + for more detail. ### Deprecations and removals: http://git-wip-us.apache.org/repos/asf/aurora/blob/bb16cade/config/legacy_untested_classes.txt ---------------------------------------------------------------------- diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt index 144b258..afe954f 100644 --- a/config/legacy_untested_classes.txt +++ b/config/legacy_untested_classes.txt @@ -12,7 +12,6 @@ org/apache/aurora/scheduler/app/SchedulerMain$4 org/apache/aurora/scheduler/async/OfferQueue$OfferQueueImpl$2 org/apache/aurora/scheduler/base/Conversions$2 org/apache/aurora/scheduler/base/Conversions$3 -org/apache/aurora/scheduler/configuration/executor/ExecutorModule org/apache/aurora/scheduler/configuration/executor/ExecutorModule$1 org/apache/aurora/scheduler/cron/quartz/CronSchedulerImpl org/apache/aurora/scheduler/cron/quartz/CronSchedulerImpl$1 http://git-wip-us.apache.org/repos/asf/aurora/blob/bb16cade/docs/operations/configuration.md ---------------------------------------------------------------------- diff --git a/docs/operations/configuration.md b/docs/operations/configuration.md index f9e8844..8f7f92a 100644 --- a/docs/operations/configuration.md +++ b/docs/operations/configuration.md @@ -163,6 +163,11 @@ into all containers on that host. The format is a comma separated list of host_p tuples. For example `-global_container_mounts=/opt/secret_keys_dir:/mnt/secret_keys_dir:ro` mounts `/opt/secret_keys_dir` from the slaves into all launched containers. Valid modes are `ro` and `rw`. +If you would like to run a container with a read-only filesystem, it may also be necessary to +pass to use the scheduler flag `-thermos_home_in_sandbox` in order to set HOME to the sandbox +before the executor runs. This will make sure that the executor/runner PEX extractions happens +inside of the sandbox instead of the container filesystem root. + If you would like to supply your own parameters to `docker run` when launching jobs in docker containers, you may use the following flags: http://git-wip-us.apache.org/repos/asf/aurora/blob/bb16cade/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java index 949c299..add1270 100644 --- a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModule.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Stream; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.inject.AbstractModule; @@ -62,7 +63,7 @@ public class ExecutorModule extends AbstractModule { private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create(); @CmdLine(name = "thermos_executor_resources", - help = "A comma seperated list of additional resources to copy into the sandbox." + help = "A comma separated list of additional resources to copy into the sandbox." + "Note: if thermos_executor_path is not the thermos_executor.pex file itself, " + "this must include it.") private static final Arg<List<String>> THERMOS_EXECUTOR_RESOURCES = @@ -76,6 +77,12 @@ public class ExecutorModule extends AbstractModule { help = "Path to the thermos observer root (by default /var/run/thermos.)") private static final Arg<String> THERMOS_OBSERVER_ROOT = Arg.create("/var/run/thermos"); + @CmdLine(name = "thermos_home_in_sandbox", + help = "If true, changes HOME to the sandbox before running the executor. " + + "This primarily has the effect of causing the executor and runner " + + "to extract themselves into the sandbox.") + private static final Arg<Boolean> THERMOS_HOME_IN_SANDBOX = Arg.create(false); + /** * Extra CPU allocated for each executor. */ @@ -92,20 +99,34 @@ public class ExecutorModule extends AbstractModule { Arg.create(Amount.of(128L, Data.MB)); @CmdLine(name = "global_container_mounts", - help = "A comma seperated list of mount points (in host:container form) to mount " + help = "A comma separated list of mount points (in host:container form) to mount " + "into all (non-mesos) containers.") private static final Arg<List<Volume>> GLOBAL_CONTAINER_MOUNTS = Arg.create(ImmutableList.of()); - private static CommandInfo makeExecutorCommand() { + @VisibleForTesting + static CommandInfo makeExecutorCommand( + String thermosExecutorPath, + List<String> thermosExecutorResources, + boolean thermosHomeInSandbox, + String thermosExecutorFlags) { + Stream<String> resourcesToFetch = Stream.concat( - ImmutableList.of(THERMOS_EXECUTOR_PATH.get()).stream(), - THERMOS_EXECUTOR_RESOURCES.get().stream()); + ImmutableList.of(thermosExecutorPath).stream(), + thermosExecutorResources.stream()); + + StringBuilder sb = new StringBuilder(); + if (thermosHomeInSandbox) { + sb.append("HOME=${MESOS_SANDBOX=.} "); + } + // Default to the value of $MESOS_SANDBOX if present. This is necessary for docker tasks, + // in which case the mesos agent is responsible for setting $MESOS_SANDBOX. + sb.append("${MESOS_SANDBOX=.}/"); + sb.append(uriBasename(thermosExecutorPath)); + sb.append(" "); + sb.append(Optional.ofNullable(thermosExecutorFlags).orElse("")); return CommandInfo.newBuilder() - // Default to the value of $MESOS_SANDBOX if present. This is necessary for docker tasks, - // in which case the mesos agent is responsible for setting $MESOS_SANDBOX. - .setValue("${MESOS_SANDBOX=.}/" + uriBasename(THERMOS_EXECUTOR_PATH.get()) - + " " + Optional.ofNullable(THERMOS_EXECUTOR_FLAGS.get()).orElse("")) + .setValue(sb.toString().trim()) .addAllUris(resourcesToFetch .map(r -> URI.newBuilder().setValue(r).setExecutable(true).build()) .collect(GuavaUtils.toImmutableList())) @@ -135,7 +156,12 @@ public class ExecutorModule extends AbstractModule { .setName("aurora.task") // Necessary as executorId is a required field. .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID) - .setCommand(makeExecutorCommand()) + .setCommand( + makeExecutorCommand( + THERMOS_EXECUTOR_PATH.get(), + THERMOS_EXECUTOR_RESOURCES.get(), + THERMOS_HOME_IN_SANDBOX.get(), + THERMOS_EXECUTOR_FLAGS.get())) .addResources(makeResource(CPUS, EXECUTOR_OVERHEAD_CPUS.get())) .addResources(makeResource(RAM_MB, EXECUTOR_OVERHEAD_RAM.get().as(Data.MB))) .build(), http://git-wip-us.apache.org/repos/asf/aurora/blob/bb16cade/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModuleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModuleTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModuleTest.java new file mode 100644 index 0000000..d597b13 --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorModuleTest.java @@ -0,0 +1,80 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.configuration.executor; + +import java.util.List; + +import com.google.common.collect.ImmutableList; + +import org.apache.aurora.GuavaUtils; +import org.apache.mesos.Protos.CommandInfo; +import org.apache.mesos.Protos.CommandInfo.URI; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ExecutorModuleTest { + + @Test + public void testMakeExecutorCommand() { + testSingleCommand( + "/path/executor.pex", + ImmutableList.of(), + false, + null, + "${MESOS_SANDBOX=.}/executor.pex", + ImmutableList.of("/path/executor.pex")); + + testSingleCommand( + "/path/executor.pex", + ImmutableList.of(), + true, + null, + "HOME=${MESOS_SANDBOX=.} ${MESOS_SANDBOX=.}/executor.pex", + ImmutableList.of("/path/executor.pex")); + + testSingleCommand( + "/path/executor.pex", + ImmutableList.of("/other/thing.pex"), + false, + null, + "${MESOS_SANDBOX=.}/executor.pex", + ImmutableList.of("/path/executor.pex", "/other/thing.pex")); + + testSingleCommand( + "/path/executor.pex", + ImmutableList.of(), + false, + "--extra=args", + "${MESOS_SANDBOX=.}/executor.pex --extra=args", + ImmutableList.of("/path/executor.pex")); + } + + private void testSingleCommand( + String path, + List<String> resources, + boolean homeInSandbox, + String flags, + String expectedCommand, + List<String> expectedUris) { + + CommandInfo info = ExecutorModule.makeExecutorCommand(path, resources, homeInSandbox, flags); + assertEquals(expectedCommand, info.getValue()); + assertEquals(expectedUris, extractUris(info.getUrisList())); + } + + private List<String> extractUris(List<URI> uris) { + return uris.stream().map(URI::getValue).collect(GuavaUtils.toImmutableList()); + } +}
