Introduce a shared filesytem isolator. Isolator supports creating private copies of system directories, e.g., /tmp, for each container while sharing the host's filesystem.
Review: https://reviews.apache.org/r/25549 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f511395e Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f511395e Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f511395e Branch: refs/heads/master Commit: f511395e84b009881e10b46d2da20b673bf452cb Parents: c18a50a Author: Ian Downes <[email protected]> Authored: Wed Oct 1 11:05:19 2014 -0700 Committer: Ian Downes <[email protected]> Committed: Mon Oct 27 10:36:37 2014 -0700 ---------------------------------------------------------------------- include/mesos/mesos.proto | 6 +- src/Makefile.am | 4 + src/common/parse.hpp | 14 + src/common/type_utils.hpp | 8 + .../isolators/filesystem/shared.cpp | 263 +++++++++++++++++++ .../isolators/filesystem/shared.hpp | 75 ++++++ src/slave/containerizer/linux_launcher.cpp | 10 +- src/slave/containerizer/mesos/containerizer.cpp | 24 +- src/slave/flags.hpp | 22 ++ src/slave/slave.cpp | 16 +- src/tests/isolator_tests.cpp | 176 +++++++++++++ src/tests/mesos.hpp | 8 + 12 files changed, 614 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/include/mesos/mesos.proto ---------------------------------------------------------------------- diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto index 6b93e90..168a7a8 100644 --- a/include/mesos/mesos.proto +++ b/include/mesos/mesos.proto @@ -837,7 +837,8 @@ message Volume { // Absolute path pointing to a directory or file in the container. required string container_path = 1; - // Absolute path pointing to a directory or file on the host. + // Absolute path pointing to a directory or file on the host or a path + // relative to the container work directory. optional string host_path = 2; enum Mode { @@ -851,12 +852,13 @@ message Volume { /** * Describes a container configuration and allows extensible - * configurations for different container implementation. + * configurations for different container implementations. */ message ContainerInfo { // All container implementation types. enum Type { DOCKER = 1; + MESOS = 2; } message DockerInfo { http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index 6820d8a..f177d87 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -343,6 +343,7 @@ if OS_LINUX libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/cgroups/cpushare.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/cgroups/mem.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/cgroups/perf_event.cpp + libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/filesystem/shared.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp else EXTRA_DIST += linux/cgroups.cpp @@ -437,12 +438,15 @@ libmesos_no_3rdparty_la_SOURCES += \ slave/containerizer/isolator.hpp \ slave/containerizer/launcher.hpp \ slave/containerizer/linux_launcher.hpp \ + slave/containerizer/mesos/containerizer.hpp \ + slave/containerizer/mesos/launch.hpp \ slave/containerizer/isolators/posix.hpp \ slave/containerizer/isolators/cgroups/constants.hpp \ slave/containerizer/isolators/cgroups/cpushare.hpp \ slave/containerizer/isolators/cgroups/mem.hpp \ slave/containerizer/isolators/cgroups/perf_event.hpp \ slave/containerizer/mesos/containerizer.hpp \ + slave/containerizer/isolators/filesystem/shared.hpp \ slave/containerizer/mesos/launch.hpp \ tests/cluster.hpp \ tests/containerizer.hpp \ http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/common/parse.hpp ---------------------------------------------------------------------- diff --git a/src/common/parse.hpp b/src/common/parse.hpp index c9ca30f..ae581e5 100644 --- a/src/common/parse.hpp +++ b/src/common/parse.hpp @@ -68,6 +68,20 @@ inline Try<mesos::internal::Modules> parse(const std::string& value) return protobuf::parse<mesos::internal::Modules>(json.get()); } + +template<> +inline Try<mesos::ContainerInfo> parse(const std::string& value) +{ + // Convert from string or file to JSON. + Try<JSON::Object> json = parse<JSON::Object>(value); + if (json.isError()) { + return Error(json.error()); + } + + // Convert from JSON to Protobuf. + return protobuf::parse<mesos::ContainerInfo>(json.get()); +} + } // namespace flags { #endif // __COMMON_PARSE_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/common/type_utils.hpp ---------------------------------------------------------------------- diff --git a/src/common/type_utils.hpp b/src/common/type_utils.hpp index f16beb8..2d22db8 100644 --- a/src/common/type_utils.hpp +++ b/src/common/type_utils.hpp @@ -55,6 +55,14 @@ inline bool operator == (const ExecutorID& left, const ExecutorID& right) } +inline std::ostream& operator << ( + std::ostream& stream, + const ContainerInfo& containerInfo) +{ + return stream << containerInfo.DebugString(); +} + + inline bool operator == (const FrameworkID& left, const FrameworkID& right) { return left.value() == right.value(); http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/slave/containerizer/isolators/filesystem/shared.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/isolators/filesystem/shared.cpp b/src/slave/containerizer/isolators/filesystem/shared.cpp new file mode 100644 index 0000000..49510b2 --- /dev/null +++ b/src/slave/containerizer/isolators/filesystem/shared.cpp @@ -0,0 +1,263 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <set> + +#include "slave/containerizer/isolators/filesystem/shared.hpp" + +using namespace process; + +using std::list; +using std::set; +using std::string; + +namespace mesos { +namespace internal { +namespace slave { + +SharedFilesystemIsolatorProcess::SharedFilesystemIsolatorProcess( + const Flags& _flags) + : flags(_flags) {} + + +SharedFilesystemIsolatorProcess::~SharedFilesystemIsolatorProcess() {} + + +Try<Isolator*> SharedFilesystemIsolatorProcess::create(const Flags& flags) +{ + Result<string> user = os::user(); + if (!user.isSome()) { + return Error("Failed to determine user: " + + (user.isError() ? user.error() : "username not found")); + } + + if (user.get() != "root") { + return Error("SharedFilesystemIsolator requires root privileges"); + } + + process::Owned<IsolatorProcess> process( + new SharedFilesystemIsolatorProcess(flags)); + + return new Isolator(process); +} + + +Future<Nothing> SharedFilesystemIsolatorProcess::recover( + const list<state::RunState>& states) +{ + // There is nothing to recover because we do not keep any state and + // do not monitor filesystem usage or perform any action on cleanup. + return Nothing(); +} + + +Future<Option<CommandInfo> > SharedFilesystemIsolatorProcess::prepare( + const ContainerID& containerId, + const ExecutorInfo& executorInfo, + const string& directory) +{ + if (executorInfo.has_container() && + executorInfo.container().type() != ContainerInfo::MESOS) { + return Failure("Can only prepare filesystem for a MESOS container"); + } + + LOG(INFO) << "Preparing shared filesystem for container: " + << stringify(containerId); + + if (!executorInfo.has_container()) { + // We don't consider this an error, there's just nothing to do so + // we return None. + + return None(); + } + + // We don't support mounting to a container path which is a parent + // to another container path as this can mask entries. We'll keep + // track of all container paths so we can check this. + set<string> containerPaths; + containerPaths.insert(directory); + + list<string> commands; + + foreach (const Volume& volume, executorInfo.container().volumes()) { + // Because the filesystem is shared we require the container path + // already exist, otherwise containers can create arbitrary paths + // outside their sandbox. + if (!os::exists(volume.container_path())) { + return Failure("Volume with container path '" + + volume.container_path() + + "' must exist on host for shared filesystem isolator"); + } + + // Host path must be provided. + if (!volume.has_host_path()) { + return Failure("Volume with container path '" + + volume.container_path() + + "' must specify host path for shared filesystem isolator"); + } + + // Check we won't mask another volume. + // NOTE: Assuming here that the container path is absolute, see + // Volume protobuf. + // TODO(idownes): This test is unnecessarily strict and could be + // relaxed if mounts could be re-ordered. + foreach (const string& containerPath, containerPaths) { + if (strings::startsWith(volume.container_path(), containerPath)) { + return Failure("Cannot mount volume to '" + + volume.container_path() + + "' because it is under volume '" + + containerPath + + "'"); + } + + if (strings::startsWith(containerPath, volume.container_path())) { + return Failure("Cannot mount volume to '" + + containerPath + + "' because it is under volume '" + + volume.container_path() + + "'"); + } + } + containerPaths.insert(volume.container_path()); + + // A relative host path will be created in the container's work + // directory, otherwise check it already exists. + string hostPath; + if (!strings::startsWith(volume.host_path(), "/")) { + hostPath = path::join(directory, volume.host_path()); + + // Do not support any relative components in the resulting path. + // There should not be any links in the work directory to + // resolve. + if (strings::contains(hostPath, "/./") || + strings::contains(hostPath, "/../")) { + return Failure("Relative host path '" + + hostPath + + "' cannot contain relative components"); + } + + Try<Nothing> mkdir = os::mkdir(hostPath, true); + if (mkdir.isError()) { + return Failure("Failed to create host_path '" + + hostPath + + "' for mount to '" + + volume.container_path() + + "': " + + mkdir.error()); + } + + // Set the ownership and permissions to match the container path + // as these are inherited from host path on bind mount. + struct stat stat; + if (::stat(volume.container_path().c_str(), &stat) < 0) { + return Failure("Failed to get permissions on '" + + volume.container_path() + "'" + + ": " + strerror(errno)); + } + + Try<Nothing> chmod = os::chmod(hostPath, stat.st_mode); + if (chmod.isError()) { + return Failure("Failed to chmod hostPath '" + + hostPath + + "': " + + chmod.error()); + } + + Try<Nothing> chown = os::chown(stat.st_uid, stat.st_gid, hostPath, false); + if (chown.isError()) { + return Failure("Failed to chown hostPath '" + + hostPath + + "': " + + chown.error()); + } + } else { + hostPath = volume.host_path(); + + if (!os::exists(hostPath)) { + return Failure("Volume with container path '" + + volume.container_path() + + "' must have host path '" + + hostPath + + "' present on host for shared filesystem isolator"); + } + } + + commands.push_back("mount -n --bind " + + hostPath + + " " + + volume.container_path()); + } + + CommandInfo command; + command.set_value(strings::join(" && ", commands)); + + return command; +} + + +Future<Nothing> SharedFilesystemIsolatorProcess::isolate( + const ContainerID& containerId, + pid_t pid) +{ + // No-op, isolation happens when unsharing the mount namespace. + + return Nothing(); +} + + +Future<Limitation> SharedFilesystemIsolatorProcess::watch( + const ContainerID& containerId) +{ + // No-op, for now. + + return Future<Limitation>(); +} + + +Future<Nothing> SharedFilesystemIsolatorProcess::update( + const ContainerID& containerId, + const Resources& resources) +{ + // No-op, nothing enforced. + + return Nothing(); +} + + +Future<ResourceStatistics> SharedFilesystemIsolatorProcess::usage( + const ContainerID& containerId) +{ + // No-op, no usage gathered. + + return ResourceStatistics(); +} + + +Future<Nothing> SharedFilesystemIsolatorProcess::cleanup( + const ContainerID& containerId) +{ + // Cleanup of mounts is done automatically done by the kernel when + // the mount namespace is destroyed after the last process + // terminates. + + return Nothing(); +} + +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/slave/containerizer/isolators/filesystem/shared.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/isolators/filesystem/shared.hpp b/src/slave/containerizer/isolators/filesystem/shared.hpp new file mode 100644 index 0000000..75172d5 --- /dev/null +++ b/src/slave/containerizer/isolators/filesystem/shared.hpp @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SHARED_FILESYSTEM_ISOLATOR_HPP__ +#define __SHARED_FILESYSTEM_ISOLATOR_HPP__ + +#include "slave/containerizer/isolator.hpp" + +namespace mesos { +namespace internal { +namespace slave { + +// This isolator is to be used when all containers share the host's +// filesystem. It supports creating mounting "volumes" from the host +// into each container's mount namespace. In particular, this can be +// used to give each container a "private" system directory, such as +// /tmp and /var/tmp. +class SharedFilesystemIsolatorProcess : public IsolatorProcess +{ +public: + static Try<Isolator*> create(const Flags& flags); + + virtual ~SharedFilesystemIsolatorProcess(); + + virtual process::Future<Nothing> recover( + const std::list<state::RunState>& states); + + virtual process::Future<Option<CommandInfo> > prepare( + const ContainerID& containerId, + const ExecutorInfo& executorInfo, + const std::string& directory); + + virtual process::Future<Nothing> isolate( + const ContainerID& containerId, + pid_t pid); + + virtual process::Future<Limitation> watch( + const ContainerID& containerId); + + virtual process::Future<Nothing> update( + const ContainerID& containerId, + const Resources& resources); + + virtual process::Future<ResourceStatistics> usage( + const ContainerID& containerId); + + virtual process::Future<Nothing> cleanup( + const ContainerID& containerId); + +private: + SharedFilesystemIsolatorProcess(const Flags& flags); + + const Flags flags; +}; + +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __SHARED_FILESYSTEM_ISOLATOR_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/slave/containerizer/linux_launcher.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/linux_launcher.cpp b/src/slave/containerizer/linux_launcher.cpp index 07ee643..7a5cdbb 100644 --- a/src/slave/containerizer/linux_launcher.cpp +++ b/src/slave/containerizer/linux_launcher.cpp @@ -91,8 +91,6 @@ Try<Launcher*> LinuxLauncher::create(const Flags& flags) LOG(INFO) << "Using " << hierarchy.get() << " as the freezer hierarchy for the Linux launcher"; - // TODO(idownes): Inspect the isolation flag to determine namespaces - // to use. int namespaces = 0; #ifdef WITH_NETWORK_ISOLATOR @@ -103,6 +101,10 @@ Try<Launcher*> LinuxLauncher::create(const Flags& flags) } #endif + if (strings::contains(flags.isolation, "filesystem/shared")) { + namespaces |= CLONE_NEWNS; + } + return new LinuxLauncher(flags, namespaces, hierarchy.get()); } @@ -347,6 +349,10 @@ Try<pid_t> LinuxLauncher::fork( Future<Nothing> LinuxLauncher::destroy(const ContainerID& containerId) { + if (!pids.contains(containerId)) { + return Failure("Unknown container"); + } + pids.erase(containerId); return cgroups::destroy( http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/slave/containerizer/mesos/containerizer.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp index ce92878..3fa249f 100644 --- a/src/slave/containerizer/mesos/containerizer.cpp +++ b/src/slave/containerizer/mesos/containerizer.cpp @@ -42,6 +42,7 @@ #include "slave/containerizer/isolators/cgroups/cpushare.hpp" #include "slave/containerizer/isolators/cgroups/mem.hpp" #include "slave/containerizer/isolators/cgroups/perf_event.hpp" +#include "slave/containerizer/isolators/filesystem/shared.hpp" #endif // __linux__ #ifdef WITH_NETWORK_ISOLATOR #include "slave/containerizer/isolators/network/port_mapping.hpp" @@ -102,6 +103,7 @@ Try<MesosContainerizer*> MesosContainerizer::create( creators["cgroups/cpu"] = &CgroupsCpushareIsolatorProcess::create; creators["cgroups/mem"] = &CgroupsMemIsolatorProcess::create; creators["cgroups/perf_event"] = &CgroupsPerfEventIsolatorProcess::create; + creators["filesystem/shared"] = &SharedFilesystemIsolatorProcess::create; #endif // __linux__ #ifdef WITH_NETWORK_ISOLATOR creators["network/port_mapping"] = &PortMappingIsolatorProcess::create; @@ -124,7 +126,13 @@ Try<MesosContainerizer*> MesosContainerizer::create( return Error( "Could not create isolator " + type + ": " + isolator.error()); } else { - isolators.push_back(Owned<Isolator>(isolator.get())); + if (type == "filesystem/shared") { + // Filesystem isolator must be the first isolator used for prepare() + // so any volume mounts are performed before anything else runs. + isolators.insert(isolators.begin(), Owned<Isolator>(isolator.get())); + } else { + isolators.push_back(Owned<Isolator>(isolator.get())); + } } } else { return Error("Unknown or unsupported isolator: " + type); @@ -135,7 +143,8 @@ Try<MesosContainerizer*> MesosContainerizer::create( // Determine which launcher to use based on the isolation flag. Try<Launcher*> launcher = (strings::contains(isolation, "cgroups") || - strings::contains(isolation, "network/port_mapping")) + strings::contains(isolation, "network/port_mapping") || + strings::contains(isolation, "filesystem/shared")) ? LinuxLauncher::create(flags) : PosixLauncher::create(flags); #else @@ -384,9 +393,10 @@ Future<bool> MesosContainerizerProcess::launch( return Failure("Container already started"); } - if (executorInfo.has_container()) { - // We return false as this containerizer does not support - // handling ContainerInfo. + // We support MESOS containers or ExecutorInfos with no + // ContainerInfo given. + if (executorInfo.has_container() && + executorInfo.container().type() != ContainerInfo::MESOS) { return false; } @@ -437,7 +447,7 @@ Future<bool> MesosContainerizerProcess::launch( { if (taskInfo.has_container()) { // We return false as this containerizer does not support - // handling ContainerInfo. + // handling TaskInfo::ContainerInfo. return false; } @@ -950,7 +960,7 @@ void MesosContainerizerProcess::_destroy( // consider cleaning up here. if (!future.isReady()) { promises[containerId]->fail( - "Failed to destroy container: " + + "Failed to destroy container " + stringify(containerId) + ": " + (future.isFailed() ? future.failure() : "discarded future")); destroying.erase(containerId); http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index bdaa5d5..621d0de 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -303,6 +303,27 @@ public: "sandbox is mapped to.\n", "/mnt/mesos/sandbox"); + add(&Flags::default_container_info, + "default_container_info", + "JSON formatted ContainerInfo that will be included into\n" + "any ExecutorInfo that does not specify a ContainerInfo.\n" + "\n" + "See the ContainerInfo protobuf in mesos.proto for\n" + "the expected format.\n" + "\n" + "Example:\n" + "{\n" + "\"type\": \"MESOS\",\n" + "\"volumes\": [\n" + " {\n" + " \"host_path\": \"./.private/tmp\",\n" + " \"container_path\": \"/tmp\",\n" + " \"mode\": \"RW\"\n" + " }\n" + " ]\n" + "}" + ); + #ifdef WITH_NETWORK_ISOLATOR add(&Flags::ephemeral_ports_per_container, "ephemeral_ports_per_container", @@ -399,6 +420,7 @@ public: Option<std::string> default_container_image; std::string docker; std::string docker_sandbox_directory; + Option<ContainerInfo> default_container_info; #ifdef WITH_NETWORK_ISOLATOR uint16_t ephemeral_ports_per_container; Option<std::string> eth0_name; http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/slave/slave.cpp ---------------------------------------------------------------------- diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp index 5e7c107..96fb5f7 100644 --- a/src/slave/slave.cpp +++ b/src/slave/slave.cpp @@ -2701,10 +2701,24 @@ ExecutorInfo Slave::getExecutorInfo( "cpus:" + stringify(DEFAULT_EXECUTOR_CPUS) + ";" + "mem:" + stringify(DEFAULT_EXECUTOR_MEM.megabytes())).get()); + // Add in any default ContainerInfo. + if (!executor.has_container() && flags.default_container_info.isSome()) { + executor.mutable_container()->CopyFrom( + flags.default_container_info.get()); + } + return executor; } - return task.executor(); + ExecutorInfo executor = task.executor(); + + // Add in any default ContainerInfo. + if (!executor.has_container() && flags.default_container_info.isSome()) { + executor.mutable_container()->CopyFrom( + flags.default_container_info.get()); + } + + return executor; } http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/tests/isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/isolator_tests.cpp b/src/tests/isolator_tests.cpp index db7a58a..a0653e2 100644 --- a/src/tests/isolator_tests.cpp +++ b/src/tests/isolator_tests.cpp @@ -48,6 +48,7 @@ #include "slave/containerizer/isolators/cgroups/cpushare.hpp" #include "slave/containerizer/isolators/cgroups/mem.hpp" #include "slave/containerizer/isolators/cgroups/perf_event.hpp" +#include "slave/containerizer/isolators/filesystem/shared.hpp" #endif // __linux__ #include "slave/containerizer/isolators/posix.hpp" @@ -75,6 +76,7 @@ using mesos::internal::master::Master; using mesos::internal::slave::CgroupsCpushareIsolatorProcess; using mesos::internal::slave::CgroupsMemIsolatorProcess; using mesos::internal::slave::CgroupsPerfEventIsolatorProcess; +using mesos::internal::slave::SharedFilesystemIsolatorProcess; #endif // __linux__ using mesos::internal::slave::Isolator; using mesos::internal::slave::IsolatorProcess; @@ -752,4 +754,178 @@ TEST_F(PerfEventIsolatorTest, ROOT_CGROUPS_Sample) delete isolator.get(); } +class SharedFilesystemIsolatorTest : public MesosTest {}; + + +// Test that a container can create a private view of a system +// directory (/var/tmp). Check that a file written by a process inside +// the container doesn't appear on the host filesystem but does appear +// under the container's work directory. +TEST_F(SharedFilesystemIsolatorTest, ROOT_RelativeVolume) +{ + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "filesystem/shared"; + + Try<Isolator*> isolator = SharedFilesystemIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = LinuxLauncher::create(flags); + CHECK_SOME(launcher); + + // Use /var/tmp so we don't mask the work directory (under /tmp). + const string containerPath = "/var/tmp"; + ASSERT_TRUE(os::isdir(containerPath)); + + // Use a host path relative to the container work directory. + const string hostPath = strings::remove(containerPath, "/", strings::PREFIX); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::MESOS); + containerInfo.add_volumes()->CopyFrom( + CREATE_VOLUME(containerPath, hostPath, Volume::RW)); + + ExecutorInfo executorInfo; + executorInfo.mutable_container()->CopyFrom(containerInfo); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + Future<Option<CommandInfo> > prepare = + isolator.get()->prepare(containerId, executorInfo, flags.work_dir); + AWAIT_READY(prepare); + ASSERT_SOME(prepare.get()); + + // The test will touch a file in container path. + const string file = path::join(containerPath, UUID::random().toString()); + ASSERT_FALSE(os::exists(file)); + + // Manually run the isolator's preparation command first, then touch + // the file. + vector<string> args; + args.push_back("/bin/sh"); + args.push_back("-x"); + args.push_back("-c"); + args.push_back(prepare.get().get().value() + " && touch " + file); + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + args, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + None()); + ASSERT_SOME(pid); + + // Set up the reaper to wait on the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + AWAIT_READY(status); + EXPECT_SOME_EQ(0, status.get()); + + // Check the correct hierarchy was created under the container work + // directory. + string dir = "/"; + foreach (const string& subdir, strings::tokenize(containerPath, "/")) { + dir = path::join(dir, subdir); + + struct stat hostStat; + EXPECT_EQ(0, ::stat(dir.c_str(), &hostStat)); + + struct stat containerStat; + EXPECT_EQ(0, + ::stat(path::join(flags.work_dir, dir).c_str(), &containerStat)); + + EXPECT_EQ(hostStat.st_mode, containerStat.st_mode); + EXPECT_EQ(hostStat.st_uid, containerStat.st_uid); + EXPECT_EQ(hostStat.st_gid, containerStat.st_gid); + } + + // Check it did *not* create a file in the host namespace. + EXPECT_FALSE(os::exists(file)); + + // Check it did create the file under the container's work directory + // on the host. + EXPECT_TRUE(os::exists(path::join(flags.work_dir, file))); + + delete launcher.get(); + delete isolator.get(); +} + + +TEST_F(SharedFilesystemIsolatorTest, ROOT_AbsoluteVolume) +{ + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "filesystem/shared"; + + Try<Isolator*> isolator = SharedFilesystemIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = LinuxLauncher::create(flags); + CHECK_SOME(launcher); + + // We'll mount the absolute test work directory as /var/tmp in the + // container. + const string hostPath = flags.work_dir; + const string containerPath = "/var/tmp"; + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::MESOS); + containerInfo.add_volumes()->CopyFrom( + CREATE_VOLUME(containerPath, hostPath, Volume::RW)); + + ExecutorInfo executorInfo; + executorInfo.mutable_container()->CopyFrom(containerInfo); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + Future<Option<CommandInfo> > prepare = + isolator.get()->prepare(containerId, executorInfo, flags.work_dir); + AWAIT_READY(prepare); + ASSERT_SOME(prepare.get()); + + // Test the volume mounting by touching a file in the container's + // /tmp, which should then be in flags.work_dir. + const string filename = UUID::random().toString(); + ASSERT_FALSE(os::exists(path::join(containerPath, filename))); + + vector<string> args; + args.push_back("/bin/sh"); + args.push_back("-x"); + args.push_back("-c"); + args.push_back(prepare.get().get().value() + + " && touch " + + path::join(containerPath, filename)); + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + args, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + None()); + ASSERT_SOME(pid); + + // Set up the reaper to wait on the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + AWAIT_READY(status); + EXPECT_SOME_EQ(0, status.get()); + + // Check the file was created in flags.work_dir. + EXPECT_TRUE(os::exists(path::join(hostPath, filename))); + + // Check it didn't get created in the host's view of containerPath. + EXPECT_FALSE(os::exists(path::join(containerPath, filename))); + + delete launcher.get(); + delete isolator.get(); +} + #endif // __linux__ http://git-wip-us.apache.org/repos/asf/mesos/blob/f511395e/src/tests/mesos.hpp ---------------------------------------------------------------------- diff --git a/src/tests/mesos.hpp b/src/tests/mesos.hpp index e36e138..c1d64a7 100644 --- a/src/tests/mesos.hpp +++ b/src/tests/mesos.hpp @@ -311,6 +311,14 @@ protected: commandInfo; }) +#define CREATE_VOLUME(containerPath, hostPath, mode) \ + ({ Volume volume; \ + volume.set_container_path(containerPath); \ + volume.set_host_path(hostPath); \ + volume.set_mode(mode); \ + volume; }) + + // TODO(bmahler): Refactor this to make the distinction between // command tasks and executor tasks clearer. inline TaskInfo createTask(
