Repository: mesos Updated Branches: refs/heads/master a16be70ef -> 15bf9f67e
Support chrooting in MesosContainerizer launch helper. Review: https://reviews.apache.org/r/31444/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/15bf9f67 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/15bf9f67 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/15bf9f67 Branch: refs/heads/master Commit: 15bf9f67e3d9489e49b2176e1e4221a1a47fd0c9 Parents: 6950310 Author: Ian Downes <[email protected]> Authored: Thu Dec 18 15:46:15 2014 -0800 Committer: Ian Downes <[email protected]> Committed: Tue Jul 7 15:39:52 2015 -0700 ---------------------------------------------------------------------- src/Makefile.am | 1 + src/linux/fs.cpp | 5 +- src/slave/containerizer/mesos/launch.cpp | 60 ++++++- src/slave/containerizer/mesos/launch.hpp | 1 + src/tests/launch_tests.cpp | 238 ++++++++++++++++++++++++++ 5 files changed, 296 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/15bf9f67/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index 37aa878..e5b5d36 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1542,6 +1542,7 @@ if OS_LINUX mesos_tests_SOURCES += tests/cgroups_isolator_tests.cpp mesos_tests_SOURCES += tests/cgroups_tests.cpp mesos_tests_SOURCES += tests/fs_tests.cpp + mesos_tests_SOURCES += tests/launch_tests.cpp mesos_tests_SOURCES += tests/memory_pressure_tests.cpp mesos_tests_SOURCES += tests/ns_tests.cpp mesos_tests_SOURCES += tests/perf_tests.cpp http://git-wip-us.apache.org/repos/asf/mesos/blob/15bf9f67/src/linux/fs.cpp ---------------------------------------------------------------------- diff --git a/src/linux/fs.cpp b/src/linux/fs.cpp index 6d5af25..ea0891e 100644 --- a/src/linux/fs.cpp +++ b/src/linux/fs.cpp @@ -83,7 +83,7 @@ Try<MountInfoTable::Entry> MountInfoTable::Entry::parse(const string& s) // First group of fields (before the separator): 6 required fields // then zero or more optional fields - std::vector<string> tokens = strings::tokenize(s.substr(0, pos), " "); + vector<string> tokens = strings::tokenize(s.substr(0, pos), " "); if (tokens.size() < 6) { return Error("Failed to parse entry"); } @@ -101,7 +101,7 @@ Try<MountInfoTable::Entry> MountInfoTable::Entry::parse(const string& s) entry.parent = parent.get(); // Parse out the major:minor device number. - std::vector<string> device = strings::split(tokens[2], ":"); + vector<string> device = strings::split(tokens[2], ":"); if (device.size() != 2) { return Error("Invalid major:minor device number"); } @@ -502,6 +502,7 @@ Try<Nothing> createStandardDevices(const string& root) } // namespace internal { + // TODO(idownes): Add unit test. Try<Nothing> enter(const string& root) { http://git-wip-us.apache.org/repos/asf/mesos/blob/15bf9f67/src/slave/containerizer/mesos/launch.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/launch.cpp b/src/slave/containerizer/mesos/launch.cpp index 51bc068..96ca47d 100644 --- a/src/slave/containerizer/mesos/launch.cpp +++ b/src/slave/containerizer/mesos/launch.cpp @@ -26,11 +26,16 @@ #include <stout/protobuf.hpp> #include <stout/unreachable.hpp> +#ifdef __linux__ +#include "linux/fs.hpp" +#endif + #include "mesos/mesos.hpp" #include "slave/containerizer/mesos/launch.hpp" using std::cerr; +using std::cout; using std::endl; using std::string; @@ -49,7 +54,15 @@ MesosContainerizerLaunch::Flags::Flags() add(&directory, "directory", - "The directory to chdir to."); + "The directory to chdir to. If rootfs is specified this must\n" + "be relative to the new root."); + + add(&rootfs, + "rootfs", + "Absolute path to the container root filesystem.\n" + "The command and directory flags are interpreted relative\n" + "to rootfs\n" + "Different platforms may implement 'chroot' differently."); add(&user, "user", @@ -193,17 +206,41 @@ int MesosContainerizerLaunch::execute() } } - // Enter working directory. - Try<Nothing> chdir = os::chdir(flags.directory.get()); - if (chdir.isError()) { - cerr << "Failed to chdir into work directory '" - << flags.directory.get() << "': " << chdir.error() << endl; - return 1; + // Change root to a new root, if provided. + if (flags.rootfs.isSome()) { + cout << "Changing root to " << flags.rootfs.get() << endl; + + // Verify that rootfs is an absolute path. + Result<string> realpath = os::realpath(flags.rootfs.get()); + if (realpath.isError()) { + cerr << "Failed to determine if rootfs is an absolute path: " + << realpath.error() << endl; + return 1; + } else if (realpath.isNone()) { + cerr << "Rootfs path does not exist" << endl; + return 1; + } else if (realpath.get() != flags.rootfs.get()) { + cerr << "Rootfs path is not an absolute path" << endl; + return 1; + } + +#ifdef __linux__ + Try<Nothing> chroot = fs::chroot::enter(flags.rootfs.get()); +#else // For any other platform we'll just use POSIX chroot. + Try<Nothing> chroot = os::chroot(flags.rootfs.get()); +#endif // __linux__ + if (chroot.isError()) { + cerr << "Failed to enter chroot '" << flags.rootfs.get() + << "': " << chroot.error(); + return 1; + } } // Change user if provided. Note that we do that after executing the // preparation commands so that those commands will be run with the // same privilege as the mesos-slave. + // NOTE: The requisite user/group information must be present if + // a container root filesystem is used. if (flags.user.isSome()) { Try<Nothing> su = os::su(flags.user.get()); if (su.isError()) { @@ -213,6 +250,15 @@ int MesosContainerizerLaunch::execute() } } + // Enter working directory, relative to the new root. + Try<Nothing> chdir = os::chdir(flags.directory.get()); + if (chdir.isError()) { + cerr << "Failed to chdir into work directory '" + << flags.directory.get() << "': " << chdir.error() << endl; + return 1; + } + + // Relay the environment variables. // TODO(jieyu): Consider using a clean environment. if (command.get().shell()) { http://git-wip-us.apache.org/repos/asf/mesos/blob/15bf9f67/src/slave/containerizer/mesos/launch.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/launch.hpp b/src/slave/containerizer/mesos/launch.hpp index 7c8b535..7a7f3d7 100644 --- a/src/slave/containerizer/mesos/launch.hpp +++ b/src/slave/containerizer/mesos/launch.hpp @@ -38,6 +38,7 @@ public: Option<JSON::Object> command; Option<std::string> directory; + Option<std::string> rootfs; Option<std::string> user; Option<int> pipe_read; Option<int> pipe_write; http://git-wip-us.apache.org/repos/asf/mesos/blob/15bf9f67/src/tests/launch_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/launch_tests.cpp b/src/tests/launch_tests.cpp new file mode 100644 index 0000000..73c8c5f --- /dev/null +++ b/src/tests/launch_tests.cpp @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> + +#include <gmock/gmock.h> + +#include <stout/foreach.hpp> +#include <stout/gtest.hpp> +#include <stout/os.hpp> +#include <stout/try.hpp> + +#include <process/gtest.hpp> +#include <process/io.hpp> +#include <process/reap.hpp> +#include <process/subprocess.hpp> + +#include "mesos/resources.hpp" + +#include "slave/containerizer/mesos/launch.hpp" + +#include "linux/fs.hpp" + +#include "tests/flags.hpp" +#include "tests/utils.hpp" + +using namespace process; + +using std::string; +using std::vector; + +namespace mesos { +namespace internal { +namespace tests { + +class Chroot +{ +public: + Chroot(const string& _rootfs) + : rootfs(_rootfs) {} + + virtual ~Chroot() {} + + virtual Try<Subprocess> run(const string& command) = 0; + + const string rootfs; +}; + + +class BasicLinuxChroot : public Chroot +{ +public: + static Try<Owned<Chroot>> create(const string& rootfs) + { + if (!os::exists(rootfs)) { + return Error("rootfs does not exist"); + } + + if (os::system("cp -r /bin " + rootfs + "/") != 0) { + return ErrnoError("Failed to copy /bin to chroot"); + } + + if (os::system("cp -r /lib " + rootfs + "/") != 0) { + return ErrnoError("Failed to copy /lib to chroot"); + } + + if (os::system("cp -r /lib64 " + rootfs + "/") != 0) { + return ErrnoError("Failed to copy /lib64 to chroot"); + } + + vector<string> directories = {"proc", "sys", "dev", "tmp"}; + foreach (const string& directory, directories) { + Try<Nothing> mkdir = os::mkdir(path::join(rootfs, directory)); + if (mkdir.isError()) { + return Error("Failed to create /" + directory + " in chroot: " + + mkdir.error()); + } + } + + // We need to bind mount the rootfs so we can pivot on it. + Try<Nothing> mount = + fs::mount(rootfs, rootfs, None(), MS_BIND | MS_SLAVE, NULL); + + if (mount.isError()) { + return Error("Failed to bind mount chroot rootfs: " + mount.error()); + } + + return Owned<Chroot>(new BasicLinuxChroot(rootfs)); + } + + virtual Try<Subprocess> run(const string& _command) + { + slave::MesosContainerizerLaunch::Flags launchFlags; + + CommandInfo command; + command.set_value(_command); + + launchFlags.command = JSON::Protobuf(command); + launchFlags.directory = "/tmp"; + launchFlags.pipe_read = open("/dev/zero", O_RDONLY); + launchFlags.pipe_write = open("/dev/null", O_WRONLY); + launchFlags.rootfs = rootfs; + + vector<string> argv(2); + argv[0] = "mesos-containerizer"; + argv[1] = slave::MesosContainerizerLaunch::NAME; + + Try<Subprocess> s = subprocess( + path::join(tests::flags.build_dir, "src", "mesos-containerizer"), + argv, + Subprocess::PATH("/dev/null"), + Subprocess::PIPE(), + Subprocess::FD(STDERR_FILENO), + launchFlags, + None(), + None(), + lambda::bind(&clone, lambda::_1)); + + if (s.isError()) { + close(launchFlags.pipe_read.get()); + close(launchFlags.pipe_write.get()); + } else { + s.get().status().onAny([=]() { + // Close when the subprocess terminates. + close(launchFlags.pipe_read.get()); + close(launchFlags.pipe_write.get()); + }); + } + + return s; + } + +private: + static pid_t clone(const lambda::function<int()>& f) + { + static unsigned long long stack[(8*1024*1024)/sizeof(unsigned long long)]; + + return ::clone( + _clone, + &stack[sizeof(stack)/sizeof(stack[0]) - 1], // Stack grows down. + CLONE_NEWNS | SIGCHLD, // Specify SIGCHLD as child termination signal. + (void*) &f); + } + + static int _clone(void* f) + { + const lambda::function<int()>* _f = + static_cast<const lambda::function<int()>*> (f); + + return (*_f)(); + } + + BasicLinuxChroot(const string& rootfs) : Chroot(rootfs) {} + + ~BasicLinuxChroot() + { + // Because the test process has the rootfs as its cwd the umount + // won't actually happen until the + // TemporaryDirectoryTest::TearDown() changes back to the original + // directory. + fs::unmount(rootfs, MNT_DETACH); + } +}; + + +template <typename T> +class LaunchChrootTest : public TemporaryDirectoryTest {}; + + +// TODO(idownes): Add tests for OSX chroots. +typedef ::testing::Types<BasicLinuxChroot> ChrootTypes; + + +TYPED_TEST_CASE(LaunchChrootTest, ChrootTypes); + + +TYPED_TEST(LaunchChrootTest, ROOT_DifferentRoot) +{ + Try<Owned<Chroot>> chroot = TypeParam::create(os::getcwd()); + ASSERT_SOME(chroot); + + // Add /usr/bin/stat into the chroot. + const string usrbin = path::join(chroot.get()->rootfs, "usr", "bin"); + ASSERT_SOME(os::mkdir(usrbin)); + ASSERT_EQ(0, os::system("cp /usr/bin/stat " + path::join(usrbin, "stat"))); + + Clock::pause(); + + Try<Subprocess> s = chroot.get()->run( + "/usr/bin/stat -c %i / >" + path::join("/", "stat.output")); + + CHECK_SOME(s); + + // Advance time until the internal reaper reaps the subprocess. + while (s.get().status().isPending()) { + Clock::advance(Seconds(1)); + Clock::settle(); + } + + AWAIT_ASSERT_READY(s.get().status()); + ASSERT_SOME(s.get().status().get()); + + int status = s.get().status().get().get(); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); + + // Check the chroot has a different root by comparing the inodes. + Try<ino_t> self = os::stat::inode("/"); + ASSERT_SOME(self); + + Try<string> read = os::read(path::join(chroot.get()->rootfs, "stat.output")); + CHECK_SOME(read); + + Try<ino_t> other = numify<ino_t>(strings::trim(read.get())); + ASSERT_SOME(other); + + EXPECT_NE(self.get(), other.get()); +} + +} // namespace tests { +} // namespace internal { +} // namespace mesos {
