http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/docker_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/docker_tests.cpp b/src/tests/containerizer/docker_tests.cpp new file mode 100644 index 0000000..a4a2725 --- /dev/null +++ b/src/tests/containerizer/docker_tests.cpp @@ -0,0 +1,421 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include <process/future.hpp> +#include <process/gtest.hpp> +#include <process/owned.hpp> +#include <process/subprocess.hpp> + +#include <stout/duration.hpp> +#include <stout/option.hpp> +#include <stout/gtest.hpp> + +#include "docker/docker.hpp" + +#include "mesos/resources.hpp" + +#include "tests/environment.hpp" +#include "tests/flags.hpp" +#include "tests/mesos.hpp" + +using namespace process; + +using std::list; +using std::string; + +namespace mesos { +namespace internal { +namespace tests { + + +static const string NAME_PREFIX="mesos-docker"; + + +class DockerTest : public MesosTest +{ + virtual void TearDown() + { + Try<Docker*> docker = Docker::create(tests::flags.docker, false); + ASSERT_SOME(docker); + + Future<list<Docker::Container>> containers = + docker.get()->ps(true, NAME_PREFIX); + + AWAIT_READY(containers); + + // Cleanup all mesos launched containers. + foreach (const Docker::Container& container, containers.get()) { + AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30)); + } + + delete docker.get(); + } +}; + +// This test tests the functionality of the docker's interfaces. +TEST_F(DockerTest, ROOT_DOCKER_interface) +{ + const string containerName = NAME_PREFIX + "-test"; + Resources resources = Resources::parse("cpus:1;mem:512").get(); + + Owned<Docker> docker(Docker::create(tests::flags.docker, false).get()); + + // Verify that we do not see the container. + Future<list<Docker::Container> > containers = docker->ps(true, containerName); + AWAIT_READY(containers); + foreach (const Docker::Container& container, containers.get()) { + EXPECT_NE("/" + containerName, container.name); + } + + Try<string> directory = environment->mkdtemp(); + CHECK_SOME(directory) << "Failed to create temporary directory"; + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + CommandInfo commandInfo; + commandInfo.set_value("sleep 120"); + + // Start the container. + Future<Nothing> status = docker->run( + containerInfo, + commandInfo, + containerName, + directory.get(), + "/mnt/mesos/sandbox", + resources); + + Future<Docker::Container> inspect = + docker->inspect(containerName, Seconds(1)); + AWAIT_READY(inspect); + + // Should be able to see the container now. + containers = docker->ps(); + AWAIT_READY(containers); + bool found = false; + foreach (const Docker::Container& container, containers.get()) { + if ("/" + containerName == container.name) { + found = true; + break; + } + } + EXPECT_TRUE(found); + + // Test some fields of the container. + EXPECT_NE("", inspect.get().id); + EXPECT_EQ("/" + containerName, inspect.get().name); + EXPECT_SOME(inspect.get().pid); + + // Stop the container. + status = docker->stop(containerName); + AWAIT_READY(status); + + // Now, the container should not appear in the result of ps(). + // But it should appear in the result of ps(true). + containers = docker->ps(); + AWAIT_READY(containers); + foreach (const Docker::Container& container, containers.get()) { + EXPECT_NE("/" + containerName, container.name); + } + + containers = docker->ps(true, containerName); + AWAIT_READY(containers); + found = false; + foreach (const Docker::Container& container, containers.get()) { + if ("/" + containerName == container.name) { + found = true; + break; + } + } + EXPECT_TRUE(found); + + // Check the container's info, both id and name should remain the + // same since we haven't removed it, but the pid should be none + // since it's not running. + inspect = docker->inspect(containerName); + AWAIT_READY(inspect); + + EXPECT_NE("", inspect.get().id); + EXPECT_EQ("/" + containerName, inspect.get().name); + EXPECT_NONE(inspect.get().pid); + + // Remove the container. + status = docker->rm(containerName); + AWAIT_READY(status); + + // Should not be able to inspect the container. + inspect = docker->inspect(containerName); + AWAIT_FAILED(inspect); + + // Also, now we should not be able to see the container by invoking + // ps(true). + containers = docker->ps(true, containerName); + AWAIT_READY(containers); + foreach (const Docker::Container& container, containers.get()) { + EXPECT_NE("/" + containerName, container.name); + } + + // Start the container again, this time we will do a "rm -f" + // directly, instead of stopping and rm. + status = docker->run( + containerInfo, + commandInfo, + containerName, + directory.get(), + "/mnt/mesos/sandbox", + resources); + + inspect = docker->inspect(containerName, Seconds(1)); + AWAIT_READY(inspect); + + // Verify that the container is there. + containers = docker->ps(); + AWAIT_READY(containers); + found = false; + foreach (const Docker::Container& container, containers.get()) { + if ("/" + containerName == container.name) { + found = true; + break; + } + } + EXPECT_TRUE(found); + + // Then do a "rm -f". + status = docker->rm(containerName, true); + AWAIT_READY(status); + + // Verify that the container is totally removed, that is we can't + // find it by ps() or ps(true). + containers = docker->ps(); + AWAIT_READY(containers); + foreach (const Docker::Container& container, containers.get()) { + EXPECT_NE("/" + containerName, container.name); + } + containers = docker->ps(true, containerName); + AWAIT_READY(containers); + foreach (const Docker::Container& container, containers.get()) { + EXPECT_NE("/" + containerName, container.name); + } +} + + +TEST_F(DockerTest, ROOT_DOCKER_CheckCommandWithShell) +{ + Owned<Docker> docker(Docker::create(tests::flags.docker, false).get()); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + CommandInfo commandInfo; + commandInfo.set_shell(true); + + Future<Nothing> run = docker->run( + containerInfo, + commandInfo, + "testContainer", + "dir", + "/mnt/mesos/sandbox"); + + ASSERT_TRUE(run.isFailed()); +} + + +TEST_F(DockerTest, ROOT_DOCKER_CheckPortResource) +{ + const string containerName = NAME_PREFIX + "-port-resource-test"; + Owned<Docker> docker(Docker::create(tests::flags.docker, false).get()); + + // Make sure the container is removed. + Future<Nothing> remove = docker->rm(containerName, true); + + ASSERT_TRUE(process::internal::await(remove, Seconds(10))); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE); + + ContainerInfo::DockerInfo::PortMapping portMapping; + portMapping.set_host_port(10000); + portMapping.set_container_port(80); + + dockerInfo.add_port_mappings()->CopyFrom(portMapping); + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + CommandInfo commandInfo; + commandInfo.set_shell(false); + commandInfo.set_value("true"); + + Resources resources = + Resources::parse("ports:[9998-9999];ports:[10001-11000]").get(); + + Future<Nothing> run = docker->run( + containerInfo, + commandInfo, + containerName, + "dir", + "/mnt/mesos/sandbox", + resources); + + // Port should be out side of the provided ranges. + AWAIT_EXPECT_FAILED(run); + + resources = Resources::parse("ports:[9998-9999];ports:[10000-11000]").get(); + + Try<string> directory = environment->mkdtemp(); + CHECK_SOME(directory) << "Failed to create temporary directory"; + + run = docker->run( + containerInfo, + commandInfo, + containerName, + directory.get(), + "/mnt/mesos/sandbox", + resources); + + AWAIT_READY(run); +} + + +TEST_F(DockerTest, ROOT_DOCKER_CancelPull) +{ + // Delete the test image if it exists. + + Try<Subprocess> s = process::subprocess( + tests::flags.docker + " rmi lingmann/1gb", + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null")); + + ASSERT_SOME(s); + + AWAIT_READY_FOR(s.get().status(), Seconds(30)); + + Owned<Docker> docker(Docker::create(tests::flags.docker, false).get()); + + Try<string> directory = environment->mkdtemp(); + + CHECK_SOME(directory) << "Failed to create temporary directory"; + + // Assume that pulling the very large image 'lingmann/1gb' will take + // sufficiently long that we can start it and discard (i.e., cancel + // it) right away and the future will indeed get discarded. + Future<Docker::Image> future = + docker->pull(directory.get(), "lingmann/1gb"); + + future.discard(); + + AWAIT_DISCARDED(future); +} + + +// This test verifies mounting in a relative path when running a +// docker container works. +TEST_F(DockerTest, ROOT_DOCKER_MountRelative) +{ + Owned<Docker> docker(Docker::create(tests::flags.docker, false).get()); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + Volume* volume = containerInfo.add_volumes(); + volume->set_host_path("test_file"); + volume->set_container_path("/tmp/test_file"); + volume->set_mode(Volume::RO); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + CommandInfo commandInfo; + commandInfo.set_shell(true); + commandInfo.set_value("ls /tmp/test_file"); + + Try<string> directory = environment->mkdtemp(); + CHECK_SOME(directory) << "Failed to create temporary directory"; + + const string testFile = path::join(directory.get(), "test_file"); + EXPECT_SOME(os::write(testFile, "data")); + + Future<Nothing> run = docker->run( + containerInfo, + commandInfo, + NAME_PREFIX + "-mount-relative-test", + directory.get(), + directory.get()); + + AWAIT_READY(run); +} + + +// This test verifies mounting in a absolute path when running a +// docker container works. +TEST_F(DockerTest, ROOT_DOCKER_MountAbsolute) +{ + Owned<Docker> docker(Docker::create(tests::flags.docker, false).get()); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + Try<string> directory = environment->mkdtemp(); + CHECK_SOME(directory) << "Failed to create temporary directory"; + + const string testFile = path::join(directory.get(), "test_file"); + EXPECT_SOME(os::write(testFile, "data")); + + Volume* volume = containerInfo.add_volumes(); + volume->set_host_path(testFile); + volume->set_container_path("/tmp/test_file"); + volume->set_mode(Volume::RO); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + CommandInfo commandInfo; + commandInfo.set_shell(true); + commandInfo.set_value("ls /tmp/test_file"); + + Future<Nothing> run = docker->run( + containerInfo, + commandInfo, + NAME_PREFIX + "-mount-absolute-test", + directory.get(), + directory.get()); + + AWAIT_READY(run); +} + + +} // namespace tests { +} // namespace internal { +} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/external_containerizer_test.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/external_containerizer_test.cpp b/src/tests/containerizer/external_containerizer_test.cpp new file mode 100644 index 0000000..4f152a4 --- /dev/null +++ b/src/tests/containerizer/external_containerizer_test.cpp @@ -0,0 +1,267 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <unistd.h> + +#include <gmock/gmock.h> + +#include <string> +#include <vector> +#include <map> + +#include <mesos/resources.hpp> + +#include <process/future.hpp> + +#include <stout/os.hpp> +#include <stout/path.hpp> + +#include "master/master.hpp" +#include "master/detector.hpp" + +#include "slave/flags.hpp" +#include "slave/slave.hpp" + +#include "slave/containerizer/containerizer.hpp" +#include "slave/containerizer/external_containerizer.hpp" + +#include "tests/mesos.hpp" +#include "tests/flags.hpp" + +using namespace process; + +using mesos::internal::master::Master; +using mesos::internal::slave::Containerizer; +using mesos::internal::slave::Slave; + +using std::string; +using std::vector; + +using testing::_; +using testing::DoAll; +using testing::Return; +using testing::SaveArg; +using testing::Invoke; + +namespace mesos { +namespace internal { +namespace tests { + +// The external containerizer tests currently rely on a Python script +// which needs the Mesos Python egg being built. +// TODO(tillt): Consider providing tests that do not rely on Python. +#ifdef MESOS_HAS_PYTHON + +// TODO(tillt): Update and enhance the ExternalContainerizer tests, +// possibly following some of the patterns used within the +// IsolatorTests or even entirely reusing the Containerizer tests. +class ExternalContainerizerTest : public MesosTest {}; + + +class MockExternalContainerizer : public slave::ExternalContainerizer +{ +public: + MOCK_METHOD8( + launch, + process::Future<bool>( + const ContainerID&, + const TaskInfo&, + const ExecutorInfo&, + const std::string&, + const Option<std::string>&, + const SlaveID&, + const process::PID<slave::Slave>&, + bool checkpoint)); + + MockExternalContainerizer(const slave::Flags& flags) + : ExternalContainerizer(flags) + { + // Set up defaults for mocked methods. + // NOTE: See TestContainerizer::setup for why we use + // 'EXPECT_CALL' and 'WillRepeatedly' here instead of + // 'ON_CALL' and 'WillByDefault'. + EXPECT_CALL(*this, launch(_, _, _, _, _, _, _, _)) + .WillRepeatedly(Invoke(this, &MockExternalContainerizer::_launch)); + } + + process::Future<bool> _launch( + const ContainerID& containerId, + const TaskInfo& taskInfo, + const ExecutorInfo& executorInfo, + const string& directory, + const Option<string>& user, + const SlaveID& slaveId, + const PID<Slave>& slavePid, + bool checkpoint) + { + return slave::ExternalContainerizer::launch( + containerId, + taskInfo, + executorInfo, + directory, + user, + slaveId, + slavePid, + checkpoint); + } +}; + + +// This test has been temporarily disabled due to MESOS-1257. +TEST_F(ExternalContainerizerTest, DISABLED_Launch) +{ + Try<PID<Master> > master = this->StartMaster(); + ASSERT_SOME(master); + + Flags testFlags; + + slave::Flags flags = this->CreateSlaveFlags(); + + flags.isolation = "external"; + flags.containerizer_path = + testFlags.build_dir + "/src/examples/python/test-containerizer"; + + MockExternalContainerizer containerizer(flags); + + Try<PID<Slave> > slave = this->StartSlave(&containerizer, flags); + ASSERT_SOME(slave); + + MockScheduler sched; + MesosSchedulerDriver driver( + &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); + + Future<FrameworkID> frameworkId; + EXPECT_CALL(sched, registered(&driver, _, _)) + .WillOnce(FutureArg<1>(&frameworkId)); + + Future<vector<Offer> > offers; + EXPECT_CALL(sched, resourceOffers(&driver, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(frameworkId); + AWAIT_READY(offers); + + EXPECT_NE(0u, offers.get().size()); + + TaskInfo task; + task.set_name("isolator_test"); + task.mutable_task_id()->set_value("1"); + task.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id()); + task.mutable_resources()->CopyFrom(offers.get()[0].resources()); + + Resources resources(offers.get()[0].resources()); + Option<Bytes> mem = resources.mem(); + ASSERT_SOME(mem); + Option<double> cpus = resources.cpus(); + ASSERT_SOME(cpus); + + const std::string& file = path::join(flags.work_dir, "ready"); + + // This task induces user/system load in a child process by + // running top in a child process for ten seconds. + task.mutable_command()->set_value( +#ifdef __APPLE__ + // Use logging mode with 30,000 samples with no interval. + "top -l 30000 -s 0 2>&1 > /dev/null & " +#else + // Batch mode, with 30,000 samples with no interval. + "top -b -d 0 -n 30000 2>&1 > /dev/null & " +#endif + "touch " + file + "; " // Signals that the top command is running. + "sleep 60"); + + Future<TaskStatus> status; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&status)) + .WillRepeatedly(Return()); // Ignore rest for now. + + Future<ContainerID> containerId; + EXPECT_CALL(containerizer, launch(_, _, _, _, _, _, _, _)) + .WillOnce(DoAll(FutureArg<0>(&containerId), + Invoke(&containerizer, + &MockExternalContainerizer::_launch))); + + driver.launchTasks(offers.get()[0].id(), {task}); + + AWAIT_READY(containerId); + + AWAIT_READY(status); + + EXPECT_EQ(TASK_RUNNING, status.get().state()); + + // Wait for the task to begin inducing cpu time. + while (!os::exists(file)); + + ExecutorID executorId; + executorId.set_value(task.task_id().value()); + + // We'll wait up to 10 seconds for the child process to induce + // 1/8 of a second of user and system cpu time in total. + // TODO(bmahler): Also induce rss memory consumption, by re-using + // the balloon framework. + ResourceStatistics statistics; + Duration waited = Duration::zero(); + do { + Future<ResourceStatistics> usage = containerizer.usage(containerId.get()); + AWAIT_READY(usage); + + statistics = usage.get(); + + // If we meet our usage expectations, we're done! + // NOTE: We are currently getting dummy-data from the test- + // containerizer python script matching these expectations. + // TODO(tillt): Consider working with real data. + if (statistics.cpus_user_time_secs() >= 0.120 && + statistics.cpus_system_time_secs() >= 0.05 && + statistics.mem_rss_bytes() >= 1024u) { + break; + } + + os::sleep(Milliseconds(100)); + waited += Milliseconds(100); + } while (waited < Seconds(10)); + + EXPECT_GE(statistics.cpus_user_time_secs(), 0.120); + EXPECT_GE(statistics.cpus_system_time_secs(), 0.05); + EXPECT_EQ(statistics.cpus_limit(), cpus.get()); + EXPECT_GE(statistics.mem_rss_bytes(), 1024u); + EXPECT_EQ(statistics.mem_limit_bytes(), mem.get().bytes()); + + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&status)); + + driver.killTask(task.task_id()); + + AWAIT_READY(status); + + EXPECT_EQ(TASK_KILLED, status.get().state()); + + driver.stop(); + driver.join(); + + this->Shutdown(); +} + +#endif // MESOS_HAS_PYTHON + +} // namespace tests { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/fs_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/fs_tests.cpp b/src/tests/containerizer/fs_tests.cpp new file mode 100644 index 0000000..34d3c41 --- /dev/null +++ b/src/tests/containerizer/fs_tests.cpp @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <paths.h> + +#include <gmock/gmock.h> + +#include <stout/foreach.hpp> +#include <stout/gtest.hpp> +#include <stout/none.hpp> +#include <stout/option.hpp> +#include <stout/try.hpp> + +#include "linux/fs.hpp" + +namespace mesos { +namespace internal { +namespace tests { + +using fs::MountTable; +using fs::FileSystemTable; +using fs::MountInfoTable; + + +TEST(FsTest, MountTableRead) +{ + Try<MountTable> table = MountTable::read(_PATH_MOUNTED); + + ASSERT_SOME(table); + + Option<MountTable::Entry> root = None(); + Option<MountTable::Entry> proc = None(); + foreach (const MountTable::Entry& entry, table.get().entries) { + if (entry.dir == "/") { + root = entry; + } else if (entry.dir == "/proc") { + proc = entry; + } + } + + EXPECT_SOME(root); + ASSERT_SOME(proc); + EXPECT_EQ(proc.get().type, "proc"); +} + + +TEST(FsTest, MountTableHasOption) +{ + Try<MountTable> table = MountTable::read(_PATH_MOUNTED); + + ASSERT_SOME(table); + + Option<MountTable::Entry> proc = None(); + foreach (const MountTable::Entry& entry, table.get().entries) { + if (entry.dir == "/proc") { + proc = entry; + } + } + + ASSERT_SOME(proc); + EXPECT_TRUE(proc.get().hasOption(MNTOPT_RW)); +} + + +TEST(FsTest, FileSystemTableRead) +{ + Try<FileSystemTable> table = FileSystemTable::read(); + + ASSERT_SOME(table); + + // NOTE: We do not check for /proc because, it is not always present in + // /etc/fstab. + Option<FileSystemTable::Entry> root = None(); + foreach (const FileSystemTable::Entry& entry, table.get().entries) { + if (entry.file == "/") { + root = entry; + } + } + + EXPECT_SOME(root); +} + + +TEST(FsTest, MountInfoTableParse) +{ + // Parse a private mount (no optional fields). + const std::string privateMount = + "19 1 8:1 / / rw,relatime - ext4 /dev/sda1 rw,seclabel,data=ordered"; + Try<MountInfoTable::Entry> entry = MountInfoTable::Entry::parse(privateMount); + + ASSERT_SOME(entry); + EXPECT_EQ(19, entry.get().id); + EXPECT_EQ(1, entry.get().parent); + EXPECT_EQ(makedev(8, 1), entry.get().devno); + EXPECT_EQ("/", entry.get().root); + EXPECT_EQ("/", entry.get().target); + EXPECT_EQ("rw,relatime", entry.get().vfsOptions); + EXPECT_EQ("rw,seclabel,data=ordered", entry.get().fsOptions); + EXPECT_EQ("", entry.get().optionalFields); + EXPECT_EQ("ext4", entry.get().type); + EXPECT_EQ("/dev/sda1", entry.get().source); + + // Parse a shared mount (includes one optional field). + const std::string sharedMount = + "19 1 8:1 / / rw,relatime shared:2 - ext4 /dev/sda1 rw,seclabel"; + entry = MountInfoTable::Entry::parse(sharedMount); + + ASSERT_SOME(entry); + EXPECT_EQ(19, entry.get().id); + EXPECT_EQ(1, entry.get().parent); + EXPECT_EQ(makedev(8, 1), entry.get().devno); + EXPECT_EQ("/", entry.get().root); + EXPECT_EQ("/", entry.get().target); + EXPECT_EQ("rw,relatime", entry.get().vfsOptions); + EXPECT_EQ("rw,seclabel", entry.get().fsOptions); + EXPECT_EQ("shared:2", entry.get().optionalFields); + EXPECT_EQ("ext4", entry.get().type); + EXPECT_EQ("/dev/sda1", entry.get().source); +} + + +TEST(FsTest, DISABLED_MountInfoTableRead) +{ + // Examine the calling process's mountinfo table. + Try<fs::MountInfoTable> table = fs::MountInfoTable::read(); + ASSERT_SOME(table); + + // Every system should have at least a rootfs mounted. + Option<MountInfoTable::Entry> root = None(); + foreach (const MountInfoTable::Entry& entry, table.get().entries) { + if (entry.target == "/") { + root = entry; + } + } + + EXPECT_SOME(root); + + // Repeat for pid 1. + table = fs::MountInfoTable::read(1); + ASSERT_SOME(table); + + // Every system should have at least a rootfs mounted. + root = None(); + foreach (const MountInfoTable::Entry& entry, table.get().entries) { + if (entry.target == "/") { + root = entry; + } + } + + EXPECT_SOME(root); +} + +} // namespace tests { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/isolator.hpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/isolator.hpp b/src/tests/containerizer/isolator.hpp new file mode 100644 index 0000000..8aaf88c --- /dev/null +++ b/src/tests/containerizer/isolator.hpp @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __TEST_ISOLATOR_HPP__ +#define __TEST_ISOLATOR_HPP__ + +#include <gmock/gmock.h> + +#include "slave/containerizer/isolator.hpp" + +namespace mesos { +namespace internal { +namespace tests { + +class TestIsolatorProcess : public slave::MesosIsolatorProcess +{ +public: + static Try<mesos::slave::Isolator*> create( + const Option<CommandInfo>& commandInfo) + { + process::Owned<MesosIsolatorProcess> process( + new TestIsolatorProcess(commandInfo)); + + return new slave::MesosIsolator(process); + } + + MOCK_METHOD2( + recover, + process::Future<Nothing>( + const std::list<mesos::slave::ExecutorRunState>&, + const hashset<ContainerID>&)); + + virtual process::Future<Option<CommandInfo>> prepare( + const ContainerID& containerId, + const ExecutorInfo& executorInfo, + const std::string& directory, + const Option<std::string>& rootfs, + const Option<std::string>& user) + { + return commandInfo; + } + + MOCK_METHOD2( + isolate, + process::Future<Nothing>(const ContainerID&, pid_t)); + + MOCK_METHOD1( + watch, + process::Future<mesos::slave::ExecutorLimitation>(const ContainerID&)); + + MOCK_METHOD2( + update, + process::Future<Nothing>(const ContainerID&, const Resources&)); + + MOCK_METHOD1( + usage, + process::Future<ResourceStatistics>(const ContainerID&)); + + MOCK_METHOD1( + cleanup, + process::Future<Nothing>(const ContainerID&)); + +private: + TestIsolatorProcess(const Option<CommandInfo>& _commandInfo) + : commandInfo(_commandInfo) + { + EXPECT_CALL(*this, watch(testing::_)) + .WillRepeatedly(testing::Return(promise.future())); + + EXPECT_CALL(*this, isolate(testing::_, testing::_)) + .WillRepeatedly(testing::Return(Nothing())); + + EXPECT_CALL(*this, cleanup(testing::_)) + .WillRepeatedly(testing::Return(Nothing())); + } + + const Option<CommandInfo> commandInfo; + + process::Promise<mesos::slave::ExecutorLimitation> promise; +}; + +} // namespace tests { +} // namespace internal { +} // namespace mesos { + +#endif // __TEST_ISOLATOR_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/isolator_tests.cpp b/src/tests/containerizer/isolator_tests.cpp new file mode 100644 index 0000000..59f08c0 --- /dev/null +++ b/src/tests/containerizer/isolator_tests.cpp @@ -0,0 +1,1317 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <unistd.h> + +#include <gmock/gmock.h> + +#include <iostream> +#include <string> +#include <vector> + +#include <mesos/resources.hpp> + +#include <mesos/module/isolator.hpp> + +#include <mesos/slave/isolator.hpp> + +#include <process/future.hpp> +#include <process/owned.hpp> +#include <process/reap.hpp> + +#include <stout/abort.hpp> +#include <stout/gtest.hpp> +#include <stout/os.hpp> +#include <stout/path.hpp> + +#ifdef __linux__ +#include "linux/ns.hpp" +#endif // __linux__ + +#include "master/master.hpp" +#include "master/detector.hpp" + +#include "slave/flags.hpp" +#include "slave/slave.hpp" + +#ifdef __linux__ +#include "slave/containerizer/isolators/cgroups/constants.hpp" +#include "slave/containerizer/isolators/cgroups/cpushare.hpp" +#include "slave/containerizer/isolators/cgroups/mem.hpp" +#include "slave/containerizer/isolators/cgroups/perf_event.hpp" +#include "slave/containerizer/isolators/filesystem/shared.hpp" +#endif // __linux__ +#include "slave/containerizer/isolators/posix.hpp" + +#include "slave/containerizer/launcher.hpp" +#ifdef __linux__ +#include "slave/containerizer/fetcher.hpp" +#include "slave/containerizer/linux_launcher.hpp" + +#include "slave/containerizer/mesos/containerizer.hpp" +#include "slave/containerizer/mesos/launch.hpp" +#endif // __linux__ + +#include "tests/flags.hpp" +#include "tests/mesos.hpp" +#include "tests/module.hpp" +#include "tests/utils.hpp" + +#include "tests/containerizer/memory_test_helper.hpp" + +using namespace process; + +using mesos::internal::master::Master; +#ifdef __linux__ +using mesos::internal::slave::CgroupsCpushareIsolatorProcess; +using mesos::internal::slave::CgroupsMemIsolatorProcess; +using mesos::internal::slave::CgroupsPerfEventIsolatorProcess; +using mesos::internal::slave::CPU_SHARES_PER_CPU_REVOCABLE; +using mesos::internal::slave::Fetcher; +using mesos::internal::slave::LinuxLauncher; +using mesos::internal::slave::SharedFilesystemIsolatorProcess; +#endif // __linux__ +using mesos::internal::slave::Launcher; +using mesos::internal::slave::MesosContainerizer; +using mesos::internal::slave::PosixLauncher; +using mesos::internal::slave::PosixCpuIsolatorProcess; +using mesos::internal::slave::PosixMemIsolatorProcess; +using mesos::internal::slave::Slave; + +using mesos::slave::Isolator; +using mesos::slave::IsolatorProcess; + +using std::ostringstream; +using std::set; +using std::string; +using std::vector; + +using testing::_; +using testing::DoAll; +using testing::Return; +using testing::SaveArg; + +namespace mesos { +namespace internal { +namespace tests { + +static int childSetup(int pipes[2]) +{ + // In child process. + while (::close(pipes[1]) == -1 && errno == EINTR); + + // Wait until the parent signals us to continue. + char dummy; + ssize_t length; + while ((length = ::read(pipes[0], &dummy, sizeof(dummy))) == -1 && + errno == EINTR); + + if (length != sizeof(dummy)) { + ABORT("Failed to synchronize with parent"); + } + + while (::close(pipes[0]) == -1 && errno == EINTR); + + return 0; +} + + +template <typename T> +class CpuIsolatorTest : public MesosTest {}; + + +typedef ::testing::Types< + PosixCpuIsolatorProcess, +#ifdef __linux__ + CgroupsCpushareIsolatorProcess, +#endif // __linux__ + tests::Module<Isolator, TestCpuIsolator>> CpuIsolatorTypes; + + +TYPED_TEST_CASE(CpuIsolatorTest, CpuIsolatorTypes); + + +TYPED_TEST(CpuIsolatorTest, UserCpuUsage) +{ + slave::Flags flags; + + Try<Isolator*> isolator = TypeParam::create(flags); + CHECK_SOME(isolator); + + // A PosixLauncher is sufficient even when testing a cgroups isolator. + Try<Launcher*> launcher = PosixLauncher::create(flags); + + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("cpus:1.0").get()); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + const string& file = path::join(dir.get(), "mesos_isolator_test_ready"); + + // Max out a single core in userspace. This will run for at most one second. + string command = "while true ; do true ; done &" + "touch " + file + "; " // Signals the command is running. + "sleep 60"; + + int pipes[2]; + ASSERT_NE(-1, ::pipe(pipes)); + + vector<string> argv(3); + argv[0] = "sh"; + argv[1] = "-c"; + argv[2] = command; + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + argv, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + lambda::bind(&childSetup, pipes)); + + ASSERT_SOME(pid); + + // Reap the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + // Continue in the parent. + ASSERT_SOME(os::close(pipes[0])); + + // Isolate the forked child. + AWAIT_READY(isolator.get()->isolate(containerId, pid.get())); + + // Now signal the child to continue. + char dummy; + ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy))); + + ASSERT_SOME(os::close(pipes[1])); + + // Wait for the command to start. + while (!os::exists(file)); + + // Wait up to 1 second for the child process to induce 1/8 of a second of + // user cpu time. + ResourceStatistics statistics; + Duration waited = Duration::zero(); + do { + Future<ResourceStatistics> usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + + statistics = usage.get(); + + // If we meet our usage expectations, we're done! + if (statistics.cpus_user_time_secs() >= 0.125) { + break; + } + + os::sleep(Milliseconds(200)); + waited += Milliseconds(200); + } while (waited < Seconds(1)); + + EXPECT_LE(0.125, statistics.cpus_user_time_secs()); + + // Ensure all processes are killed. + AWAIT_READY(launcher.get()->destroy(containerId)); + + // Make sure the child was reaped. + AWAIT_READY(status); + + // Let the isolator clean up. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); + delete launcher.get(); +} + + +TYPED_TEST(CpuIsolatorTest, SystemCpuUsage) +{ + slave::Flags flags; + + Try<Isolator*> isolator = TypeParam::create(flags); + CHECK_SOME(isolator); + + // A PosixLauncher is sufficient even when testing a cgroups isolator. + Try<Launcher*> launcher = PosixLauncher::create(flags); + + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("cpus:1.0").get()); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + const string& file = path::join(dir.get(), "mesos_isolator_test_ready"); + + // Generating random numbers is done by the kernel and will max out a single + // core and run almost exclusively in the kernel, i.e., system time. + string command = "cat /dev/urandom > /dev/null & " + "touch " + file + "; " // Signals the command is running. + "sleep 60"; + + int pipes[2]; + ASSERT_NE(-1, ::pipe(pipes)); + + vector<string> argv(3); + argv[0] = "sh"; + argv[1] = "-c"; + argv[2] = command; + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + argv, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + lambda::bind(&childSetup, pipes)); + + ASSERT_SOME(pid); + + // Reap the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + // Continue in the parent. + ASSERT_SOME(os::close(pipes[0])); + + // Isolate the forked child. + AWAIT_READY(isolator.get()->isolate(containerId, pid.get())); + + // Now signal the child to continue. + char dummy; + ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy))); + + ASSERT_SOME(os::close(pipes[1])); + + // Wait for the command to start. + while (!os::exists(file)); + + // Wait up to 1 second for the child process to induce 1/8 of a second of + // system cpu time. + ResourceStatistics statistics; + Duration waited = Duration::zero(); + do { + Future<ResourceStatistics> usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + + statistics = usage.get(); + + // If we meet our usage expectations, we're done! + if (statistics.cpus_system_time_secs() >= 0.125) { + break; + } + + os::sleep(Milliseconds(200)); + waited += Milliseconds(200); + } while (waited < Seconds(1)); + + EXPECT_LE(0.125, statistics.cpus_system_time_secs()); + + // Ensure all processes are killed. + AWAIT_READY(launcher.get()->destroy(containerId)); + + // Make sure the child was reaped. + AWAIT_READY(status); + + // Let the isolator clean up. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); + delete launcher.get(); +} + + +#ifdef __linux__ +class RevocableCpuIsolatorTest : public MesosTest {}; + + +TEST_F(RevocableCpuIsolatorTest, ROOT_CGROUPS_RevocableCpu) +{ + slave::Flags flags; + + Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = PosixLauncher::create(flags); + + // Include revocable CPU in the executor's resources. + Resource cpu = Resources::parse("cpus", "1", "*").get(); + cpu.mutable_revocable(); + + ExecutorInfo executorInfo; + executorInfo.add_resources()->CopyFrom(cpu); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + os::getcwd(), + None(), + None())); + + vector<string> argv{"sleep", "100"}; + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sleep", + argv, + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null"), + None(), + None(), + None()); + + ASSERT_SOME(pid); + + AWAIT_READY(isolator.get()->isolate(containerId, pid.get())); + + // Executor should have proper cpu.shares for revocable containers. + Result<string> cpuHierarchy = cgroups::hierarchy("cpu"); + ASSERT_SOME(cpuHierarchy); + + Result<string> cpuCgroup = cgroups::cpu::cgroup(pid.get()); + ASSERT_SOME(cpuCgroup); + + EXPECT_SOME_EQ( + CPU_SHARES_PER_CPU_REVOCABLE, + cgroups::cpu::shares(cpuHierarchy.get(), cpuCgroup.get())); + + // Kill the container and clean up. + Future<Option<int>> status = process::reap(pid.get()); + + AWAIT_READY(launcher.get()->destroy(containerId)); + + AWAIT_READY(status); + + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); + delete launcher.get(); +} +#endif // __linux__ + + +#ifdef __linux__ +class LimitedCpuIsolatorTest : public MesosTest {}; + + +TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_Cfs) +{ + slave::Flags flags; + + // Enable CFS to cap CPU utilization. + flags.cgroups_enable_cfs = true; + + Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = + LinuxLauncher::create(flags, isolator.get()->namespaces().get()); + CHECK_SOME(launcher); + + // Set the executor's resources to 0.5 cpu. + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("cpus:0.5").get()); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + // Generate random numbers to max out a single core. We'll run this for 0.5 + // seconds of wall time so it should consume approximately 250 ms of total + // cpu time when limited to 0.5 cpu. We use /dev/urandom to prevent blocking + // on Linux when there's insufficient entropy. + string command = "cat /dev/urandom > /dev/null & " + "export MESOS_TEST_PID=$! && " + "sleep 0.5 && " + "kill $MESOS_TEST_PID"; + + int pipes[2]; + ASSERT_NE(-1, ::pipe(pipes)); + + vector<string> argv(3); + argv[0] = "sh"; + argv[1] = "-c"; + argv[2] = command; + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + argv, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + lambda::bind(&childSetup, pipes)); + + ASSERT_SOME(pid); + + // Reap the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + // Continue in the parent. + ASSERT_SOME(os::close(pipes[0])); + + // Isolate the forked child. + AWAIT_READY(isolator.get()->isolate(containerId, pid.get())); + + // Now signal the child to continue. + char dummy; + ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy))); + + ASSERT_SOME(os::close(pipes[1])); + + // Wait for the command to complete. + AWAIT_READY(status); + + Future<ResourceStatistics> usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + + // Expect that no more than 300 ms of cpu time has been consumed. We also + // check that at least 50 ms of cpu time has been consumed so this test will + // fail if the host system is very heavily loaded. This behavior is correct + // because under such conditions we aren't actually testing the CFS cpu + // limiter. + double cpuTime = usage.get().cpus_system_time_secs() + + usage.get().cpus_user_time_secs(); + + EXPECT_GE(0.30, cpuTime); + EXPECT_LE(0.05, cpuTime); + + // Ensure all processes are killed. + AWAIT_READY(launcher.get()->destroy(containerId)); + + // Let the isolator clean up. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); + delete launcher.get(); +} + + +// This test verifies that we can successfully launch a container with +// a big (>= 10 cpus) cpu quota. This is to catch the regression +// observed in MESOS-1049. +// TODO(vinod): Revisit this if/when the isolator restricts the number +// of cpus that an executor can use based on the slave cpus. +TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_Cfs_Big_Quota) +{ + slave::Flags flags; + + // Enable CFS to cap CPU utilization. + flags.cgroups_enable_cfs = true; + + Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = + LinuxLauncher::create(flags, isolator.get()->namespaces().get()); + CHECK_SOME(launcher); + + // Set the executor's resources to 100.5 cpu. + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("cpus:100.5").get()); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + int pipes[2]; + ASSERT_NE(-1, ::pipe(pipes)); + + vector<string> argv(3); + argv[0] = "sh"; + argv[1] = "-c"; + argv[2] = "exit 0"; + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + argv, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + lambda::bind(&childSetup, pipes)); + + ASSERT_SOME(pid); + + // Reap the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + // Continue in the parent. + ASSERT_SOME(os::close(pipes[0])); + + // Isolate the forked child. + AWAIT_READY(isolator.get()->isolate(containerId, pid.get())); + + // Now signal the child to continue. + char dummy; + ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy))); + + ASSERT_SOME(os::close(pipes[1])); + + // Wait for the command to complete successfully. + AWAIT_READY(status); + ASSERT_SOME_EQ(0, status.get()); + + // Ensure all processes are killed. + AWAIT_READY(launcher.get()->destroy(containerId)); + + // Let the isolator clean up. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); + delete launcher.get(); +} + + +// A test to verify the number of processes and threads in a +// container. +TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_Pids_and_Tids) +{ + slave::Flags flags; + flags.cgroups_cpu_enable_pids_and_tids_count = true; + + Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = + LinuxLauncher::create(flags, isolator.get()->namespaces().get()); + CHECK_SOME(launcher); + + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("cpus:0.5;mem:512").get()); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + // Right after the creation of the cgroup, which happens in + // 'prepare', we check that it is empty. + Future<ResourceStatistics> usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + EXPECT_EQ(0U, usage.get().processes()); + EXPECT_EQ(0U, usage.get().threads()); + + int pipes[2]; + ASSERT_NE(-1, ::pipe(pipes)); + + vector<string> argv(3); + argv[0] = "sh"; + argv[1] = "-c"; + argv[2] = "while true; do sleep 1; done;"; + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + argv, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + lambda::bind(&childSetup, pipes)); + + ASSERT_SOME(pid); + + // Reap the forked child. + Future<Option<int>> status = process::reap(pid.get()); + + // Continue in the parent. + ASSERT_SOME(os::close(pipes[0])); + + // Before isolation, the cgroup is empty. + usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + EXPECT_EQ(0U, usage.get().processes()); + EXPECT_EQ(0U, usage.get().threads()); + + // Isolate the forked child. + AWAIT_READY(isolator.get()->isolate(containerId, pid.get())); + + // After the isolation, the cgroup is not empty, even though the + // process hasn't exec'd yet. + usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + EXPECT_EQ(1U, usage.get().processes()); + EXPECT_EQ(1U, usage.get().threads()); + + // Now signal the child to continue. + char dummy; + ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy))); + + ASSERT_SOME(os::close(pipes[1])); + + // Process count should be 1 since 'sleep' is still sleeping. + usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + EXPECT_EQ(1U, usage.get().processes()); + EXPECT_EQ(1U, usage.get().threads()); + + // Ensure all processes are killed. + AWAIT_READY(launcher.get()->destroy(containerId)); + + // Wait for the command to complete. + AWAIT_READY(status); + + // After the process is killed, the cgroup should be empty again. + usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + EXPECT_EQ(0U, usage.get().processes()); + EXPECT_EQ(0U, usage.get().threads()); + + // Let the isolator clean up. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); + delete launcher.get(); +} +#endif // __linux__ + + +template <typename T> +class MemIsolatorTest : public MesosTest {}; + + +typedef ::testing::Types< + PosixMemIsolatorProcess, +#ifdef __linux__ + CgroupsMemIsolatorProcess, +#endif // __linux__ + tests::Module<Isolator, TestMemIsolator>> MemIsolatorTypes; + + +TYPED_TEST_CASE(MemIsolatorTest, MemIsolatorTypes); + + +TYPED_TEST(MemIsolatorTest, MemUsage) +{ + slave::Flags flags; + + Try<Isolator*> isolator = TypeParam::create(flags); + CHECK_SOME(isolator); + + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("mem:1024").get()); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + MemoryTestHelper helper; + ASSERT_SOME(helper.spawn()); + ASSERT_SOME(helper.pid()); + + // Set up the reaper to wait on the subprocess. + Future<Option<int>> status = process::reap(helper.pid().get()); + + // Isolate the subprocess. + AWAIT_READY(isolator.get()->isolate(containerId, helper.pid().get())); + + const Bytes allocation = Megabytes(128); + EXPECT_SOME(helper.increaseRSS(allocation)); + + Future<ResourceStatistics> usage = isolator.get()->usage(containerId); + AWAIT_READY(usage); + + EXPECT_GE(usage.get().mem_rss_bytes(), allocation.bytes()); + + // Ensure the process is killed. + helper.cleanup(); + + // Make sure the subprocess was reaped. + AWAIT_READY(status); + + // Let the isolator clean up. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); +} + + +#ifdef __linux__ +class PerfEventIsolatorTest : public MesosTest {}; + + +TEST_F(PerfEventIsolatorTest, ROOT_CGROUPS_Sample) +{ + slave::Flags flags; + + flags.perf_events = "cycles,task-clock"; + flags.perf_duration = Milliseconds(250); + flags.perf_interval = Milliseconds(500); + + Try<Isolator*> isolator = CgroupsPerfEventIsolatorProcess::create(flags); + ASSERT_SOME(isolator); + + ExecutorInfo executorInfo; + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Use a relative temporary directory so it gets cleaned up + // automatically with the test. + Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX")); + ASSERT_SOME(dir); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + dir.get(), + None(), + None())); + + // This first sample is likely to be empty because perf hasn't + // completed yet but we should still have the required fields. + Future<ResourceStatistics> statistics1 = isolator.get()->usage(containerId); + AWAIT_READY(statistics1); + ASSERT_TRUE(statistics1.get().has_perf()); + EXPECT_TRUE(statistics1.get().perf().has_timestamp()); + EXPECT_TRUE(statistics1.get().perf().has_duration()); + + // Wait until we get the next sample. We use a generous timeout of + // two seconds because we currently have a one second reap interval; + // when running perf with perf_duration of 250ms we won't notice the + // exit for up to one second. + ResourceStatistics statistics2; + Duration waited = Duration::zero(); + do { + Future<ResourceStatistics> statistics = isolator.get()->usage(containerId); + AWAIT_READY(statistics); + + statistics2 = statistics.get(); + + ASSERT_TRUE(statistics2.has_perf()); + + if (statistics1.get().perf().timestamp() != + statistics2.perf().timestamp()) { + break; + } + + os::sleep(Milliseconds(250)); + waited += Milliseconds(250); + } while (waited < Seconds(2)); + + sleep(2); + + EXPECT_NE(statistics1.get().perf().timestamp(), + statistics2.perf().timestamp()); + + EXPECT_TRUE(statistics2.perf().has_cycles()); + EXPECT_LE(0u, statistics2.perf().cycles()); + + EXPECT_TRUE(statistics2.perf().has_task_clock()); + EXPECT_LE(0.0, statistics2.perf().task_clock()); + + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); +} + + +class SharedFilesystemIsolatorTest : public MesosTest {}; + + +// Test that a container can create a private view of a system +// directory (/var/tmp). Check that a file written by a process inside +// the container doesn't appear on the host filesystem but does appear +// under the container's work directory. +// This test is disabled since we're planning to remove the shared +// filesystem isolator and this test is not working on other distros +// such as CentOS 7.1 +// TODO(tnachen): Remove this test when shared filesystem isolator +// is removed. +TEST_F(SharedFilesystemIsolatorTest, DISABLED_ROOT_RelativeVolume) +{ + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "filesystem/shared"; + + Try<Isolator*> isolator = SharedFilesystemIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = + LinuxLauncher::create(flags, isolator.get()->namespaces().get()); + CHECK_SOME(launcher); + + // Use /var/tmp so we don't mask the work directory (under /tmp). + const string containerPath = "/var/tmp"; + ASSERT_TRUE(os::stat::isdir(containerPath)); + + // Use a host path relative to the container work directory. + const string hostPath = strings::remove(containerPath, "/", strings::PREFIX); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::MESOS); + containerInfo.add_volumes()->CopyFrom( + CREATE_VOLUME(containerPath, hostPath, Volume::RW)); + + ExecutorInfo executorInfo; + executorInfo.mutable_container()->CopyFrom(containerInfo); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + Future<Option<CommandInfo> > prepare = + isolator.get()->prepare( + containerId, + executorInfo, + flags.work_dir, + None(), + None()); + + AWAIT_READY(prepare); + ASSERT_SOME(prepare.get()); + + // The test will touch a file in container path. + const string file = path::join(containerPath, UUID::random().toString()); + ASSERT_FALSE(os::exists(file)); + + // Manually run the isolator's preparation command first, then touch + // the file. + vector<string> args; + args.push_back("/bin/sh"); + args.push_back("-x"); + args.push_back("-c"); + args.push_back(prepare.get().get().value() + " && touch " + file); + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + args, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + None()); + ASSERT_SOME(pid); + + // Set up the reaper to wait on the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + AWAIT_READY(status); + EXPECT_SOME_EQ(0, status.get()); + + // Check the correct hierarchy was created under the container work + // directory. + string dir = "/"; + foreach (const string& subdir, strings::tokenize(containerPath, "/")) { + dir = path::join(dir, subdir); + + struct stat hostStat; + EXPECT_EQ(0, ::stat(dir.c_str(), &hostStat)); + + struct stat containerStat; + EXPECT_EQ(0, + ::stat(path::join(flags.work_dir, dir).c_str(), &containerStat)); + + EXPECT_EQ(hostStat.st_mode, containerStat.st_mode); + EXPECT_EQ(hostStat.st_uid, containerStat.st_uid); + EXPECT_EQ(hostStat.st_gid, containerStat.st_gid); + } + + // Check it did *not* create a file in the host namespace. + EXPECT_FALSE(os::exists(file)); + + // Check it did create the file under the container's work directory + // on the host. + EXPECT_TRUE(os::exists(path::join(flags.work_dir, file))); + + delete launcher.get(); + delete isolator.get(); +} + + +// This test is disabled since we're planning to remove the shared +// filesystem isolator and this test is not working on other distros +// such as CentOS 7.1 +// TODO(tnachen): Remove this test when shared filesystem isolator +// is removed. +TEST_F(SharedFilesystemIsolatorTest, DISABLED_ROOT_AbsoluteVolume) +{ + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "filesystem/shared"; + + Try<Isolator*> isolator = SharedFilesystemIsolatorProcess::create(flags); + CHECK_SOME(isolator); + + Try<Launcher*> launcher = + LinuxLauncher::create(flags, isolator.get()->namespaces().get()); + CHECK_SOME(launcher); + + // We'll mount the absolute test work directory as /var/tmp in the + // container. + const string hostPath = flags.work_dir; + const string containerPath = "/var/tmp"; + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::MESOS); + containerInfo.add_volumes()->CopyFrom( + CREATE_VOLUME(containerPath, hostPath, Volume::RW)); + + ExecutorInfo executorInfo; + executorInfo.mutable_container()->CopyFrom(containerInfo); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + Future<Option<CommandInfo> > prepare = + isolator.get()->prepare( + containerId, + executorInfo, + flags.work_dir, + None(), + None()); + + AWAIT_READY(prepare); + ASSERT_SOME(prepare.get()); + + // Test the volume mounting by touching a file in the container's + // /tmp, which should then be in flags.work_dir. + const string filename = UUID::random().toString(); + ASSERT_FALSE(os::exists(path::join(containerPath, filename))); + + vector<string> args; + args.push_back("/bin/sh"); + args.push_back("-x"); + args.push_back("-c"); + args.push_back(prepare.get().get().value() + + " && touch " + + path::join(containerPath, filename)); + + Try<pid_t> pid = launcher.get()->fork( + containerId, + "/bin/sh", + args, + Subprocess::FD(STDIN_FILENO), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO), + None(), + None(), + None()); + ASSERT_SOME(pid); + + // Set up the reaper to wait on the forked child. + Future<Option<int> > status = process::reap(pid.get()); + + AWAIT_READY(status); + EXPECT_SOME_EQ(0, status.get()); + + // Check the file was created in flags.work_dir. + EXPECT_TRUE(os::exists(path::join(hostPath, filename))); + + // Check it didn't get created in the host's view of containerPath. + EXPECT_FALSE(os::exists(path::join(containerPath, filename))); + + delete launcher.get(); + delete isolator.get(); +} + + +class NamespacesPidIsolatorTest : public MesosTest {}; + + +TEST_F(NamespacesPidIsolatorTest, ROOT_PidNamespace) +{ + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "namespaces/pid"; + + string directory = os::getcwd(); // We're inside a temporary sandbox. + + Fetcher fetcher; + + Try<MesosContainerizer*> containerizer = + MesosContainerizer::create(flags, false, &fetcher); + ASSERT_SOME(containerizer); + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + // Write the command's pid namespace inode and init name to files. + const string command = + "stat -c %i /proc/self/ns/pid > ns && (cat /proc/1/comm > init)"; + + process::Future<bool> launch = containerizer.get()->launch( + containerId, + CREATE_EXECUTOR_INFO("executor", command), + directory, + None(), + SlaveID(), + process::PID<Slave>(), + false); + AWAIT_READY(launch); + ASSERT_TRUE(launch.get()); + + // Wait on the container. + process::Future<containerizer::Termination> wait = + containerizer.get()->wait(containerId); + AWAIT_READY(wait); + + // Check the executor exited correctly. + EXPECT_TRUE(wait.get().has_status()); + EXPECT_EQ(0, wait.get().status()); + + // Check that the command was run in a different pid namespace. + Try<ino_t> testPidNamespace = ns::getns(::getpid(), "pid"); + ASSERT_SOME(testPidNamespace); + + Try<string> containerPidNamespace = os::read(path::join(directory, "ns")); + ASSERT_SOME(containerPidNamespace); + + EXPECT_NE(stringify(testPidNamespace.get()), + strings::trim(containerPidNamespace.get())); + + // Check that 'sh' is the container's 'init' process. + // This verifies that /proc has been correctly mounted for the container. + Try<string> init = os::read(path::join(directory, "init")); + ASSERT_SOME(init); + + EXPECT_EQ("sh", strings::trim(init.get())); + + delete containerizer.get(); +} + + +// Username for the unprivileged user that will be created to test +// unprivileged cgroup creation. It will be removed after the tests. +// It is presumed this user does not normally exist. +const string UNPRIVILEGED_USERNAME = "mesos.test.unprivileged.user"; + + +template <typename T> +class UserCgroupIsolatorTest : public MesosTest +{ +public: + static void SetUpTestCase() + { + // Remove the user in case it wasn't cleaned up from a previous + // test. + os::system("userdel -r " + UNPRIVILEGED_USERNAME + " > /dev/null"); + + ASSERT_EQ(0, os::system("useradd " + UNPRIVILEGED_USERNAME)); + } + + + static void TearDownTestCase() + { + ASSERT_EQ(0, os::system("userdel -r " + UNPRIVILEGED_USERNAME)); + } +}; + + +// Test all isolators that use cgroups. +typedef ::testing::Types< + CgroupsMemIsolatorProcess, + CgroupsCpushareIsolatorProcess, + CgroupsPerfEventIsolatorProcess> CgroupsIsolatorTypes; + + +TYPED_TEST_CASE(UserCgroupIsolatorTest, CgroupsIsolatorTypes); + + +TYPED_TEST(UserCgroupIsolatorTest, ROOT_CGROUPS_UserCgroup) +{ + slave::Flags flags; + flags.perf_events = "cpu-cycles"; // Needed for CgroupsPerfEventIsolator. + + Try<Isolator*> isolator = TypeParam::create(flags); + ASSERT_SOME(isolator); + + ExecutorInfo executorInfo; + executorInfo.mutable_resources()->CopyFrom( + Resources::parse("mem:1024;cpus:1").get()); // For cpu/mem isolators. + + ContainerID containerId; + containerId.set_value(UUID::random().toString()); + + AWAIT_READY(isolator.get()->prepare( + containerId, + executorInfo, + os::getcwd(), + None(), + UNPRIVILEGED_USERNAME)); + + // Isolators don't provide a way to determine the cgroups they use + // so we'll inspect the cgroups for an isolated dummy process. + pid_t pid = fork(); + if (pid == 0) { + // Child just sleeps. + ::sleep(100); + + ABORT("Child process should not reach here"); + } + ASSERT_GT(pid, 0); + + AWAIT_READY(isolator.get()->isolate(containerId, pid)); + + // Get the container's cgroups from /proc/$PID/cgroup. We're only + // interested in the cgroups that this isolator has created which we + // can do explicitly by selecting those that have the path that + // corresponds to the 'cgroups_root' slave flag. For example: + // + // $ cat /proc/pid/cgroup + // 6:blkio:/ + // 5:perf_event:/ + // 4:memory:/mesos/b7410ed8-c85b-445e-b50e-3a1698d0e18c + // 3:freezer:/ + // 2:cpuacct:/ + // 1:cpu:/ + // + // Our 'grep' will only select the 'memory' line and then 'awk' will + // output 'memory/mesos/b7410ed8-c85b-445e-b50e-3a1698d0e18c'. + ostringstream output; + Try<int> status = os::shell( + &output, + "grep '" + path::join("/", flags.cgroups_root) + "' /proc/" + + stringify(pid) + "/cgroup | awk -F ':' '{print $2$3}'"); + + ASSERT_SOME(status); + + // Kill the dummy child process. + ::kill(pid, SIGKILL); + int exitStatus; + EXPECT_NE(-1, ::waitpid(pid, &exitStatus, 0)); + + vector<string> cgroups = strings::tokenize(output.str(), "\n"); + ASSERT_FALSE(cgroups.empty()); + + foreach (const string& cgroup, cgroups) { + // Check the user cannot manipulate the container's cgroup control + // files. + EXPECT_NE(0, os::system( + "su - " + UNPRIVILEGED_USERNAME + + " -c 'echo $$ >" + + path::join(flags.cgroups_hierarchy, cgroup, "cgroup.procs") + + "'")); + + // Check the user can create a cgroup under the container's + // cgroup. + string userCgroup = path::join(cgroup, "user"); + + EXPECT_EQ(0, os::system( + "su - " + + UNPRIVILEGED_USERNAME + + " -c 'mkdir " + + path::join(flags.cgroups_hierarchy, userCgroup) + + "'")); + + // Check the user can manipulate control files in the created + // cgroup. + EXPECT_EQ(0, os::system( + "su - " + + UNPRIVILEGED_USERNAME + + " -c 'echo $$ >" + + path::join(flags.cgroups_hierarchy, userCgroup, "cgroup.procs") + + "'")); + } + + // Clean up the container. This will also remove the nested cgroups. + AWAIT_READY(isolator.get()->cleanup(containerId)); + + delete isolator.get(); +} +#endif // __linux__ + +} // namespace tests { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/launch_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/launch_tests.cpp b/src/tests/containerizer/launch_tests.cpp new file mode 100644 index 0000000..73c8c5f --- /dev/null +++ b/src/tests/containerizer/launch_tests.cpp @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> +#include <vector> + +#include <gmock/gmock.h> + +#include <stout/foreach.hpp> +#include <stout/gtest.hpp> +#include <stout/os.hpp> +#include <stout/try.hpp> + +#include <process/gtest.hpp> +#include <process/io.hpp> +#include <process/reap.hpp> +#include <process/subprocess.hpp> + +#include "mesos/resources.hpp" + +#include "slave/containerizer/mesos/launch.hpp" + +#include "linux/fs.hpp" + +#include "tests/flags.hpp" +#include "tests/utils.hpp" + +using namespace process; + +using std::string; +using std::vector; + +namespace mesos { +namespace internal { +namespace tests { + +class Chroot +{ +public: + Chroot(const string& _rootfs) + : rootfs(_rootfs) {} + + virtual ~Chroot() {} + + virtual Try<Subprocess> run(const string& command) = 0; + + const string rootfs; +}; + + +class BasicLinuxChroot : public Chroot +{ +public: + static Try<Owned<Chroot>> create(const string& rootfs) + { + if (!os::exists(rootfs)) { + return Error("rootfs does not exist"); + } + + if (os::system("cp -r /bin " + rootfs + "/") != 0) { + return ErrnoError("Failed to copy /bin to chroot"); + } + + if (os::system("cp -r /lib " + rootfs + "/") != 0) { + return ErrnoError("Failed to copy /lib to chroot"); + } + + if (os::system("cp -r /lib64 " + rootfs + "/") != 0) { + return ErrnoError("Failed to copy /lib64 to chroot"); + } + + vector<string> directories = {"proc", "sys", "dev", "tmp"}; + foreach (const string& directory, directories) { + Try<Nothing> mkdir = os::mkdir(path::join(rootfs, directory)); + if (mkdir.isError()) { + return Error("Failed to create /" + directory + " in chroot: " + + mkdir.error()); + } + } + + // We need to bind mount the rootfs so we can pivot on it. + Try<Nothing> mount = + fs::mount(rootfs, rootfs, None(), MS_BIND | MS_SLAVE, NULL); + + if (mount.isError()) { + return Error("Failed to bind mount chroot rootfs: " + mount.error()); + } + + return Owned<Chroot>(new BasicLinuxChroot(rootfs)); + } + + virtual Try<Subprocess> run(const string& _command) + { + slave::MesosContainerizerLaunch::Flags launchFlags; + + CommandInfo command; + command.set_value(_command); + + launchFlags.command = JSON::Protobuf(command); + launchFlags.directory = "/tmp"; + launchFlags.pipe_read = open("/dev/zero", O_RDONLY); + launchFlags.pipe_write = open("/dev/null", O_WRONLY); + launchFlags.rootfs = rootfs; + + vector<string> argv(2); + argv[0] = "mesos-containerizer"; + argv[1] = slave::MesosContainerizerLaunch::NAME; + + Try<Subprocess> s = subprocess( + path::join(tests::flags.build_dir, "src", "mesos-containerizer"), + argv, + Subprocess::PATH("/dev/null"), + Subprocess::PIPE(), + Subprocess::FD(STDERR_FILENO), + launchFlags, + None(), + None(), + lambda::bind(&clone, lambda::_1)); + + if (s.isError()) { + close(launchFlags.pipe_read.get()); + close(launchFlags.pipe_write.get()); + } else { + s.get().status().onAny([=]() { + // Close when the subprocess terminates. + close(launchFlags.pipe_read.get()); + close(launchFlags.pipe_write.get()); + }); + } + + return s; + } + +private: + static pid_t clone(const lambda::function<int()>& f) + { + static unsigned long long stack[(8*1024*1024)/sizeof(unsigned long long)]; + + return ::clone( + _clone, + &stack[sizeof(stack)/sizeof(stack[0]) - 1], // Stack grows down. + CLONE_NEWNS | SIGCHLD, // Specify SIGCHLD as child termination signal. + (void*) &f); + } + + static int _clone(void* f) + { + const lambda::function<int()>* _f = + static_cast<const lambda::function<int()>*> (f); + + return (*_f)(); + } + + BasicLinuxChroot(const string& rootfs) : Chroot(rootfs) {} + + ~BasicLinuxChroot() + { + // Because the test process has the rootfs as its cwd the umount + // won't actually happen until the + // TemporaryDirectoryTest::TearDown() changes back to the original + // directory. + fs::unmount(rootfs, MNT_DETACH); + } +}; + + +template <typename T> +class LaunchChrootTest : public TemporaryDirectoryTest {}; + + +// TODO(idownes): Add tests for OSX chroots. +typedef ::testing::Types<BasicLinuxChroot> ChrootTypes; + + +TYPED_TEST_CASE(LaunchChrootTest, ChrootTypes); + + +TYPED_TEST(LaunchChrootTest, ROOT_DifferentRoot) +{ + Try<Owned<Chroot>> chroot = TypeParam::create(os::getcwd()); + ASSERT_SOME(chroot); + + // Add /usr/bin/stat into the chroot. + const string usrbin = path::join(chroot.get()->rootfs, "usr", "bin"); + ASSERT_SOME(os::mkdir(usrbin)); + ASSERT_EQ(0, os::system("cp /usr/bin/stat " + path::join(usrbin, "stat"))); + + Clock::pause(); + + Try<Subprocess> s = chroot.get()->run( + "/usr/bin/stat -c %i / >" + path::join("/", "stat.output")); + + CHECK_SOME(s); + + // Advance time until the internal reaper reaps the subprocess. + while (s.get().status().isPending()) { + Clock::advance(Seconds(1)); + Clock::settle(); + } + + AWAIT_ASSERT_READY(s.get().status()); + ASSERT_SOME(s.get().status().get()); + + int status = s.get().status().get().get(); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); + + // Check the chroot has a different root by comparing the inodes. + Try<ino_t> self = os::stat::inode("/"); + ASSERT_SOME(self); + + Try<string> read = os::read(path::join(chroot.get()->rootfs, "stat.output")); + CHECK_SOME(read); + + Try<ino_t> other = numify<ino_t>(strings::trim(read.get())); + ASSERT_SOME(other); + + EXPECT_NE(self.get(), other.get()); +} + +} // namespace tests { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/launcher.hpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/launcher.hpp b/src/tests/containerizer/launcher.hpp new file mode 100644 index 0000000..78216e0 --- /dev/null +++ b/src/tests/containerizer/launcher.hpp @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <list> +#include <map> +#include <string> +#include <vector> + +#include <gmock/gmock.h> + +#include <mesos/mesos.hpp> + +#include <mesos/slave/isolator.hpp> + +#include <process/future.hpp> +#include <process/owned.hpp> +#include <process/subprocess.hpp> + +#include <stout/flags.hpp> +#include <stout/lambda.hpp> +#include <stout/nothing.hpp> +#include <stout/option.hpp> + +#include "slave/containerizer/launcher.hpp" + +namespace mesos { +namespace internal { +namespace tests { + + +ACTION_P(InvokeRecover, launcher) +{ + return launcher->real->recover(arg0); +} + + +ACTION_P(InvokeFork, launcher) +{ + return launcher->real->fork( + arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); +} + + +ACTION_P(InvokeDestroy, launcher) +{ + return launcher->real->destroy(arg0); +} + + +class TestLauncher : public slave::Launcher +{ +public: + TestLauncher(const process::Owned<slave::Launcher>& _real) + : real(_real) + { + using testing::_; + using testing::DoDefault; + + ON_CALL(*this, recover(_)) + .WillByDefault(InvokeRecover(this)); + EXPECT_CALL(*this, recover(_)) + .WillRepeatedly(DoDefault()); + + ON_CALL(*this, fork(_, _, _, _, _, _, _, _, _)) + .WillByDefault(InvokeFork(this)); + EXPECT_CALL(*this, fork(_, _, _, _, _, _, _, _, _)) + .WillRepeatedly(DoDefault()); + + ON_CALL(*this, destroy(_)) + .WillByDefault(InvokeDestroy(this)); + EXPECT_CALL(*this, destroy(_)) + .WillRepeatedly(DoDefault()); + } + + ~TestLauncher() {} + + MOCK_METHOD1( + recover, + process::Future<hashset<ContainerID>>( + const std::list<mesos::slave::ExecutorRunState>& states)); + + MOCK_METHOD9( + fork, + Try<pid_t>( + const ContainerID& containerId, + const std::string& path, + const std::vector<std::string>& argv, + const process::Subprocess::IO& in, + const process::Subprocess::IO& out, + const process::Subprocess::IO& err, + const Option<flags::FlagsBase>& flags, + const Option<std::map<std::string, std::string> >& env, + const Option<lambda::function<int()> >& setup)); + + MOCK_METHOD1( + destroy, + process::Future<Nothing>(const ContainerID& containerId)); + + process::Owned<slave::Launcher> real; +}; + +} // namespace tests { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/96351372/src/tests/containerizer/memory_pressure_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/memory_pressure_tests.cpp b/src/tests/containerizer/memory_pressure_tests.cpp new file mode 100644 index 0000000..8089879 --- /dev/null +++ b/src/tests/containerizer/memory_pressure_tests.cpp @@ -0,0 +1,293 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include <vector> + +#include <mesos/resources.hpp> +#include <mesos/scheduler.hpp> + +#include <process/gtest.hpp> + +#include <stout/gtest.hpp> +#include <stout/os.hpp> + +#include "master/master.hpp" + +#include "slave/slave.hpp" + +#include "slave/containerizer/containerizer.hpp" +#include "slave/containerizer/fetcher.hpp" + +#include "messages/messages.hpp" + +#include "tests/mesos.hpp" + +using namespace process; + +using mesos::internal::master::Master; + +using mesos::internal::slave::Fetcher; +using mesos::internal::slave::MesosContainerizer; +using mesos::internal::slave::Slave; + +using std::vector; + +using testing::_; +using testing::Eq; +using testing::Return; + +namespace mesos { +namespace internal { +namespace tests { + +class MemoryPressureMesosTest : public ContainerizerTest<MesosContainerizer> +{ +public: + static void SetUpTestCase() + { + // Verify that the dd command and its flags used in a bit are valid + // on this system. + ASSERT_EQ(0, os::system("dd count=1 bs=1M if=/dev/zero of=/dev/null")) + << "Cannot find a compatible 'dd' command"; + } +}; + + +TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_Statistics) +{ + Try<PID<Master>> master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + + // We only care about memory cgroup for this test. + flags.isolation = "cgroups/mem"; + flags.slave_subsystems = None(); + + Fetcher fetcher; + + Try<MesosContainerizer*> containerizer = + MesosContainerizer::create(flags, true, &fetcher); + + ASSERT_SOME(containerizer); + + Try<PID<Slave>> slave = StartSlave(containerizer.get(), flags); + ASSERT_SOME(slave); + + MockScheduler sched; + + MesosSchedulerDriver driver( + &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); + + EXPECT_CALL(sched, registered(_, _, _)); + + Future<vector<Offer>> offers; + EXPECT_CALL(sched, resourceOffers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(offers); + EXPECT_NE(0u, offers.get().size()); + + Offer offer = offers.get()[0]; + + // Run a task that triggers memory pressure event. We request 1G + // disk because we are going to write a 512 MB file repeatedly. + TaskInfo task = createTask( + offer.slave_id(), + Resources::parse("cpus:1;mem:256;disk:1024").get(), + "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); + + Future<TaskStatus> status; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&status)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + driver.launchTasks(offer.id(), {task}); + + AWAIT_READY(status); + EXPECT_EQ(task.task_id(), status.get().task_id()); + EXPECT_EQ(TASK_RUNNING, status.get().state()); + + Future<hashset<ContainerID>> containers = containerizer.get()->containers(); + AWAIT_READY(containers); + ASSERT_EQ(1u, containers.get().size()); + + ContainerID containerId = *(containers.get().begin()); + + Duration waited = Duration::zero(); + do { + Future<ResourceStatistics> usage = containerizer.get()->usage(containerId); + AWAIT_READY(usage); + + if (usage.get().mem_low_pressure_counter() > 0) { + EXPECT_GE(usage.get().mem_low_pressure_counter(), + usage.get().mem_medium_pressure_counter()); + EXPECT_GE(usage.get().mem_medium_pressure_counter(), + usage.get().mem_critical_pressure_counter()); + break; + } + + os::sleep(Milliseconds(100)); + waited += Milliseconds(100); + } while (waited < Seconds(5)); + + EXPECT_LE(waited, Seconds(5)); + + driver.stop(); + driver.join(); + + Shutdown(); + delete containerizer.get(); +} + + +// Test that memory pressure listening is restarted after recovery. +TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_SlaveRecovery) +{ + Try<PID<Master>> master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + + // We only care about memory cgroup for this test. + flags.isolation = "cgroups/mem"; + flags.slave_subsystems = None(); + + Fetcher fetcher; + + Try<MesosContainerizer*> containerizer1 = + MesosContainerizer::create(flags, true, &fetcher); + + ASSERT_SOME(containerizer1); + + Try<PID<Slave>> slave = StartSlave(containerizer1.get(), flags); + ASSERT_SOME(slave); + + MockScheduler sched; + + // Enable checkpointing for the framework. + FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; + frameworkInfo.set_checkpoint(true); + + MesosSchedulerDriver driver( + &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); + + EXPECT_CALL(sched, registered(_, _, _)); + + Future<vector<Offer>> offers; + EXPECT_CALL(sched, resourceOffers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(offers); + EXPECT_NE(0u, offers.get().size()); + + Offer offer = offers.get()[0]; + + // Run a task that triggers memory pressure event. We request 1G + // disk because we are going to write a 512 MB file repeatedly. + TaskInfo task = createTask( + offer.slave_id(), + Resources::parse("cpus:1;mem:256;disk:1024").get(), + "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); + + Future<TaskStatus> status; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&status)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + driver.launchTasks(offers.get()[0].id(), {task}); + + AWAIT_READY(status); + EXPECT_EQ(task.task_id(), status.get().task_id()); + EXPECT_EQ(TASK_RUNNING, status.get().state()); + + // We restart the slave to let it recover. + Stop(slave.get()); + delete containerizer1.get(); + + Future<Nothing> _recover = FUTURE_DISPATCH(_, &Slave::_recover); + + Future<SlaveReregisteredMessage> slaveReregisteredMessage = + FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); + + // Use the same flags. + Try<MesosContainerizer*> containerizer2 = + MesosContainerizer::create(flags, true, &fetcher); + + ASSERT_SOME(containerizer2); + + slave = StartSlave(containerizer2.get(), flags); + ASSERT_SOME(slave); + + Clock::pause(); + + AWAIT_READY(_recover); + + // Wait for slave to schedule reregister timeout. + Clock::settle(); + + // Ensure the slave considers itself recovered. + Clock::advance(slave::EXECUTOR_REREGISTER_TIMEOUT); + + Clock::resume(); + + // Wait for the slave to re-register. + AWAIT_READY(slaveReregisteredMessage); + + Future<hashset<ContainerID>> containers = containerizer2.get()->containers(); + AWAIT_READY(containers); + ASSERT_EQ(1u, containers.get().size()); + + ContainerID containerId = *(containers.get().begin()); + + Duration waited = Duration::zero(); + do { + Future<ResourceStatistics> usage = containerizer2.get()->usage(containerId); + AWAIT_READY(usage); + + if (usage.get().mem_low_pressure_counter() > 0) { + EXPECT_GE(usage.get().mem_low_pressure_counter(), + usage.get().mem_medium_pressure_counter()); + EXPECT_GE(usage.get().mem_medium_pressure_counter(), + usage.get().mem_critical_pressure_counter()); + break; + } + + os::sleep(Milliseconds(100)); + waited += Milliseconds(100); + } while (waited < Seconds(5)); + + EXPECT_LE(waited, Seconds(5)); + + driver.stop(); + driver.join(); + + Shutdown(); + delete containerizer2.get(); +} + +} // namespace tests { +} // namespace internal { +} // namespace mesos {
