Made the capabilities isolator work with nested containers. Make the capabilities isolator support nesting. Without this patch capabilities sets for tasks launched by the DefaultExecutor are silently ignored.
Review: https://reviews.apache.org/r/61394/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/50a553dd Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/50a553dd Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/50a553dd Branch: refs/heads/master Commit: 50a553dd10a2adbf653d5ad751723294a14720a8 Parents: cbb2262 Author: Gastón Kleiman <[email protected]> Authored: Fri Aug 4 15:51:27 2017 -0700 Committer: Jie Yu <[email protected]> Committed: Fri Aug 4 15:51:27 2017 -0700 ---------------------------------------------------------------------- .../mesos/isolators/linux/capabilities.cpp | 6 + .../mesos/isolators/linux/capabilities.hpp | 2 + .../linux_capabilities_isolator_tests.cpp | 147 +++++++++++++++++++ 3 files changed, 155 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/50a553dd/src/slave/containerizer/mesos/isolators/linux/capabilities.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/isolators/linux/capabilities.cpp b/src/slave/containerizer/mesos/isolators/linux/capabilities.cpp index f85a84d..21d851e 100644 --- a/src/slave/containerizer/mesos/isolators/linux/capabilities.cpp +++ b/src/slave/containerizer/mesos/isolators/linux/capabilities.cpp @@ -74,6 +74,12 @@ Try<Isolator*> LinuxCapabilitiesIsolatorProcess::create(const Flags& flags) } +bool LinuxCapabilitiesIsolatorProcess::supportsNesting() +{ + return true; +} + + Future<Option<ContainerLaunchInfo>> LinuxCapabilitiesIsolatorProcess::prepare( const ContainerID& containerId, const ContainerConfig& containerConfig) http://git-wip-us.apache.org/repos/asf/mesos/blob/50a553dd/src/slave/containerizer/mesos/isolators/linux/capabilities.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/isolators/linux/capabilities.hpp b/src/slave/containerizer/mesos/isolators/linux/capabilities.hpp index c3afe20..b9862a2 100644 --- a/src/slave/containerizer/mesos/isolators/linux/capabilities.hpp +++ b/src/slave/containerizer/mesos/isolators/linux/capabilities.hpp @@ -32,6 +32,8 @@ class LinuxCapabilitiesIsolatorProcess : public MesosIsolatorProcess public: static Try<mesos::slave::Isolator*> create(const Flags& flags); + virtual bool supportsNesting(); + virtual process::Future<Option<mesos::slave::ContainerLaunchInfo>> prepare( const ContainerID& containerId, const mesos::slave::ContainerConfig& containerConfig); http://git-wip-us.apache.org/repos/asf/mesos/blob/50a553dd/src/tests/containerizer/linux_capabilities_isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/linux_capabilities_isolator_tests.cpp b/src/tests/containerizer/linux_capabilities_isolator_tests.cpp index 8050876..6d95d60 100644 --- a/src/tests/containerizer/linux_capabilities_isolator_tests.cpp +++ b/src/tests/containerizer/linux_capabilities_isolator_tests.cpp @@ -41,6 +41,8 @@ #include <mesos/master/detector.hpp> +#include <mesos/resources.hpp> + #include "linux/capabilities.hpp" #include "master/detector/standalone.hpp" @@ -328,6 +330,151 @@ TEST_P(LinuxCapabilitiesIsolatorTest, ROOT_Ping) } +// Parameterized test confirming the behavior of the capabilities +// isolator with nested containers. We here use the fact has `ping` +// has `NET_RAW` and `NET_ADMIN` in its file capabilities. This test +// should be instantiated with above `TestParam` struct. +TEST_P(LinuxCapabilitiesIsolatorTest, ROOT_NestedPing) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "linux/capabilities"; + flags.effective_capabilities = param.operator_effective; + flags.bounding_capabilities = param.operator_bounding; + +#ifndef USE_SSL_SOCKET + // Disable operator API authentication for the default executor. Executor + // authentication currently has SSL as a dependency, so we cannot require + // executors to authenticate with the agent operator API if Mesos was not + // built with SSL support. + flags.authenticate_http_readwrite = false; +#endif // USE_SSL_SOCKET + + if (param.useImage == TestParam::WITH_IMAGE) { + const string registry = path::join(sandbox.get(), "registry"); + AWAIT_READY(DockerArchive::create(registry, "test_image")); + + flags.docker_registry = registry; + flags.docker_store_dir = path::join(os::getcwd(), "store"); + flags.image_providers = "docker"; + flags.isolation += ",docker/runtime,filesystem/linux"; + } + + Owned<MasterDetector> detector = master.get()->createDetector(); + + Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); + ASSERT_SOME(slave); + + MockScheduler sched; + + MesosSchedulerDriver driver( + &sched, + DEFAULT_FRAMEWORK_INFO, + master.get()->pid, + DEFAULT_CREDENTIAL); + + Future<FrameworkID> frameworkId; + EXPECT_CALL(sched, registered(&driver, _, _)) + .WillOnce(FutureArg<1>(&frameworkId)); + + Future<vector<Offer>> offers; + + EXPECT_CALL(sched, resourceOffers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(frameworkId); + + AWAIT_READY(offers); + ASSERT_NE(0u, offers->size()); + + Resources resources = Resources::parse("cpus:0.1;mem:32;disk:32").get(); + + ExecutorInfo executorInfo; + executorInfo.set_type(ExecutorInfo::DEFAULT); + executorInfo.mutable_executor_id()->CopyFrom(DEFAULT_EXECUTOR_ID); + executorInfo.mutable_framework_id()->CopyFrom(frameworkId.get()); + executorInfo.mutable_resources()->CopyFrom(resources); + + const Offer& offer = offers->front(); + const SlaveID& slaveId = offer.slave_id(); + + // We use 'ping' as the command since it has file capabilities + // (`NET_RAW` and `NET_ADMIN` in permitted set). This allows us to + // test if capabilities are properly set. + CommandInfo command; + command.set_shell(false); + command.set_value("/bin/ping"); + command.add_arguments("ping"); + command.add_arguments("-c"); + command.add_arguments("1"); + command.add_arguments("127.0.0.1"); + + TaskInfo task = createTask(slaveId, resources, command); + + if (param.framework_effective.isSome() || + param.framework_bounding.isSome()) { + ContainerInfo* container = task.mutable_container(); + container->set_type(ContainerInfo::MESOS); + + LinuxInfo* linux = container->mutable_linux_info(); + + if (param.framework_effective.isSome()) { + CapabilityInfo* capabilities = linux->mutable_effective_capabilities(); + capabilities->CopyFrom(param.framework_effective.get()); + } + + if (param.framework_bounding.isSome()) { + CapabilityInfo* capabilities = linux->mutable_bounding_capabilities(); + capabilities->CopyFrom(param.framework_bounding.get()); + } + } + + if (param.useImage == TestParam::WITH_IMAGE) { + ContainerInfo* container = task.mutable_container(); + container->set_type(ContainerInfo::MESOS); + + Image* image = container->mutable_mesos()->mutable_image(); + image->set_type(Image::DOCKER); + image->mutable_docker()->set_name("test_image"); + } + + Queue<TaskStatus> statuses; + EXPECT_CALL(sched, statusUpdate(_, _)) + .WillRepeatedly(PushTaskStatus<1>(&statuses)); + + TaskGroupInfo taskGroup = createTaskGroupInfo({task}); + + driver.acceptOffers({offer.id()}, {LAUNCH_GROUP(executorInfo, taskGroup)}); + + // Wait for the terminal status update. + for (;;) { + Future<TaskStatus> status = statuses.get(); + AWAIT_READY(status); + + TaskState state = status->state(); + if (protobuf::isTerminalState(state)) { + switch (param.result) { + case TestParam::SUCCESS: + EXPECT_EQ(TASK_FINISHED, state) << status->DebugString(); + break; + case TestParam::FAILURE: + EXPECT_EQ(TASK_FAILED, state) << status->DebugString(); + break; + } + break; + } + } + + driver.stop(); + driver.join(); +} + + // TODO(jieyu): We used DAC_READ_SEARCH capability below so that test // results won't be affected even if the executable (e.g., command // executor) to launch is not accessible (e.g., under someone's home
