Repository: mesos Updated Branches: refs/heads/master 9b5895606 -> 1453a4775
Enabled bridge network for Docker Containerizer. Review: https://reviews.apache.org/r/25270 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1453a477 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1453a477 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1453a477 Branch: refs/heads/master Commit: 1453a477511c8f6f22ff16e3dd13d0532e019c5b Parents: 9b58956 Author: Timothy Chen <[email protected]> Authored: Tue Sep 16 18:29:36 2014 -0700 Committer: Vinod Kone <[email protected]> Committed: Tue Sep 16 18:29:37 2014 -0700 ---------------------------------------------------------------------- include/mesos/mesos.proto | 17 ++ src/docker/docker.cpp | 61 ++++++- src/slave/slave.cpp | 3 +- src/tests/docker_containerizer_tests.cpp | 246 +++++++++++++++++++++++++- src/tests/docker_tests.cpp | 65 ++++++- 5 files changed, 380 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/include/mesos/mesos.proto ---------------------------------------------------------------------- diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto index dea51f9..be45494 100644 --- a/include/mesos/mesos.proto +++ b/include/mesos/mesos.proto @@ -855,6 +855,23 @@ message ContainerInfo { message DockerInfo { // The docker image that is going to be passed to the registry. required string image = 1; + + // Network options. + enum Network { + HOST = 1; + BRIDGE = 2; + } + + optional Network network = 2 [default = HOST]; + + message PortMapping { + required uint32 host_port = 1; + required uint32 container_port = 2; + // Protocol to expose as (ie: tcp, udp). + optional string protocol = 3; + } + + repeated PortMapping port_mappings = 3; } required Type type = 1; http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/docker/docker.cpp ---------------------------------------------------------------------- diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp index af51ac9..a8becb8 100644 --- a/src/docker/docker.cpp +++ b/src/docker/docker.cpp @@ -253,7 +253,7 @@ Future<Nothing> Docker::run( argv.push_back("-d"); if (resources.isSome()) { - // TODO(yifan): Support other resources (e.g. disk, ports). + // TODO(yifan): Support other resources (e.g. disk). Option<double> cpus = resources.get().cpus(); if (cpus.isSome()) { uint64_t cpuShare = @@ -311,14 +311,59 @@ Future<Nothing> Docker::run( const string& image = dockerInfo.image(); - // TODO(tnachen): Support more network options other than host - // networking that docker provides (ie: BRIDGE). We currently - // require host networking since if the docker container is - // expected to be an executor it needs to be able to communicate - // with the slave by the slave's PID. There can be more future work - // to allow a bridge to connect but this is not yet implemented. argv.push_back("--net"); - argv.push_back("host"); + string network; + switch (dockerInfo.network()) { + case ContainerInfo::DockerInfo::HOST: network = "host"; break; + case ContainerInfo::DockerInfo::BRIDGE: network = "bridge"; break; + default: return Failure("Unsupported Network mode: " + + stringify(dockerInfo.network())); + } + + argv.push_back(network); + + if (dockerInfo.port_mappings().size() > 0) { + if (network != "bridge") { + return Failure("Port mappings are only supported for bridge network"); + } + + if (!resources.isSome()) { + return Failure("Port mappings require resources"); + } + + Option<Value::Ranges> portRanges = resources.get().ports(); + + if (!portRanges.isSome()) { + return Failure("Port mappings require port resources"); + } + + foreach (const ContainerInfo::DockerInfo::PortMapping& mapping, + dockerInfo.port_mappings()) { + bool found = false; + foreach (const Value::Range& range, portRanges.get().range()) { + if (mapping.host_port() >= range.begin() && + mapping.host_port() <= range.end()) { + found = true; + break; + } + } + + if (!found) { + return Failure("Port [" + stringify(mapping.host_port()) + "] not " + + "included in resources"); + } + + string portMapping = stringify(mapping.host_port()) + ":" + + stringify(mapping.container_port()); + + if (mapping.has_protocol()) { + portMapping += "/" + strings::lower(mapping.protocol()); + } + + argv.push_back("-p"); + argv.push_back(portMapping); + } + } argv.push_back("--name"); argv.push_back(name); http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/slave/slave.cpp ---------------------------------------------------------------------- diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp index 1b3dc73..28eb028 100644 --- a/src/slave/slave.cpp +++ b/src/slave/slave.cpp @@ -1756,7 +1756,8 @@ void Slave::registerExecutor( const ExecutorID& executorId) { LOG(INFO) << "Got registration for executor '" << executorId - << "' of framework " << frameworkId; + << "' of framework " << frameworkId << " from " + << stringify(from); CHECK(state == RECOVERING || state == DISCONNECTED || state == RUNNING || state == TERMINATING) http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/tests/docker_containerizer_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp index 8654f9c..1981f49 100644 --- a/src/tests/docker_containerizer_tests.cpp +++ b/src/tests/docker_containerizer_tests.cpp @@ -240,7 +240,122 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor) containerInfo.set_type(ContainerInfo::DOCKER); ContainerInfo::DockerInfo dockerInfo; - dockerInfo.set_image("mesosphere/test-executor"); + dockerInfo.set_image("tnachen/test-executor"); + + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + executorInfo.mutable_container()->CopyFrom(containerInfo); + + task.mutable_executor()->CopyFrom(executorInfo); + + vector<TaskInfo> tasks; + tasks.push_back(task); + + Future<ContainerID> containerId; + EXPECT_CALL(dockerContainerizer, launch(_, _, _, _, _, _, _)) + .WillOnce(DoAll(FutureArg<0>(&containerId), + Invoke(&dockerContainerizer, + &MockDockerContainerizer::_launchExecutor))); + + Future<TaskStatus> statusRunning; + Future<TaskStatus> statusFinished; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusRunning)) + .WillOnce(FutureArg<1>(&statusFinished)); + + driver.launchTasks(offers.get()[0].id(), tasks); + + AWAIT_READY_FOR(containerId, Seconds(60)); + AWAIT_READY_FOR(statusRunning, Seconds(60)); + EXPECT_EQ(TASK_RUNNING, statusRunning.get().state()); + AWAIT_READY_FOR(statusFinished, Seconds(60)); + EXPECT_EQ(TASK_FINISHED, statusFinished.get().state()); + + Future<list<Docker::Container> > containers = + docker.ps(true, slave::DOCKER_NAME_PREFIX); + + AWAIT_READY(containers); + + ASSERT_TRUE(exists(containers.get(), containerId.get())); + + Future<containerizer::Termination> termination = + dockerContainerizer.wait(containerId.get()); + + driver.stop(); + driver.join(); + + AWAIT_READY(termination); + + containers = docker.ps(true, slave::DOCKER_NAME_PREFIX); + AWAIT_READY(containers); + + ASSERT_FALSE(exists(containers.get(), containerId.get())); + + Shutdown(); +} + + +// This test verifies that a custom executor can be launched and +// registered with the slave with docker bridge network enabled. +// We're assuming that the custom executor is registering it's public +// ip instead of 0.0.0.0 or equivelent to the slave as that's the +// default behavior for libprocess. +TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged) +{ + Try<PID<Master> > master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + + Docker docker = Docker::create(tests::flags.docker, false).get(); + + MockDockerContainerizer dockerContainerizer(flags, docker); + + Try<PID<Slave> > slave = StartSlave(&dockerContainerizer, flags); + ASSERT_SOME(slave); + + MockScheduler sched; + MesosSchedulerDriver driver( + &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); + + Future<FrameworkID> frameworkId; + EXPECT_CALL(sched, registered(&driver, _, _)) + .WillOnce(FutureArg<1>(&frameworkId)); + + Future<vector<Offer> > offers; + EXPECT_CALL(sched, resourceOffers(&driver, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(frameworkId); + + AWAIT_READY(offers); + EXPECT_NE(0u, offers.get().size()); + + const Offer& offer = offers.get()[0]; + + TaskInfo task; + task.set_name(""); + task.mutable_task_id()->set_value("1"); + task.mutable_slave_id()->CopyFrom(offer.slave_id()); + task.mutable_resources()->CopyFrom(offer.resources()); + + ExecutorInfo executorInfo; + ExecutorID executorId; + executorId.set_value("e1"); + executorInfo.mutable_executor_id()->CopyFrom(executorId); + + CommandInfo command; + command.set_value("test-executor"); + executorInfo.mutable_command()->CopyFrom(command); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("tnachen/test-executor"); + dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE); containerInfo.mutable_docker()->CopyFrom(dockerInfo); executorInfo.mutable_container()->CopyFrom(containerInfo); @@ -1630,3 +1745,132 @@ TEST_F(DockerContainerizerTest, delete dockerContainerizer2; } + + +// This test verifies that port mapping with bridge network is +// exposing the host port to the container port, by sending data +// to the host port and receiving it in the container by listening +// to the mapped container port. +TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping) +{ + Try<PID<Master> > master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + + flags.resources = "cpus:1;mem:1024;ports:[10000-10000]"; + + Docker docker = Docker::create(tests::flags.docker, false).get(); + + MockDockerContainerizer dockerContainerizer(flags, docker); + + Try<PID<Slave> > slave = StartSlave(&dockerContainerizer, flags); + ASSERT_SOME(slave); + + MockScheduler sched; + MesosSchedulerDriver driver( + &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); + + Future<FrameworkID> frameworkId; + EXPECT_CALL(sched, registered(&driver, _, _)) + .WillOnce(FutureArg<1>(&frameworkId)); + + Future<vector<Offer> > offers; + EXPECT_CALL(sched, resourceOffers(&driver, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(frameworkId); + + AWAIT_READY(offers); + EXPECT_NE(0u, offers.get().size()); + + const Offer& offer = offers.get()[0]; + + TaskInfo task; + task.set_name(""); + task.mutable_task_id()->set_value("1"); + task.mutable_slave_id()->CopyFrom(offer.slave_id()); + task.mutable_resources()->CopyFrom(offer.resources()); + + CommandInfo command; + command.set_shell(false); + command.set_value("nc"); + command.add_arguments("-l"); + command.add_arguments("-p"); + command.add_arguments("1000"); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE); + + ContainerInfo::DockerInfo::PortMapping portMapping; + portMapping.set_host_port(10000); + portMapping.set_container_port(1000); + + dockerInfo.add_port_mappings()->CopyFrom(portMapping); + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + task.mutable_command()->CopyFrom(command); + task.mutable_container()->CopyFrom(containerInfo); + + vector<TaskInfo> tasks; + tasks.push_back(task); + + Future<ContainerID> containerId; + Future<string> directory; + EXPECT_CALL(dockerContainerizer, launch(_, _, _, _, _, _, _, _)) + .WillOnce(DoAll(FutureArg<0>(&containerId), + FutureArg<3>(&directory), + Invoke(&dockerContainerizer, + &MockDockerContainerizer::_launch))); + + Future<TaskStatus> statusRunning; + Future<TaskStatus> statusFinished; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusRunning)) + .WillOnce(FutureArg<1>(&statusFinished)) + .WillRepeatedly(DoDefault()); + + driver.launchTasks(offers.get()[0].id(), tasks); + + AWAIT_READY_FOR(containerId, Seconds(60)); + AWAIT_READY(directory); + AWAIT_READY_FOR(statusRunning, Seconds(60)); + EXPECT_EQ(TASK_RUNNING, statusRunning.get().state()); + + string uuid = UUID::random().toString(); + + // Write uuid to docker mapped host port. + Try<process::Subprocess> s = process::subprocess( + "echo " + uuid + " | nc localhost 10000"); + + ASSERT_SOME(s); + AWAIT_READY(s.get().status()); + + string container = slave::DOCKER_NAME_PREFIX + stringify(containerId.get()); + + AWAIT_READY_FOR(docker.kill(container, false), Seconds(30)); + + AWAIT_READY_FOR(statusFinished, Seconds(60)); + EXPECT_EQ(TASK_FINISHED, statusFinished.get().state()); + + // Now check that the proper output is in stdout. + Try<string> read = os::read(path::join(directory.get(), "stdout")); + + ASSERT_SOME(read); + + // We expect the uuid that is sent to host port to be written + // to stdout by the docker container running nc -l. + EXPECT_TRUE(strings::contains(read.get(), uuid)); + + driver.stop(); + driver.join(); + + Shutdown(); +} http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/tests/docker_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/docker_tests.cpp b/src/tests/docker_tests.cpp index 826a8c1..e6c228a 100644 --- a/src/tests/docker_tests.cpp +++ b/src/tests/docker_tests.cpp @@ -214,8 +214,69 @@ TEST(DockerTest, ROOT_DOCKER_CheckCommandWithShell) commandInfo, "testContainer", "dir", - "/mnt/mesos/sandbox", - None()); + "/mnt/mesos/sandbox"); ASSERT_TRUE(run.isFailed()); } + + +TEST(DockerTest, ROOT_DOCKER_CheckPortResource) +{ + string containerName = "mesos-docker-port-resource-test"; + Docker docker = Docker::create(tests::flags.docker, false).get(); + + // Make sure the container is removed. + Future<Nothing> remove = docker.rm(containerName, true); + + ASSERT_TRUE(process::internal::await(remove, Seconds(10))); + + ContainerInfo containerInfo; + containerInfo.set_type(ContainerInfo::DOCKER); + + ContainerInfo::DockerInfo dockerInfo; + dockerInfo.set_image("busybox"); + dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE); + + ContainerInfo::DockerInfo::PortMapping portMapping; + portMapping.set_host_port(10000); + portMapping.set_container_port(80); + + dockerInfo.add_port_mappings()->CopyFrom(portMapping); + containerInfo.mutable_docker()->CopyFrom(dockerInfo); + + CommandInfo commandInfo; + commandInfo.set_shell(false); + commandInfo.set_value("true"); + + Resources resources = + Resources::parse("ports:[9998-9999];ports:[10001-11000]").get(); + + Future<Nothing> run = docker.run( + containerInfo, + commandInfo, + containerName, + "dir", + "/mnt/mesos/sandbox", + resources); + + // Port should be out side of the provided ranges. + AWAIT_EXPECTED_FAILED(run); + + resources = Resources::parse("ports:[9998-9999];ports:[10000-11000]").get(); + + Try<string> directory = environment->mkdtemp(); + CHECK_SOME(directory) << "Failed to create temporary directory"; + + run = docker.run( + containerInfo, + commandInfo, + containerName, + directory.get(), + "/mnt/mesos/sandbox", + resources); + + AWAIT_READY(run); + + Future<Nothing> status = docker.rm(containerName, true); + AWAIT_READY(status); +}
