Adjusted the ordering in CNI tests file.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5324b599 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5324b599 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5324b599 Branch: refs/heads/master Commit: 5324b5999145116d95b3c324f443cf0a70c78356 Parents: 48e79f5 Author: Jie Yu <[email protected]> Authored: Sun Mar 19 09:48:47 2017 -0700 Committer: Jie Yu <[email protected]> Committed: Sun Mar 19 09:48:47 2017 -0700 ---------------------------------------------------------------------- src/tests/containerizer/cni_isolator_tests.cpp | 464 ++++++++++---------- 1 file changed, 232 insertions(+), 232 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/5324b599/src/tests/containerizer/cni_isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/cni_isolator_tests.cpp b/src/tests/containerizer/cni_isolator_tests.cpp index 4940fc8..2ad7515 100644 --- a/src/tests/containerizer/cni_isolator_tests.cpp +++ b/src/tests/containerizer/cni_isolator_tests.cpp @@ -191,78 +191,6 @@ public: }; -class CniIsolatorPortMapperTest : public CniIsolatorTest -{ -public: - virtual void SetUp() - { - CniIsolatorTest::SetUp(); - - Try<string> mockConfig = os::read( - path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG)); - - ASSERT_SOME(mockConfig); - - // Create a CNI configuration to be used with the port-mapper plugin. - Try<string> portMapperConfig = strings::format(R"~( - { - "name": "%s", - "type": "mesos-cni-port-mapper", - "chain": "%s", - "delegate": %s - } - )~", - MESOS_CNI_PORT_MAPPER_NETWORK, - MESOS_TEST_PORT_MAPPER_CHAIN, - mockConfig.get()); - - ASSERT_SOME(portMapperConfig); - - Try<Nothing> write = os::write( - path::join(cniConfigDir, "mockPortMapperConfig"), - portMapperConfig.get()); - - ASSERT_SOME(write); - } - - virtual void TearDown() - { - // This is a best effort cleanup of the - // `MESOS_TEST_PORT_MAPPER_CHAIN`. We shouldn't fail and bail on - // rest of the `TearDown` if we are not able to clean up the - // chain. - string script = strings::format( - R"~( - #!/bin/sh - set -x - - iptables -w -t nat --list %s - - if [ $? -eq 0 ]; then - iptables -w -t nat -D OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j %s - iptables -w -t nat -D PREROUTING -m addrtype --dst-type LOCAL -j %s - iptables -w -t nat -F %s - iptables -w -t nat -X %s - fi)~", - stringify(MESOS_TEST_PORT_MAPPER_CHAIN), - stringify(MESOS_TEST_PORT_MAPPER_CHAIN), - stringify(MESOS_TEST_PORT_MAPPER_CHAIN), - stringify(MESOS_TEST_PORT_MAPPER_CHAIN), - stringify(MESOS_TEST_PORT_MAPPER_CHAIN), - stringify(MESOS_TEST_PORT_MAPPER_CHAIN)).get(); - - Try<string> result = os::shell(script); - if (result.isError()) { - LOG(ERROR) << "Unable to cleanup chain " - << stringify(MESOS_TEST_PORT_MAPPER_CHAIN) - << ": " << result.error(); - } - - CniIsolatorTest::TearDown(); - } -}; - - // This test verifies that a container is created and joins a mock CNI // network, and a command task is executed in the container successfully. TEST_F(CniIsolatorTest, ROOT_INTERNET_CURL_LaunchCommandTask) @@ -1051,166 +979,6 @@ TEST_F(CniIsolatorTest, ROOT_OverrideHostname) } -TEST_F(CniIsolatorPortMapperTest, ROOT_INETERNET_CURL_PortMapper) -{ - Try<Owned<cluster::Master>> master = StartMaster(); - ASSERT_SOME(master); - - slave::Flags flags = CreateSlaveFlags(); - flags.isolation = "docker/runtime,filesystem/linux"; - flags.image_providers = "docker"; - flags.docker_store_dir = path::join(sandbox.get(), "store"); - - // Augment the CNI plugins search path so that the `network/cni` - // isolator can find the port-mapper CNI plugin. - flags.network_cni_plugins_dir = cniPluginDir + ":" + getLauncherDir(); - flags.network_cni_config_dir = cniConfigDir; - - // Need to increase the registration timeout to give time for - // downloading and provisioning the "nginx:alpine" image. - flags.executor_registration_timeout = Minutes(5); - - Owned<MasterDetector> detector = master.get()->createDetector(); - - Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); - ASSERT_SOME(slave); - - MockScheduler sched; - - MesosSchedulerDriver driver( - &sched, - DEFAULT_FRAMEWORK_INFO, - master.get()->pid, - DEFAULT_CREDENTIAL); - - EXPECT_CALL(sched, registered(&driver, _, _)); - - Future<vector<Offer>> offers; - EXPECT_CALL(sched, resourceOffers(&driver, _)) - .WillOnce(FutureArg<1>(&offers)) - .WillRepeatedly(Return()); // Ignore subsequent offers. - - driver.start(); - - AWAIT_READY(offers); - ASSERT_EQ(1u, offers->size()); - - const Offer& offer = offers.get()[0]; - - Resources resources(offers.get()[0].resources()); - - // Make sure we have a `ports` resource. - ASSERT_SOME(resources.ports()); - ASSERT_LE(1u, resources.ports()->range().size()); - - // Select a random port from the offer. - std::srand(std::time(0)); - Value::Range ports = resources.ports()->range(0); - uint16_t hostPort = - ports.begin() + std::rand() % (ports.end() - ports.begin() + 1); - - CommandInfo command; - command.set_shell(false); - - TaskInfo task = createTask( - offer.slave_id(), - Resources::parse( - "cpus:1;mem:128;" - "ports:[" + stringify(hostPort) + "," + stringify(hostPort) + "]") - .get(), - command); - - ContainerInfo container = createContainerInfo("nginx:alpine"); - - // Make sure the container joins the test CNI port-mapper network. - NetworkInfo* networkInfo = container.add_network_infos(); - networkInfo->set_name(MESOS_CNI_PORT_MAPPER_NETWORK); - - NetworkInfo::PortMapping* portMapping = networkInfo->add_port_mappings(); - portMapping->set_container_port(80); - portMapping->set_host_port(hostPort); - - // Set the container for the task. - task.mutable_container()->CopyFrom(container); - - Future<TaskStatus> statusRunning; - EXPECT_CALL(sched, statusUpdate(&driver, _)) - .WillOnce(FutureArg<1>(&statusRunning)); - - driver.launchTasks(offer.id(), {task}); - - AWAIT_READY_FOR(statusRunning, Seconds(300)); - EXPECT_EQ(task.task_id(), statusRunning->task_id()); - EXPECT_EQ(TASK_RUNNING, statusRunning->state()); - ASSERT_TRUE(statusRunning->has_container_status()); - - ContainerID containerId = statusRunning->container_status().container_id(); - ASSERT_EQ(1u, statusRunning->container_status().network_infos().size()); - - // Try connecting to the nginx server on port 80 through a - // non-loopback IP address on `hostPort`. - Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP(); - ASSERT_SOME(hostIPNetwork); - - // `TASK_RUNNING` does not guarantee that the service is running. - // Hence, we need to re-try the service multiple times. - Duration waited = Duration::zero(); - do { - Try<string> connect = os::shell( - "curl -I http://"; + stringify(hostIPNetwork->address()) + - ":" + stringify(hostPort)); - - if (connect.isSome()) { - LOG(INFO) << "Connection to nginx successful: " << connect.get(); - break; - } - - os::sleep(Milliseconds(100)); - waited += Milliseconds(100); - } while (waited < Seconds(10)); - - EXPECT_LE(waited, Seconds(5)); - - // Kill the task. - Future<TaskStatus> statusKilled; - EXPECT_CALL(sched, statusUpdate(&driver, _)) - .WillOnce(FutureArg<1>(&statusKilled)); - - // Wait for the executor to exit. We are using 'gc.schedule' as a - // proxy event to monitor the exit of the executor. - Future<Nothing> gcSchedule = FUTURE_DISPATCH( - _, &slave::GarbageCollectorProcess::schedule); - - driver.killTask(task.task_id()); - - AWAIT_READY(statusKilled); - - // The executor would issue a SIGTERM to the container, followed by - // a SIGKILL (in case the container ignores the SIGTERM). The - // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on - // receiving a SIGTERM making the executor send a `TASK_FINISHED` - // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED` - // instead of `TASK_KILLED`. - EXPECT_EQ(TASK_FINISHED, statusKilled.get().state()); - - AWAIT_READY(gcSchedule); - - // Make sure the iptables chain `MESOS-TEST-PORT-MAPPER-CHAIN` - // doesn't have any iptable rules once the task is killed. The only - // rule that should exist in this chain is the `-N - // MESOS-TEST-PORT-MAPPER-CHAIN` rule. - Try<string> rules = os::shell( - "iptables -w -t nat -S " + - stringify(MESOS_TEST_PORT_MAPPER_CHAIN) + "| wc -l"); - - ASSERT_SOME(rules); - ASSERT_EQ("1", strings::trim(rules.get())); - - driver.stop(); - driver.join(); -} - - // This test checks that a CNI DNS configuration ends up generating // the right settings in /etc/resolv.conf. TEST_F(CniIsolatorTest, ROOT_VerifyResolverConfig) @@ -1445,6 +1213,238 @@ TEST_F(CniIsolatorTest, ROOT_INTERNET_VerifyResolverConfig) driver.join(); } + +class CniIsolatorPortMapperTest : public CniIsolatorTest +{ +public: + virtual void SetUp() + { + CniIsolatorTest::SetUp(); + + Try<string> mockConfig = os::read( + path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG)); + + ASSERT_SOME(mockConfig); + + // Create a CNI configuration to be used with the port-mapper plugin. + Try<string> portMapperConfig = strings::format(R"~( + { + "name": "%s", + "type": "mesos-cni-port-mapper", + "chain": "%s", + "delegate": %s + } + )~", + MESOS_CNI_PORT_MAPPER_NETWORK, + MESOS_TEST_PORT_MAPPER_CHAIN, + mockConfig.get()); + + ASSERT_SOME(portMapperConfig); + + Try<Nothing> write = os::write( + path::join(cniConfigDir, "mockPortMapperConfig"), + portMapperConfig.get()); + + ASSERT_SOME(write); + } + + virtual void TearDown() + { + // This is a best effort cleanup of the + // `MESOS_TEST_PORT_MAPPER_CHAIN`. We shouldn't fail and bail on + // rest of the `TearDown` if we are not able to clean up the + // chain. + string script = strings::format( + R"~( + #!/bin/sh + set -x + + iptables -w -t nat --list %s + + if [ $? -eq 0 ]; then + iptables -w -t nat -D OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j %s + iptables -w -t nat -D PREROUTING -m addrtype --dst-type LOCAL -j %s + iptables -w -t nat -F %s + iptables -w -t nat -X %s + fi)~", + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN)).get(); + + Try<string> result = os::shell(script); + if (result.isError()) { + LOG(ERROR) << "Unable to cleanup chain " + << stringify(MESOS_TEST_PORT_MAPPER_CHAIN) + << ": " << result.error(); + } + + CniIsolatorTest::TearDown(); + } +}; + + +TEST_F(CniIsolatorPortMapperTest, ROOT_INETERNET_CURL_PortMapper) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "docker/runtime,filesystem/linux"; + flags.image_providers = "docker"; + flags.docker_store_dir = path::join(sandbox.get(), "store"); + + // Augment the CNI plugins search path so that the `network/cni` + // isolator can find the port-mapper CNI plugin. + flags.network_cni_plugins_dir = cniPluginDir + ":" + getLauncherDir(); + flags.network_cni_config_dir = cniConfigDir; + + // Need to increase the registration timeout to give time for + // downloading and provisioning the "nginx:alpine" image. + flags.executor_registration_timeout = Minutes(5); + + Owned<MasterDetector> detector = master.get()->createDetector(); + + Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); + ASSERT_SOME(slave); + + MockScheduler sched; + + MesosSchedulerDriver driver( + &sched, + DEFAULT_FRAMEWORK_INFO, + master.get()->pid, + DEFAULT_CREDENTIAL); + + EXPECT_CALL(sched, registered(&driver, _, _)); + + Future<vector<Offer>> offers; + EXPECT_CALL(sched, resourceOffers(&driver, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(offers); + ASSERT_EQ(1u, offers->size()); + + const Offer& offer = offers.get()[0]; + + Resources resources(offers.get()[0].resources()); + + // Make sure we have a `ports` resource. + ASSERT_SOME(resources.ports()); + ASSERT_LE(1u, resources.ports()->range().size()); + + // Select a random port from the offer. + std::srand(std::time(0)); + Value::Range ports = resources.ports()->range(0); + uint16_t hostPort = + ports.begin() + std::rand() % (ports.end() - ports.begin() + 1); + + CommandInfo command; + command.set_shell(false); + + TaskInfo task = createTask( + offer.slave_id(), + Resources::parse( + "cpus:1;mem:128;" + "ports:[" + stringify(hostPort) + "," + stringify(hostPort) + "]") + .get(), + command); + + ContainerInfo container = createContainerInfo("nginx:alpine"); + + // Make sure the container joins the test CNI port-mapper network. + NetworkInfo* networkInfo = container.add_network_infos(); + networkInfo->set_name(MESOS_CNI_PORT_MAPPER_NETWORK); + + NetworkInfo::PortMapping* portMapping = networkInfo->add_port_mappings(); + portMapping->set_container_port(80); + portMapping->set_host_port(hostPort); + + // Set the container for the task. + task.mutable_container()->CopyFrom(container); + + Future<TaskStatus> statusRunning; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusRunning)); + + driver.launchTasks(offer.id(), {task}); + + AWAIT_READY_FOR(statusRunning, Seconds(300)); + EXPECT_EQ(task.task_id(), statusRunning->task_id()); + EXPECT_EQ(TASK_RUNNING, statusRunning->state()); + ASSERT_TRUE(statusRunning->has_container_status()); + + ContainerID containerId = statusRunning->container_status().container_id(); + ASSERT_EQ(1u, statusRunning->container_status().network_infos().size()); + + // Try connecting to the nginx server on port 80 through a + // non-loopback IP address on `hostPort`. + Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP(); + ASSERT_SOME(hostIPNetwork); + + // `TASK_RUNNING` does not guarantee that the service is running. + // Hence, we need to re-try the service multiple times. + Duration waited = Duration::zero(); + do { + Try<string> connect = os::shell( + "curl -I http://"; + stringify(hostIPNetwork->address()) + + ":" + stringify(hostPort)); + + if (connect.isSome()) { + LOG(INFO) << "Connection to nginx successful: " << connect.get(); + break; + } + + os::sleep(Milliseconds(100)); + waited += Milliseconds(100); + } while (waited < Seconds(10)); + + EXPECT_LE(waited, Seconds(5)); + + // Kill the task. + Future<TaskStatus> statusKilled; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusKilled)); + + // Wait for the executor to exit. We are using 'gc.schedule' as a + // proxy event to monitor the exit of the executor. + Future<Nothing> gcSchedule = FUTURE_DISPATCH( + _, &slave::GarbageCollectorProcess::schedule); + + driver.killTask(task.task_id()); + + AWAIT_READY(statusKilled); + + // The executor would issue a SIGTERM to the container, followed by + // a SIGKILL (in case the container ignores the SIGTERM). The + // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on + // receiving a SIGTERM making the executor send a `TASK_FINISHED` + // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED` + // instead of `TASK_KILLED`. + EXPECT_EQ(TASK_FINISHED, statusKilled.get().state()); + + AWAIT_READY(gcSchedule); + + // Make sure the iptables chain `MESOS-TEST-PORT-MAPPER-CHAIN` + // doesn't have any iptable rules once the task is killed. The only + // rule that should exist in this chain is the `-N + // MESOS-TEST-PORT-MAPPER-CHAIN` rule. + Try<string> rules = os::shell( + "iptables -w -t nat -S " + + stringify(MESOS_TEST_PORT_MAPPER_CHAIN) + "| wc -l"); + + ASSERT_SOME(rules); + ASSERT_EQ("1", strings::trim(rules.get())); + + driver.stop(); + driver.join(); +} + } // namespace tests { } // namespace internal { } // namespace mesos {
