Added test for CNI port-mapper plugin. Review: https://reviews.apache.org/r/53264/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2467fa49 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2467fa49 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2467fa49 Branch: refs/heads/master Commit: 2467fa4977f45f0350900b28b61f5dfb4ab2ae70 Parents: e7f756d Author: Avinash sridharan <[email protected]> Authored: Sun Mar 19 09:09:17 2017 -0700 Committer: Jie Yu <[email protected]> Committed: Sun Mar 19 09:09:17 2017 -0700 ---------------------------------------------------------------------- src/tests/containerizer/cni_isolator_tests.cpp | 273 +++++++++++++++++--- 1 file changed, 242 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/2467fa49/src/tests/containerizer/cni_isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/cni_isolator_tests.cpp b/src/tests/containerizer/cni_isolator_tests.cpp index 393c3e5..d30f251 100644 --- a/src/tests/containerizer/cni_isolator_tests.cpp +++ b/src/tests/containerizer/cni_isolator_tests.cpp @@ -18,8 +18,6 @@ #include <process/clock.hpp> -#include <stout/ip.hpp> - #include "slave/containerizer/fetcher.hpp" #include "slave/containerizer/mesos/containerizer.hpp" #include "slave/containerizer/mesos/isolators/network/cni/paths.hpp" @@ -55,6 +53,11 @@ namespace mesos { namespace internal { namespace tests { +constexpr char MESOS_CNI_PORT_MAPPER_NETWORK[] = "__MESOS_TEST__portMapper"; +constexpr char MESOS_MOCK_CNI_CONFIG[] = "mockConfig"; +constexpr char MESOS_TEST_PORT_MAPPER_CHAIN[] = "MESOS-TEST-PORT-MAPPER-CHAIN"; + + TEST(CniSpecTest, GenerateResolverConfig) { spec::DNS dns; @@ -99,7 +102,8 @@ public: cniPluginDir = path::join(sandbox.get(), "plugins"); cniConfigDir = path::join(sandbox.get(), "configs"); - Result<net::IPNetwork> hostIPNetwork = findHostNetwork(); + Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP(); + ASSERT_SOME(hostIPNetwork); // Get the first external name server. @@ -145,7 +149,7 @@ public: ASSERT_SOME(os::mkdir(cniConfigDir)); result = os::write( - path::join(cniConfigDir, "mockConfig"), + path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG), R"~( { "name": "__MESOS_TEST__", @@ -155,31 +159,6 @@ public: ASSERT_SOME(result); } - Try<net::IPNetwork> findHostNetwork() - { - Try<set<string>> links = net::links(); - - if (links.isError()) { - return Error("Failed to enumerate network interfaces: " + links.error()); - } - - Result<net::IPNetwork> hostIPNetwork = None(); - foreach (const string& link, links.get()) { - hostIPNetwork = net::IPNetwork::fromLinkDevice(link, AF_INET); - - if (hostIPNetwork.isError()) { - return Error("Failed to get address of " + link + ": " + links.error()); - } - - if (hostIPNetwork.isSome() && - (hostIPNetwork.get() != net::IPNetwork::LOOPBACK_V4())) { - return hostIPNetwork.get(); - } - } - - return Error("Failed to find host network address"); - } - // Generate the mock CNI plugin based on the given script. Try<Nothing> setupMockPlugin(const string& pluginScript) { @@ -212,6 +191,78 @@ public: }; +class CniIsolatorPortMapperTest : public CniIsolatorTest +{ +public: + virtual void SetUp() + { + CniIsolatorTest::SetUp(); + + Try<string> mockConfig = os::read( + path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG)); + + ASSERT_SOME(mockConfig); + + // Create a CNI configuration to be used with the port-mapper plugin. + Try<string> portMapperConfig = strings::format(R"~( + { + "name": "%s", + "type": "mesos-cni-port-mapper", + "chain": "%s", + "delegate": %s + } + )~", + MESOS_CNI_PORT_MAPPER_NETWORK, + MESOS_TEST_PORT_MAPPER_CHAIN, + mockConfig.get()); + + ASSERT_SOME(portMapperConfig); + + Try<Nothing> write = os::write( + path::join(cniConfigDir, "mockPortMapperConfig"), + portMapperConfig.get()); + + ASSERT_SOME(write); + } + + virtual void TearDown() + { + // This is a best effort cleanup of the + // `MESOS_TEST_PORT_MAPPER_CHAIN`. We shouldn't fail and bail on + // rest of the `TearDown` if we are not able to clean up the + // chain. + string script = strings::format( + R"~( + #!/bin/sh + set -x + + iptables -w -t nat --list %s + + if [ $? -eq 0 ]; then + iptables -w -t nat -D OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j %s + iptables -w -t nat -D PREROUTING -m addrtype --dst-type LOCAL -j %s + iptables -w -t nat -F %s + iptables -w -t nat -X %s + fi)~", + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN), + stringify(MESOS_TEST_PORT_MAPPER_CHAIN)).get(); + + Try<string> result = os::shell(script); + if (result.isError()) { + LOG(ERROR) << "Unable to cleanup chain " + << stringify(MESOS_TEST_PORT_MAPPER_CHAIN) + << ": " << result.error(); + } + + CniIsolatorTest::TearDown(); + } +}; + + // This test verifies that a container is created and joins a mock CNI // network, and a command task is executed in the container successfully. TEST_F(CniIsolatorTest, ROOT_INTERNET_CURL_LaunchCommandTask) @@ -1000,11 +1051,171 @@ TEST_F(CniIsolatorTest, ROOT_OverrideHostname) } +TEST_F(CniIsolatorPortMapperTest, ROOT_INETERNET_CURL_PortMapper) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + slave::Flags flags = CreateSlaveFlags(); + flags.isolation = "docker/runtime,filesystem/linux"; + flags.image_providers = "docker"; + flags.docker_store_dir = path::join(sandbox.get(), "store"); + + // Augment the CNI plugins search path so that the `network/cni` + // isolator can find the port-mapper CNI plugin. + flags.network_cni_plugins_dir = cniPluginDir + ":" + getLauncherDir(); + + flags.network_cni_config_dir = cniConfigDir; + + // Need to increase the registration timeout to give time for + // downloading and provisioning the "nginx:alpine" image. + flags.executor_registration_timeout = Minutes(5); + + Owned<MasterDetector> detector = master.get()->createDetector(); + + Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); + ASSERT_SOME(slave); + + MockScheduler sched; + + MesosSchedulerDriver driver( + &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); + + EXPECT_CALL(sched, registered(&driver, _, _)); + + Future<vector<Offer>> offers; + EXPECT_CALL(sched, resourceOffers(&driver, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(offers); + ASSERT_EQ(1u, offers->size()); + + const Offer& offer = offers.get()[0]; + + Resources resources(offers.get()[0].resources()); + + // Make sure we have a `ports` resource. + ASSERT_SOME(resources.ports()); + ASSERT_LE(1u, resources.ports()->range().size()); + + // Select a random port from the offer. + std::srand(std::time(0)); + + Value::Range ports = resources.ports()->range(0); + + uint16_t hostPort = + ports.begin() + std::rand() % (ports.end() - ports.begin() + 1); + + CommandInfo command; + command.set_shell(false); + + TaskInfo task = createTask( + offer.slave_id(), + Resources::parse( + "cpus:1;mem:128;" + "ports:[" + stringify(hostPort) + "," + stringify(hostPort) + "]") + .get(), + command); + + ContainerInfo container = createContainerInfo("nginx:alpine"); + + // Make sure the container joins the test CNI port-mapper network. + NetworkInfo* networkInfo = container.add_network_infos(); + networkInfo->set_name(MESOS_CNI_PORT_MAPPER_NETWORK); + + NetworkInfo::PortMapping* portMapping = networkInfo->add_port_mappings(); + portMapping->set_container_port(80); + portMapping->set_host_port(hostPort); + + // Set the container for the task. + task.mutable_container()->CopyFrom(container); + + Future<TaskStatus> statusRunning; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusRunning)); + + driver.launchTasks(offer.id(), {task}); + + AWAIT_READY_FOR(statusRunning, Seconds(300)); + EXPECT_EQ(task.task_id(), statusRunning->task_id()); + EXPECT_EQ(TASK_RUNNING, statusRunning->state()); + ASSERT_TRUE(statusRunning->has_container_status()); + + ContainerID containerId = statusRunning->container_status().container_id(); + ASSERT_EQ(1u, statusRunning->container_status().network_infos().size()); + + // Try connecting to the nginx server on port 80 through a + // non-loopback IP address on `hostPort`. + Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP(); + ASSERT_SOME(hostIPNetwork); + + // `TASK_RUNNING` does not guarantee that the service is running. + // Hence, we need to re-try the service multiple times. + Duration waited = Duration::zero(); + do { + Try<string> connect = os::shell( + "curl -I http://"; + stringify(hostIPNetwork->address()) + + ":" + stringify(hostPort)); + + if (connect.isSome()) { + LOG(INFO) << "Connection to nginx successful: " << connect.get(); + break; + } + + os::sleep(Milliseconds(100)); + waited += Milliseconds(100); + } while (waited < Seconds(10)); + + EXPECT_LE(waited, Seconds(5)); + + // Kill the task. + Future<TaskStatus> statusKilled; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusKilled)); + + // Wait for the executor to exit. We are using 'gc.schedule' as a proxy event + // to monitor the exit of the executor. + Future<Nothing> gcSchedule = FUTURE_DISPATCH( + _, &slave::GarbageCollectorProcess::schedule); + + driver.killTask(task.task_id()); + + AWAIT_READY(statusKilled); + + // The executor would issue a SIGTERM to the container, followed by + // a SIGKILL (in case the container ignores the SIGTERM). The + // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on + // receiving a SIGTERM making the executor send a `TASK_FINISHED` + // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED` + // instead of `TASK_KILLED`. + EXPECT_EQ(TASK_FINISHED, statusKilled.get().state()); + + AWAIT_READY(gcSchedule); + + // Make sure the iptables chain `MESOS-TEST-PORT-MAPPER-CHAIN` + // doesn't have any iptable rules once the task is killed. The only + // rule that should exist in this chain is the + // `-N MESOS-TEST-PORT-MAPPER-CHAIN` rule. + Try<string> rules = os::shell( + "iptables -w -t nat -S " + + stringify(MESOS_TEST_PORT_MAPPER_CHAIN) + "| wc -l"); + + ASSERT_SOME(rules); + ASSERT_EQ("1", strings::trim(rules.get())); + + driver.stop(); + driver.join(); +} + + // This test checks that a CNI DNS configuration ends up generating // the right settings in /etc/resolv.conf. TEST_F(CniIsolatorTest, ROOT_VerifyResolverConfig) { - Try<net::IPNetwork> hostIPNetwork = findHostNetwork(); + Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP(); ASSERT_SOME(hostIPNetwork); Try<string> mockPlugin = strings::format( @@ -1122,7 +1333,7 @@ TEST_F(CniIsolatorTest, ROOT_VerifyResolverConfig) // that glibc accepts by using it to ping a host. TEST_F(CniIsolatorTest, ROOT_INTERNET_VerifyResolverConfig) { - Try<net::IPNetwork> hostIPNetwork = findHostNetwork(); + Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP(); ASSERT_SOME(hostIPNetwork); // Note: We set a dummy nameserver IP address followed by the
