This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 0a58ecd Added a CNI test for networking statistics.
0a58ecd is described below
commit 0a58ecd86dfd025526c6a2f719df096ec8195c99
Author: Sergey Urbanovich <[email protected]>
AuthorDate: Tue Aug 28 12:15:00 2018 -0700
Added a CNI test for networking statistics.
This is a veth CNI plugin that is written in bash. It creates a veth
virtual network pair, one end of the pair will be moved to container
network namespace.
The veth CNI plugin uses 203.0.113.0/24 subnet, it is reserved for
documentation and examples [rfc5737]. The plugin can allocate up to
128 veth pairs.
Review: https://reviews.apache.org/r/68355/
---
src/tests/containerizer/cni_isolator_tests.cpp | 200 ++++++++++++++++++++++++-
src/tests/environment.cpp | 74 +++++++++
2 files changed, 269 insertions(+), 5 deletions(-)
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp
b/src/tests/containerizer/cni_isolator_tests.cpp
index 63b109b..ed14bb1 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -196,31 +196,110 @@ public:
"type": "mockPlugin"
})~");
+ // This is a veth CNI plugin that is written in bash. It creates a
+ // veth virtual network pair, one end of the pair will be moved to
+ // container network namespace.
+ //
+ // The veth CNI plugin uses 203.0.113.0/24 subnet, it is reserved
+ // for documentation and examples [rfc5737]. The plugin can
+ // allocate up to 128 veth pairs.
+ string vethPlugin =
+ R"~(
+ #!/bin/sh
+ set -e
+ IP_ADDR="203.0.113.%s"
+
+ NETNS="mesos-test-veth-${PPID}"
+ mkdir -p /var/run/netns
+ cleanup() {
+ rm -f /var/run/netns/$NETNS
+ }
+ trap cleanup EXIT
+ ln -sf $CNI_NETNS /var/run/netns/$NETNS
+
+ case $CNI_COMMAND in
+ "ADD"*)
+ for idx in `seq 0 127`; do
+ VETH0="vmesos${idx}"
+ VETH1="vmesos${idx}ns"
+ ip link add name $VETH0 type veth peer name $VETH1 2> /dev/null
|| continue
+ VETH0_IP=$(printf $IP_ADDR $(($idx * 2)))
+ VETH1_IP=$(printf $IP_ADDR $(($idx * 2 + 1)))
+ ip addr add "${VETH0_IP}/31" dev $VETH0
+ ip link set $VETH0 up
+ ip link set $VETH1 netns $NETNS
+ ip netns exec $NETNS ip addr add "${VETH1_IP}/31" dev $VETH1
+ ip netns exec $NETNS ip link set $VETH1 name $CNI_IFNAME up
+ ip netns exec $NETNS ip route add default via $VETH0_IP dev
$CNI_IFNAME
+ break
+ done
+ echo '{'
+ echo ' "cniVersion": "0.2.3",'
+ if [ -z "$VETH1_IP" ]; then
+ echo ' "code": 100,'
+ echo ' "msg": "Bad IP address"'
+ echo '}'
+ exit 100
+ else
+ echo ' "ip4": {'
+ echo ' "ip": "'$VETH1_IP'/31"'
+ echo ' }'
+ echo '}'
+ fi
+ ;;
+ "DEL"*)
+ # $VETH0 on host network namespace will be removed automatically.
+ # If the plugin can't destroy the veth pair, it will be destroyed
+ # with the container network namespace.
+ ip netns exec $NETNS ip link del $CNI_IFNAME || true
+ ;;
+ esac
+ )~";
+
+ ASSERT_SOME(setupPlugin(vethPlugin, "vethPlugin"));
+
+ // Generate the mock CNI config for veth CNI plugin.
+ ASSERT_SOME(os::mkdir(cniConfigDir));
+
+ result = os::write(
+ path::join(cniConfigDir, "vethConfig"),
+ R"~(
+ {
+ "name": "veth",
+ "type": "vethPlugin"
+ })~");
+
ASSERT_SOME(result);
}
// Generate the mock CNI plugin based on the given script.
Try<Nothing> setupMockPlugin(const string& pluginScript)
{
+ return setupPlugin(pluginScript, "mockPlugin");
+ }
+
+ // Generate the CNI plugin based on the given script.
+ Try<Nothing> setupPlugin(const string& pluginScript, const string&
pluginName)
+ {
Try<Nothing> mkdir = os::mkdir(cniPluginDir);
if (mkdir.isError()) {
return Error("Failed to mkdir '" + cniPluginDir + "': " + mkdir.error());
}
- string mockPlugin = path::join(cniPluginDir, "mockPlugin");
+ string pluginPath = path::join(cniPluginDir, pluginName);
- Try<Nothing> write = os::write(mockPlugin, pluginScript);
+ Try<Nothing> write = os::write(pluginPath, pluginScript);
if (write.isError()) {
- return Error("Failed to write '" + mockPlugin + "': " + write.error());
+ return Error("Failed to write '" + pluginPath + "': " + write.error());
}
// Make sure the plugin has execution permission.
Try<Nothing> chmod = os::chmod(
- mockPlugin,
+ pluginPath,
S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
if (chmod.isError()) {
- return Error("Failed to chmod '" + mockPlugin + "': " + chmod.error());
+ return Error("Failed to chmod '" + pluginPath + "': " + chmod.error());
}
return Nothing();
@@ -2346,6 +2425,117 @@ TEST_P(DefaultContainerDNSCniTest,
ROOT_VerifyDefaultDNS)
driver.join();
}
+
+// This test verifies the ResourceStatistics.
+TEST_F(CniIsolatorTest, VETH_VerifyResourceStatistics)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.isolation = "network/cni";
+
+ flags.network_cni_plugins_dir = cniPluginDir;
+ flags.network_cni_config_dir = cniConfigDir;
+
+ Fetcher fetcher(flags);
+
+ Try<MesosContainerizer*> _containerizer =
+ MesosContainerizer::create(flags, true, &fetcher);
+
+ ASSERT_SOME(_containerizer);
+ Owned<MesosContainerizer> containerizer(_containerizer.get());
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave =
+ StartSlave(detector.get(), containerizer.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+
+ const Offer& offer = offers.get()[0];
+
+ CommandInfo command;
+ command.set_value("sleep 1000");
+
+ TaskInfo task = createTask(
+ offer.slave_id(),
+ Resources::parse("cpus:1;mem:128").get(),
+ command);
+
+ ContainerInfo* container = task.mutable_container();
+ container->set_type(ContainerInfo::MESOS);
+
+ // Make sure the container joins the mock CNI network.
+ container->add_network_infos()->set_name("veth");
+
+ Future<TaskStatus> statusStarting;
+ Future<TaskStatus> statusRunning;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusStarting))
+ .WillOnce(FutureArg<1>(&statusRunning));
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(statusStarting);
+ EXPECT_EQ(task.task_id(), statusStarting->task_id());
+ EXPECT_EQ(TASK_STARTING, statusStarting->state());
+
+ AWAIT_READY(statusRunning);
+ EXPECT_EQ(task.task_id(), statusRunning->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+ Future<hashset<ContainerID>> containers = containerizer->containers();
+ AWAIT_READY(containers);
+ ASSERT_EQ(1u, containers->size());
+
+ ContainerID containerId = *(containers->begin());
+
+ // Verify networking metrics.
+ Future<ResourceStatistics> usage = containerizer.get()->usage(containerId);
+ AWAIT_READY(usage);
+
+ // RX: Receive statistics.
+ ASSERT_TRUE(usage->has_net_rx_packets());
+ ASSERT_TRUE(usage->has_net_rx_bytes());
+ ASSERT_TRUE(usage->has_net_rx_errors());
+ ASSERT_TRUE(usage->has_net_rx_dropped());
+
+ // TX: Transmit statistics.
+ ASSERT_TRUE(usage->has_net_tx_packets());
+ ASSERT_TRUE(usage->has_net_tx_bytes());
+ ASSERT_TRUE(usage->has_net_tx_errors());
+ ASSERT_TRUE(usage->has_net_tx_dropped());
+
+ // Kill the task.
+ Future<TaskStatus> statusKilled;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusKilled));
+
+ driver.killTask(task.task_id());
+
+ AWAIT_READY(statusKilled);
+ EXPECT_EQ(TASK_KILLED, statusKilled->state());
+
+ driver.stop();
+ driver.join();
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
diff --git a/src/tests/environment.cpp b/src/tests/environment.cpp
index 8c6ec58..bc05e4b 100644
--- a/src/tests/environment.cpp
+++ b/src/tests/environment.cpp
@@ -973,6 +973,79 @@ private:
};
+// This is a test filter for the veth CNI plugin.
+class VEthFilter : public TestFilter
+{
+public:
+ VEthFilter()
+ {
+#ifdef __linux__
+ vector<string> messages;
+
+ // Checking if it runs as root.
+ Result<string> user = os::user();
+ CHECK_SOME(user);
+
+ if (user.get() != "root") {
+ messages.emplace_back("non-root user");
+ }
+
+ // This command returns `ip utility, iproute2-YYMMDD` where
+ // `YYMMDD` is a release (snapshot) date of iproute2.
+ Try<string> ipVersion = os::shell("ip -Version");
+
+ // Checking if iproute2 exists.
+ if (ipVersion.isError()) {
+ messages.emplace_back("iproute2 not found");
+ } else {
+ // Checking if it supports `ip link set ... netns ...`.
+ const string version = strings::trim(ipVersion.get());
+ if (version.size() < 6) {
+ messages.emplace_back("unexpected version");
+ } else {
+ Try<int> snapshot = numify<int>(version.substr(version.size() - 6));
+ if (snapshot.isError()) {
+ messages.emplace_back("iproute2 version is not an integer");
+ } else if (snapshot.get() < 100224) {
+ // Support for `netns` was added to iproute2 in v2.6.33.
+ messages.emplace_back("iproute2 doesn't support network namespaces");
+ }
+ }
+ }
+
+ // Checking if libprocess is bound on loopback address, in that
+ // case network namespace with veth network won't be able to
+ // connect to parent process on host network namespace.
+ // TODO(urbanserj): Improve the network connectivity check.
+ process::network::inet::Address address = process::address();
+ if (address.ip.isLoopback()) {
+ messages.emplace_back("libprocess is bound on loopback address");
+ }
+
+ disabled = !messages.empty();
+ if (disabled) {
+ std::cerr
+ << "-------------------------------------------------------------\n"
+ << "We can't run any VETH tests:\n"
+ << strings::join("\n", messages) << "\n"
+ << "-------------------------------------------------------------"
+ << std::endl;
+ }
+#else
+ disabled = true;
+#endif // __linux__
+ }
+
+ bool disable(const ::testing::TestInfo* test) const override
+ {
+ return matches(test, "VETH_") && disabled;
+ }
+
+private:
+ bool disabled;
+};
+
+
Environment::Environment(const Flags& _flags)
: stout::internal::tests::Environment(
std::vector<std::shared_ptr<TestFilter>>{
@@ -996,6 +1069,7 @@ Environment::Environment(const Flags& _flags)
std::make_shared<RootFilter>(),
std::make_shared<UnprivilegedUserFilter>(),
std::make_shared<UnzipFilter>(),
+ std::make_shared<VEthFilter>(),
std::make_shared<XfsFilter>()}),
flags(_flags)
{
