This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new b4a3ad764 [cgroups2] Add cgroups v2 setup and teardown logic to
ContainerizerTest.
b4a3ad764 is described below
commit b4a3ad764f4f118a0c928f03bdf11c52f61426fd
Author: Devin Leamy <[email protected]>
AuthorDate: Wed Apr 24 16:02:23 2024 -0400
[cgroups2] Add cgroups v2 setup and teardown logic to ContainerizerTest.
`StartSlave()` and similar test-setup functions mounted cgroups v1
hierarchies
and initialized controllers. On cgroups v2 machines, this setup would fail
or
result in irregular cgroup setups. As a step towards end-to-end testing for
the `MesosContainerizer`, we update the Agent test fixtures such that they
work correctly on both cgroups v1 and v2 hosts.
This closes #573
---
src/tests/mesos.cpp | 96 ++++++++++++++++++++++++++++++++++++++++++++++++-----
src/tests/mesos.hpp | 7 ++++
2 files changed, 95 insertions(+), 8 deletions(-)
diff --git a/src/tests/mesos.cpp b/src/tests/mesos.cpp
index 7d23dbc72..762200cda 100644
--- a/src/tests/mesos.cpp
+++ b/src/tests/mesos.cpp
@@ -36,6 +36,11 @@
#ifdef __linux__
#include "linux/cgroups.hpp"
#include "linux/fs.hpp"
+
+#ifdef ENABLE_CGROUPS_V2
+#include "linux/cgroups2.hpp"
+#endif // ENABLE_CGROUPS_V2
+
#endif
#ifdef ENABLE_PORT_MAPPING_ISOLATOR
@@ -55,6 +60,7 @@
using mesos::master::detector::MasterDetector;
using std::list;
+using std::set;
using std::shared_ptr;
using std::string;
using std::vector;
@@ -579,7 +585,13 @@ slave::Flags
ContainerizerTest<slave::MesosContainerizer>::CreateSlaveFlags()
// Use cgroup isolators if they're available and we're root.
// TODO(idownes): Refactor the cgroups/non-cgroups code.
- if (cgroups::enabled() && user.get() == "root") {
+ if (cgroupsV2() && *user == "root") {
+ // TODO(dleamy): Add the memory isolator once it's supported by the cgroups
+ // v2 isolator.
+ flags.isolation = "cgroups/cpu";
+ flags.cgroups_hierarchy = "/sys/fs/cgroup";
+ flags.cgroups_root = TEST_CGROUPS_ROOT;
+ } else if (cgroups::enabled() && *user == "root") {
flags.isolation = "cgroups/cpu,cgroups/mem";
flags.cgroups_hierarchy = baseHierarchy;
flags.cgroups_root =
@@ -625,9 +637,16 @@ void
ContainerizerTest<slave::MesosContainerizer>::SetUpTestCase()
Result<string> user = os::user();
EXPECT_SOME(user);
- if (cgroups::enabled() && user.get() == "root") {
+ if (cgroupsV2() && *user == "root") {
+#ifdef ENABLE_CGROUPS_V2
+ // Clean up test cgroups.
+ if (cgroups2::exists(TEST_CGROUPS_ROOT)) {
+ AWAIT_ASSERT_READY(cgroups2::destroy(TEST_CGROUPS_ROOT));
+ }
+#endif // ENABLE_CGROUPS_V2
+ } else if (cgroups::enabled() && *user == "root") {
// Clean up any testing hierarchies.
- Try<std::set<string>> hierarchies = cgroups::hierarchies();
+ Try<set<string>> hierarchies = cgroups::hierarchies();
ASSERT_SOME(hierarchies);
foreach (const string& hierarchy, hierarchies.get()) {
if (strings::startsWith(hierarchy, TEST_CGROUPS_HIERARCHY)) {
@@ -645,9 +664,16 @@ void
ContainerizerTest<slave::MesosContainerizer>::TearDownTestCase()
Result<string> user = os::user();
EXPECT_SOME(user);
- if (cgroups::enabled() && user.get() == "root") {
+ if (cgroupsV2() && *user == "root") {
+#ifdef ENABLE_CGROUPS_V2
+ // Clean up test cgroups.
+ if (cgroups2::exists(TEST_CGROUPS_ROOT)) {
+ AWAIT_ASSERT_READY(cgroups2::destroy(TEST_CGROUPS_ROOT));
+ }
+#endif // ENABLE_CGROUPS_V2
+ } else if (cgroups::enabled() && *user == "root") {
// Clean up any testing hierarchies.
- Try<std::set<string>> hierarchies = cgroups::hierarchies();
+ Try<set<string>> hierarchies = cgroups::hierarchies();
ASSERT_SOME(hierarchies);
foreach (const string& hierarchy, hierarchies.get()) {
if (strings::startsWith(hierarchy, TEST_CGROUPS_HIERARCHY)) {
@@ -665,8 +691,10 @@ void ContainerizerTest<slave::MesosContainerizer>::SetUp()
Result<string> user = os::user();
EXPECT_SOME(user);
- if (cgroups::enabled() && *user == "root") {
- SetupCgroups();
+ if (cgroupsV2() && *user == "root") {
+ SetUpCgroupsV2();
+ } else if (cgroups::enabled() && *user == "root") {
+ SetUpCgroups();
}
}
@@ -678,7 +706,9 @@ void
ContainerizerTest<slave::MesosContainerizer>::TearDown()
Result<string> user = os::user();
EXPECT_SOME(user);
- if (cgroups::enabled() && *user == "root") {
+ if (cgroupsV2() && *user == "root") {
+ TearDownCgroupsV2();
+ } else if (cgroups::enabled() && *user == "root") {
TearDownCgroups();
}
}
@@ -764,6 +794,33 @@ void
ContainerizerTest<slave::MesosContainerizer>::SetUpCgroups()
}
+void ContainerizerTest<slave::MesosContainerizer>::SetUpCgroupsV2()
+{
+#ifdef ENABLE_CGROUPS_V2
+ // When the agent binary is run, cgroups are initialized inside
+ // `slave/main.cpp`. This cgroups setup is done to in place of that
+ // initialization.
+ if (!cgroups2::exists(TEST_CGROUPS_ROOT)) {
+ ASSERT_SOME(cgroups2::create(TEST_CGROUPS_ROOT));
+ }
+
+ Try<set<string>> _controllers = cgroups2::controllers::available(
+ cgroups2::ROOT_CGROUP);
+ ASSERT_SOME(_controllers);
+ subsystems = *_controllers;
+ vector<string> controllers(std::make_move_iterator(_controllers->begin()),
+ std::make_move_iterator(_controllers->end()));
+
+ // Enable all of the controllers inside of the test root cgroup so they
+ // are accessible from the child container cgroups.
+ ASSERT_TRUE(cgroups2::exists(TEST_CGROUPS_ROOT));
+ ASSERT_SOME(
+ cgroups2::controllers::enable(cgroups2::ROOT_CGROUP, controllers));
+ ASSERT_SOME(cgroups2::controllers::enable(TEST_CGROUPS_ROOT, controllers));
+#endif // ENABLE_CGROUPS_V2
+}
+
+
void ContainerizerTest<slave::MesosContainerizer>::TearDownCgroups()
{
foreach (const string& subsystem, subsystems) {
@@ -796,9 +853,32 @@ void
ContainerizerTest<slave::MesosContainerizer>::TearDownCgroups()
}
}
}
+
+
+void ContainerizerTest<slave::MesosContainerizer>::TearDownCgroupsV2()
+{
+#ifdef ENABLE_CGROUPS_V2
+ // Destroy all cgroups that were created under the test root cgroup.
+ AWAIT_ASSERT_READY(cgroups2::destroy(TEST_CGROUPS_ROOT));
+#endif // ENABLE_CGROUPS_V2
+}
#endif // __linux__
+bool ContainerizerTest<slave::MesosContainerizer>::cgroupsV2()
+{
+#ifdef ENABLE_CGROUPS_V2
+ Try<bool> mounted = cgroups2::mounted();
+ if (mounted.isError()) {
+ return false;
+ }
+ return *mounted;
+#else
+ return false;
+#endif // ENABLE_CGROUPS_V2
+}
+
+
string ParamDiskQuota::Printer::operator()(
const ::testing::TestParamInfo<ParamDiskQuota::Type>& info) const
{
diff --git a/src/tests/mesos.hpp b/src/tests/mesos.hpp
index bc0aa1e41..8d7a7b706 100644
--- a/src/tests/mesos.hpp
+++ b/src/tests/mesos.hpp
@@ -411,7 +411,14 @@ protected:
void SetUp() override;
void TearDown() override;
void SetUpCgroups();
+ void SetUpCgroupsV2();
void TearDownCgroups();
+ void TearDownCgroupsV2();
+
+ // Returns true if cgroups v2 is being used. Returns false if there was
+ // an error while determining if cgroups v2 is configured correctly or the
+ // `--enable-cgroups-v2` flag was not provided.
+ static bool cgroupsV2();
private:
// Base hierarchy for separately mounted cgroup controllers, e.g., if the
// base hierarchy is /sys/fs/cgroup then each controller will be mounted to
