Repository: mesos Updated Branches: refs/heads/master 6b11f5739 -> 68ddce414
Allowed whitelist additional devices in cgroups devices subsystem. Review: https://reviews.apache.org/r/58603/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/68ddce41 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/68ddce41 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/68ddce41 Branch: refs/heads/master Commit: 68ddce4143d0f5cd5d54ec9a013f9bb09c37c645 Parents: 6b11f57 Author: Zhongbo Tian <[email protected]> Authored: Sun Apr 30 23:03:23 2017 +0800 Committer: Haosdent Huang <[email protected]> Committed: Sun Apr 30 23:20:31 2017 +0800 ---------------------------------------------------------------------- docs/configuration.md | 42 +++++++++ include/mesos/mesos.proto | 27 ++++++ include/mesos/type_utils.hpp | 9 ++ include/mesos/v1/mesos.proto | 27 ++++++ src/common/parse.hpp | 14 +++ src/common/type_utils.cpp | 6 ++ .../isolators/cgroups/subsystems/devices.cpp | 89 ++++++++++++++++---- .../isolators/cgroups/subsystems/devices.hpp | 9 +- src/slave/flags.cpp | 35 ++++++++ src/slave/flags.hpp | 1 + 10 files changed, 241 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/docs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index 159f946..79cada3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1153,6 +1153,48 @@ effect only when the <code>--cgroups_net_cls_primary_handle</code> is set. </tr> <tr> <td> + --allowed_devices + </td> + <td> +JSON object representing the devices that will be additionally +whitelisted by cgroups devices subsystem. Noted that the following +devices always be whitelisted by default: +<pre><code> * /dev/console + * /dev/tty0 + * /dev/tty1 + * /dev/pts/* + * /dev/ptmx + * /dev/net/tun + * /dev/null + * /dev/zero + * /dev/full + * /dev/tty + * /dev/urandom + * /dev/random +</code></pre> +This flag will take effect only when <code>cgroups/devices</code> is set in +<code>--isolation</code> flag. +<p/> +Example: +<pre><code>{ + "allowed_devices": [ + { + "device": { + "path": "/path/to/device" + }, + "access": { + "read": true, + "write": false, + "mknod": false + } + } + ] +} +</code></pre> + </td> +</tr> +<tr> + <td> --cgroups_root=VALUE </td> <td> http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/include/mesos/mesos.proto ---------------------------------------------------------------------- diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto index 8efedb5..1935f47 100644 --- a/include/mesos/mesos.proto +++ b/include/mesos/mesos.proto @@ -2752,3 +2752,30 @@ message FileInfo { // Group ID of owner. optional string gid = 7; } + + +/** + * Describes information abount a device. + */ +message Device { + required string path = 1; +} + + +/** + * Describes a device whitelist entry that expose from host to container. + */ +message DeviceAccess { + message Access { + optional bool read = 1; + optional bool write = 2; + optional bool mknod = 3; + } + required Device device = 1; + required Access access = 2; +} + + +message DeviceWhitelist { + repeated DeviceAccess allowed_devices = 1; +} http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/include/mesos/type_utils.hpp ---------------------------------------------------------------------- diff --git a/include/mesos/type_utils.hpp b/include/mesos/type_utils.hpp index 5f771aa..4f0a59c 100644 --- a/include/mesos/type_utils.hpp +++ b/include/mesos/type_utils.hpp @@ -269,6 +269,11 @@ std::ostream& operator<<( std::ostream& operator<<( std::ostream& stream, + const DeviceWhitelist& deviceWhitelist); + + +std::ostream& operator<<( + std::ostream& stream, const CheckStatusInfo& checkStatusInfo); @@ -370,6 +375,10 @@ std::ostream& operator<<( std::ostream& stream, const hashmap<std::string, std::string>& map); +std::ostream& operator<<( + std::ostream& stream, + const ::google::protobuf::Message& map); + } // namespace mesos { namespace std { http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/include/mesos/v1/mesos.proto ---------------------------------------------------------------------- diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto index 6d0bd8c..ef562a2 100644 --- a/include/mesos/v1/mesos.proto +++ b/include/mesos/v1/mesos.proto @@ -2735,3 +2735,30 @@ message FileInfo { // Group ID of owner. optional string gid = 7; } + + +/** + * Describes information abount a device. + */ +message Device { + required string path = 1; +} + + +/** + * Describes a device whitelist entry that expose from host to container. + */ +message DeviceAccess { + message Access { + optional bool read = 1; + optional bool write = 2; + optional bool mknod = 3; + } + required Device device = 1; + required Access access = 2; +} + + +message DeviceWhitelist { + repeated DeviceAccess allowed_devices = 1; +} http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/src/common/parse.hpp ---------------------------------------------------------------------- diff --git a/src/common/parse.hpp b/src/common/parse.hpp index e90738a..64eabf8 100644 --- a/src/common/parse.hpp +++ b/src/common/parse.hpp @@ -88,6 +88,20 @@ inline Try<mesos::ContainerInfo> parse(const std::string& value) } +template <> +inline Try<mesos::DeviceWhitelist> parse(const std::string& value) +{ + // Convert from string or file to JSON. + Try<JSON::Object> json = parse<JSON::Object>(value); + if (json.isError()) { + return Error(json.error()); + } + + // Convert from JSON to Protobuf. + return protobuf::parse<mesos::DeviceWhitelist>(json.get()); +} + + // When the same variable is listed multiple times, // uses only the last value. template <> http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/src/common/type_utils.cpp ---------------------------------------------------------------------- diff --git a/src/common/type_utils.cpp b/src/common/type_utils.cpp index 9bc32af..6bfb558 100644 --- a/src/common/type_utils.cpp +++ b/src/common/type_utils.cpp @@ -468,6 +468,12 @@ ostream& operator<<(ostream& stream, const CapabilityInfo& capabilityInfo) } +ostream& operator<<(ostream& stream, const DeviceWhitelist& deviceWhitelist) +{ + return stream << JSON::protobuf(deviceWhitelist); +} + + ostream& operator<<(ostream& stream, const CheckStatusInfo& checkStatusInfo) { switch (checkStatusInfo.type()) { http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.cpp b/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.cpp index 9b5cf83..3055b7d 100644 --- a/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.cpp +++ b/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.cpp @@ -14,12 +14,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <sys/stat.h> + #include <process/id.hpp> #include <stout/nothing.hpp> #include <stout/try.hpp> - -#include "linux/cgroups.hpp" +#include <stout/os.hpp> #include "slave/containerizer/mesos/isolators/cgroups/subsystems/devices.hpp" @@ -30,6 +31,7 @@ using process::Future; using process::Owned; using std::string; +using std::vector; namespace mesos { namespace internal { @@ -63,15 +65,76 @@ Try<Owned<Subsystem>> DevicesSubsystem::create( const Flags& flags, const string& hierarchy) { - return Owned<Subsystem>(new DevicesSubsystem(flags, hierarchy)); + vector<cgroups::devices::Entry> whitelistDeviceEntries; + + foreach (const char* _entry, DEFAULT_WHITELIST_ENTRIES) { + Try<cgroups::devices::Entry> entry = + cgroups::devices::Entry::parse(_entry); + + CHECK_SOME(entry); + whitelistDeviceEntries.push_back(entry.get()); + } + + if (flags.allowed_devices.isSome()) { + foreach (const DeviceAccess& device_access, + flags.allowed_devices->allowed_devices()) { + string path = device_access.device().path(); + const DeviceAccess_Access access = device_access.access(); + bool readAccess = (access.has_read() && access.read()); + bool writeAccess = (access.has_write() && access.write()); + bool mknodAccess = (access.has_mknod() && access.mknod()); + + if (!(readAccess || writeAccess || mknodAccess)) { + return Error("Could not whitelist device '" + path + + "' without any access privileges"); + } + + Try<dev_t> device = os::stat::rdev(path); + if (device.isError()) { + return Error("Failed to obtain device ID for '" + path + + "': " + device.error()); + } + + Try<mode_t> mode = os::stat::mode(path); + if (mode.isError()) { + return Error("Failed to obtain device mode for '" + path + + "': " + mode.error()); + } + + Entry::Selector::Type type; + if (S_ISBLK(mode.get())) { + type = Entry::Selector::Type::BLOCK; + } else if (S_ISCHR(mode.get())) { + type = Entry::Selector::Type::CHARACTER; + } else { + return Error("Failed to determine device type for '" + path + + "'"); + } + + cgroups::devices::Entry entry; + entry.selector.type = type; + entry.selector.major = major(device.get()); + entry.selector.minor = minor(device.get()); + entry.access.read = readAccess; + entry.access.write = writeAccess; + entry.access.mknod = mknodAccess; + + whitelistDeviceEntries.push_back(entry); + } + } + + return Owned<Subsystem>( + new DevicesSubsystem(flags, hierarchy, whitelistDeviceEntries)); } DevicesSubsystem::DevicesSubsystem( const Flags& _flags, - const string& _hierarchy) + const string& _hierarchy, + const vector<cgroups::devices::Entry>& _whitelistDeviceEntries) : ProcessBase(process::ID::generate("cgroups-devices-subsystem")), - Subsystem(_flags, _hierarchy) {} + Subsystem(_flags, _hierarchy), + whitelistDeviceEntries(_whitelistDeviceEntries) {} Future<Nothing> DevicesSubsystem::recover( @@ -127,20 +190,12 @@ Future<Nothing> DevicesSubsystem::prepare( return Failure("Failed to deny all devices: " + deny.error()); } - foreach (const char* _entry, DEFAULT_WHITELIST_ENTRIES) { - Try<cgroups::devices::Entry> entry = - cgroups::devices::Entry::parse(_entry); - - CHECK_SOME(entry); - - Try<Nothing> allow = cgroups::devices::allow( - hierarchy, - cgroup, - entry.get()); + foreach (const cgroups::devices::Entry& entry, whitelistDeviceEntries) { + Try<Nothing> allow = cgroups::devices::allow(hierarchy, cgroup, entry); if (allow.isError()) { - return Failure("Failed to whitelist default device " - "'" + stringify(entry.get()) + "': " + allow.error()); + return Failure("Failed to whitelist device " + "'" + stringify(entry) + "': " + allow.error()); } } http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.hpp b/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.hpp index ca27271..4ab224e 100644 --- a/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.hpp +++ b/src/slave/containerizer/mesos/isolators/cgroups/subsystems/devices.hpp @@ -18,12 +18,15 @@ #define __CGROUPS_ISOLATOR_SUBSYSTEMS_DEVICES_HPP__ #include <string> +#include <vector> #include <process/owned.hpp> #include <stout/hashset.hpp> #include <stout/try.hpp> +#include "linux/cgroups.hpp" + #include "slave/flags.hpp" #include "slave/containerizer/mesos/isolators/cgroups/constants.hpp" @@ -63,9 +66,13 @@ public: const std::string& cgroup); private: - DevicesSubsystem(const Flags& flags, const std::string& hierarchy); + DevicesSubsystem( + const Flags& flags, + const std::string& hierarchy, + const std::vector<cgroups::devices::Entry>& whitelistDeviceEntries); hashset<ContainerID> containerIds; + std::vector<cgroups::devices::Entry> whitelistDeviceEntries; }; } // namespace slave { http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/src/slave/flags.cpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp index c50e43c..ed99fad 100644 --- a/src/slave/flags.cpp +++ b/src/slave/flags.cpp @@ -454,6 +454,41 @@ mesos::internal::slave::Flags::Flags() "handles that can be used with the primary handle. This will take\n" "effect only when the `--cgroups_net_cls_primary_handle is set."); + add(&Flags::allowed_devices, + "allowed_devices", + "JSON array representing the devices that will be additionally\n" + "whitelisted by cgroups devices subsystem. Noted that the following\n" + "devices always be whitelisted by default:\n" + " * /dev/console\n" + " * /dev/tty0\n" + " * /dev/tty1\n" + " * /dev/pts/*\n" + " * /dev/ptmx\n" + " * /dev/net/tun\n" + " * /dev/null\n" + " * /dev/zero\n" + " * /dev/full\n" + " * /dev/tty\n" + " * /dev/urandom\n" + " * /dev/random\n" + "This flag will take effect only when `cgroups/devices` is set in\n" + "`--isolation` flag.\n" + "Example:\n" + "{\n" + " \"allowed_devices\": [\n" + " {\n" + " \"device\": {\n" + " \"path\": \"/path/to/device\"\n" + " },\n" + " \"access\": {\n" + " \"read\": true,\n" + " \"write\": false,\n" + " \"mknod\": false\n" + " }\n" + " }\n" + " ]\n" + "}\n"); + add(&Flags::agent_subsystems, "agent_subsystems", flags::DeprecatedName("slave_subsystems"), http://git-wip-us.apache.org/repos/asf/mesos/blob/68ddce41/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index c7a4604..e5784ef 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -99,6 +99,7 @@ public: bool cgroups_cpu_enable_pids_and_tids_count; Option<std::string> cgroups_net_cls_primary_handle; Option<std::string> cgroups_net_cls_secondary_handles; + Option<DeviceWhitelist> allowed_devices; Option<std::string> agent_subsystems; Option<std::vector<unsigned int>> nvidia_gpu_devices; Option<std::string> perf_events;
