Clarify NetworkInfo semantics for IP addresses and group policies. In Mesos 0.25.0, a new message called NetworkInfo was introduced. This message allows framework authors to communicate with network isolation modules via a first-class message type to request IP addresses and network group isolation policies.
Unfortunately, the structure is somewhat confusing to both framework authors and module implementors. 1) It's unclear how IP addresses map to virtual interfaces inside the container. 2) It's difficult for application developers to understand the final policy when multiple IP addresses can be assigned with differing isolation policies. Review: https://reviews.apache.org/r/39531 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5520587e Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5520587e Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5520587e Branch: refs/heads/master Commit: 5520587ef29ef7b27ebff62f6b454da5d0f97a5a Parents: 3ea2540 Author: Connor Doyle <[email protected]> Authored: Tue Nov 3 15:07:01 2015 -0500 Committer: Kapil Arya <[email protected]> Committed: Wed Nov 4 17:33:06 2015 -0500 ---------------------------------------------------------------------- docs/networking-for-mesos-managed-containers.md | 114 ++++++++++++++----- include/mesos/mesos.proto | 68 ++++++++--- include/mesos/v1/mesos.proto | 83 ++++++++++++++ src/common/http.cpp | 9 ++ src/common/http.hpp | 1 - src/docker/executor.cpp | 6 + src/examples/test_hook_module.cpp | 4 + src/slave/slave.cpp | 6 + src/tests/common/http_tests.cpp | 36 ++++++ src/tests/hook_tests.cpp | 8 +- 10 files changed, 289 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/docs/networking-for-mesos-managed-containers.md ---------------------------------------------------------------------- diff --git a/docs/networking-for-mesos-managed-containers.md b/docs/networking-for-mesos-managed-containers.md index 33568a8..594fe2b 100644 --- a/docs/networking-for-mesos-managed-containers.md +++ b/docs/networking-for-mesos-managed-containers.md @@ -125,13 +125,21 @@ A new NetworkInfo message has been introduced: ```{.proto} message NetworkInfo { enum Protocol { - IPv4 = 0, - IPv6 = 1 + IPv4 = 1; + IPv6 = 2; } - optional Protocol protocol = 1; + message IPAddress { + optional Protocol protocol = 1; - optional string ip_address = 2; + optional string ip_address = 2; + } + + repeated IPAddress ip_addresses = 5; + + optional Protocol protocol = 1 [deprecated = true]; // Since 0.26.0 + + optional string ip_address = 2 [deprecated = true]; // Since 0.26.0 repeated string groups = 3; @@ -162,8 +170,12 @@ message in TaskInfo. Here are a few examples: Â container: ContainerInfo { Â Â Â network_infos: [ Â Â Â Â Â NetworkInfo { - Â Â Â Â Â Â Â protocol: None; - Â Â Â Â Â Â Â ip_address: None; + ip_addresses: [ + IPAddress { + Â Â Â Â Â Â Â protocol: None; + Â Â Â Â Â Â Â ip_address: None; + } + ] Â Â Â Â Â Â Â groups: []; Â Â Â Â Â Â Â labels: None; Â Â Â Â Â } @@ -172,33 +184,71 @@ message in TaskInfo. Here are a few examples: } ``` -2. A request for one IPv4 and one IPv6 address, in two separate groups using the +2. A request for one IPv4 and one IPv6 address, in two groups using the default command executor + ``` TaskInfo { - ... - command: ..., - container: ContainerInfo { - network_infos: [ - NetworkInfo { - protocol: IPv4; - ip_address: None; - groups: ["public"]; - labels: None; - }, - NetworkInfo { - protocol: IPv6; - ip_address: None; - groups: ["private"]; - labels: None; - } - ] - } + Â ... + Â command: ..., + Â container: ContainerInfo { + Â Â Â network_infos: [ + Â Â Â Â Â NetworkInfo { + ip_addresses: [ + IPAddress { + protocol: IPv4; + ip_address: None; + }, + IPAddress { + protocol: IPv6; + ip_address: None; + } + ] + Â Â Â Â Â Â Â groups: ["dev", "test"]; + Â Â Â Â Â Â Â labels: None; + Â Â Â Â Â } + Â Â Â ] + Â } + } + ``` + +3. A request for two network interfaces, each with one IP address, each in + a different network group using the default command executor + + + ``` + TaskInfo { + Â ... + Â command: ..., + Â container: ContainerInfo { + Â Â Â network_infos: [ + Â Â Â Â Â NetworkInfo { + ip_addresses: [ + IPAddress { + protocol: None; + ip_address: None; + } + ] + Â Â Â Â Â Â Â groups: ["foo"]; + Â Â Â Â Â Â Â labels: None; + Â Â Â Â Â }, + Â Â Â Â Â NetworkInfo { + ip_addresses: [ + IPAddress { + protocol: None; + ip_address: None; + } + ] + Â Â Â Â Â Â Â groups: ["bar"]; + Â Â Â Â Â Â Â labels: None; + Â Â Â Â Â }, + Â Â Â ] + Â } } ``` -3. A request for a specific IP address using a custom executor +4. A request for a specific IP address using a custom executor ``` TaskInfo { @@ -208,8 +258,12 @@ message in TaskInfo. Here are a few examples: container: ContainerInfo { network_infos: [ NetworkInfo { - protocol: None; - ip_address: "10.1.2.3"; + ip_addresses: [ + IPAddress { + protocol: None; + ip_address: "10.1.2.3"; + } + ] groups: []; labels: None; } @@ -219,7 +273,9 @@ message in TaskInfo. Here are a few examples: } ``` -NOTE: The Mesos Containerizer will reject any CommandInfo that has a ContainerInfo. For this reason, when opting in to network isolation when using the Mesos Containerizer, set TaskInfo.ContainerInfo.NetworkInfo. +NOTE: The Mesos Containerizer will reject any CommandInfo that has a +ContainerInfo. For this reason, when opting in to network isolation when +using the Mesos Containerizer, set TaskInfo.ContainerInfo.NetworkInfo. ## Address Discovery http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/include/mesos/mesos.proto ---------------------------------------------------------------------- diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto index 9400434..1b36e66 100644 --- a/include/mesos/mesos.proto +++ b/include/mesos/mesos.proto @@ -1361,16 +1361,20 @@ message Volume { /** - * Describes a network request by framework as well as network resolution - * provided by the executor or Agent. + * Describes a network request from a framework as well as network resolution + * provided by Mesos. * - * A framework may request the network isolator on the Agent to assign an IP - * address to the container being launched. Alternatively, it can provide a - * specific IP address to be assigned to the container. The NetworkInfo message - * is not interpreted by the Master or Agent and is intended to be use by Agent - * modules implementing network isolation. If the modules are missing, the - * message is simply ignored. In future, the task launch will fail if there is - * no module providing the network isolation capabilities (MESOS-3390). + * A Framework may request the network isolator on the Agent to isolate the + * container in a network namespace and create a virtual network interface. + * The `NetworkInfo` message describes the properties of that virtual + * interface, including the IP addresses and network isolation policy + * (network group membership). + * + * The NetworkInfo message is not interpreted by the Master or Agent and is + * intended to be used by Agent and Master modules implementing network + * isolation. If the modules are missing, the message is simply ignored. In + * future, the task launch will fail if there is no module providing the + * network isolation capabilities (MESOS-3390). * * An executor, Agent, or an Agent module may append NetworkInfos inside * TaskStatus::container_status to provide information such as the container IP @@ -1382,23 +1386,57 @@ message NetworkInfo { IPv6 = 2; } + // Specifies a request for an IP address, or reports the assigned container + // IP address. + // + // Users can request an automatically assigned IP (for example, via an + // IPAM service) or a specific IP by adding a NetworkInfo to the + // ContainerInfo for a task. On a request, specifying neither `protocol` + // nor `ip_address` means that any available address may be assigned. + message IPAddress { + // Specify IP address requirement. Set protocol to the desired value to + // request the network isolator on the Agent to assign an IP address to the + // container being launched. If a specific IP address is specified in + // ip_address, this field should not be set. + optional Protocol protocol = 1; + + // Statically assigned IP provided by the Framework. This IP will be + // assigned to the container by the network isolator module on the Agent. + // This field should not be used with the protocol field above. + // + // If an explicit address is requested but is unavailable, the network + // isolator should fail the task. + optional string ip_address = 2; + } + + // When included in a ContainerInfo, each of these represent a + // request for an IP address. Each request can specify an explicit address + // or the IP protocol to use. + // + // When included in a TaskStatus message, these inform the framework + // scheduler about the IP addresses that are bound to the container + // interface. When there are no custom network isolator modules installed, + // this field is filled in automatically with the Agent IP address. + repeated IPAddress ip_addresses = 5; + // Specify IP address requirement. Set protocol to the desired value to // request the network isolator on the Agent to assign an IP address to the // container being launched. If a specific IP address is specified in // ip_address, this field should not be set. - optional Protocol protocol = 1; + optional Protocol protocol = 1 [deprecated = true]; // Since 0.26.0 // Statically assigned IP provided by the Framework. This IP will be assigned // to the container by the network isolator module on the Agent. This field // should not be used with the protocol field above. // NOTE: It is up to the networking 'provider' (IPAM/Isolator) to interpret // this either as a hint of as a requirement for assigning the IP. - optional string ip_address = 2; + optional string ip_address = 2 [deprecated = true]; // Since 0.26.0 - // A group is the name given to a set of logically-related IPs that are - // allowed to communicate within themselves. For example, one might want - // to create separate groups for isolating dev, testing, qa and prod - // deployment environments. + // A group is the name given to a set of logically-related interfaces that + // are allowed to communicate among themselves. Network traffic is allowed + // between two container interfaces that share at least one network group. + // For example, one might want to create separate groups for isolating dev, + // testing, qa and prod deployment environments. repeated string groups = 3; // To tag certain metadata to be used by Isolator/IPAM, e.g., rack, etc. http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/include/mesos/v1/mesos.proto ---------------------------------------------------------------------- diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto index 8131778..7b725d3 100644 --- a/include/mesos/v1/mesos.proto +++ b/include/mesos/v1/mesos.proto @@ -1337,6 +1337,89 @@ message Volume { /** + * Describes a network request by framework as well as network resolution + * provided by the the executor or Agent. + * + * A framework may request the network isolator on the Agent to isolate the + * container in a network namespace and create a virtual network interface. + * The `NetworkInfo` message describes the properties of that virtual + * interface, including the IP addresses and network isolation policy + * (network group membership). + * + * The NetworkInfo message is not interpreted by the Master or Agent and is + * intended to be used by Agent modules implementing network isolation. If the + * modules are missing, the message is simply ignored. In future, the task + * launch will fail if there is no module providing the network isolation + * capabilities (MESOS-3390). + * + * An executor, Agent, or an Agent module may append NetworkInfos inside + * TaskStatus::container_status to provide information such as the container IP + * address and isolation groups. + */ +message NetworkInfo { + enum Protocol { + IPv4 = 1; + IPv6 = 2; + } + + // Specifies either a request for an IP address, or the actual assigned + // IP address. + // + // On a request (included in ContainerInfo) specifying neither `protocol` + // nor `ip_address` means that the isolator is free to assign any available + // address. + message IPAddress { + // Specify IP address requirement. Set protocol to the desired value to + // request the network isolator on the Agent to assign an IP address to the + // container being launched. If a specific IP address is specified in + // ip_address, this field should not be set. + optional Protocol protocol = 1; + + // Statically assigned IP provided by the Framework. This IP will be + // assigned to the container by the network isolator module on the Agent. + // This field should not be used with the protocol field above. + // + // If an explicit address is requested but is unavailable, the network + // isolator should fail the task. + optional string ip_address = 2; + } + + // When included in a ContainerInfo, each of these represent a + // request for an IP address. Each request can specify an explicit address + // or the IP protocol to use. + // + // When included in a TaskStatus message, these inform the framework + // scheduler about the IP addresses that are bound to the container + // interface. When there are no custom network isolator modules installed, + // this field is filled in automatically with the Agent IP address. + repeated IPAddress ip_addresses = 5; + + // Specify IP address requirement. Set protocol to the desired value to + // request the network isolator on the Agent to assign an IP address to the + // container being launched. If a specific IP address is specified in + // ip_address, this field should not be set. + optional Protocol protocol = 1 [deprecated = true]; // Since 0.26.0 + + // Statically assigned IP provided by the Framework. This IP will be assigned + // to the container by the network isolator module on the Agent. This field + // should not be used with the protocol field above. + // NOTE: It is up to the networking 'provider' (IPAM/Isolator) to interpret + // this either as a hint of as a requirement for assigning the IP. + optional string ip_address = 2 [deprecated = true]; // Since 0.26.0 + + // A group is the name given to a set of logically-related interfaces that + // are allowed to communicate among themselves. Network traffic is allowed + // between two container interfaces that share at least one network group. + // For example, one might want to create separate groups for isolating dev, + // testing, qa and prod deployment environments. + repeated string groups = 3; + + // To tag certain metadata to be used by Isolator/IPAM, e.g., rack, etc. + optional Labels labels = 4; +}; + + +/** * Describes a container configuration and allows extensible * configurations for different container implementations. */ http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/common/http.cpp ---------------------------------------------------------------------- diff --git a/src/common/http.cpp b/src/common/http.cpp index f56d8a1..abb4c6b 100644 --- a/src/common/http.cpp +++ b/src/common/http.cpp @@ -183,6 +183,15 @@ JSON::Object model(const NetworkInfo& info) object.values["labels"] = std::move(model(info.labels())); } + if (info.ip_addresses().size() > 0) { + JSON::Array array; + array.values.reserve(info.ip_addresses().size()); // MESOS-2353. + foreach (const NetworkInfo::IPAddress& ipAddress, info.ip_addresses()) { + array.values.push_back(JSON::Protobuf(ipAddress)); + } + object.values["ip_addresses"] = std::move(array); + } + return object; } http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/common/http.hpp ---------------------------------------------------------------------- diff --git a/src/common/http.hpp b/src/common/http.hpp index 0cc98a8..606509c 100644 --- a/src/common/http.hpp +++ b/src/common/http.hpp @@ -80,7 +80,6 @@ JSON::Object model(const Attributes& attributes); JSON::Object model(const CommandInfo& command); JSON::Object model(const ExecutorInfo& executorInfo); JSON::Array model(const Labels& labels); -JSON::Object model(const NetworkInfo& info); JSON::Object model(const ContainerStatus& status); // These are the two identical ways to model a task, depending on http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/docker/executor.cpp ---------------------------------------------------------------------- diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp index d4c05c2..9d084d4 100644 --- a/src/docker/executor.cpp +++ b/src/docker/executor.cpp @@ -174,7 +174,13 @@ public: NetworkInfo* networkInfo = status.mutable_container_status()->add_network_infos(); + + // TODO(CD): Deprecated -- Remove after 0.27.0. networkInfo->set_ip_address(container.ipAddress.get()); + + NetworkInfo::IPAddress* ipAddress = + networkInfo->add_ip_addresses(); + ipAddress->set_ip_address(container.ipAddress.get()); } driver->sendStatusUpdate(status); } http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/examples/test_hook_module.cpp ---------------------------------------------------------------------- diff --git a/src/examples/test_hook_module.cpp b/src/examples/test_hook_module.cpp index 43d6cb9..f1c4266 100644 --- a/src/examples/test_hook_module.cpp +++ b/src/examples/test_hook_module.cpp @@ -258,7 +258,11 @@ public: // 'HookTest.VerifySlaveTaskStatusDecorator' test. NetworkInfo* networkInfo = result.mutable_container_status()->add_network_infos(); + // TODO(CD): Deprecated -- remove after 0.27.0. networkInfo->set_ip_address("4.3.2.1"); + NetworkInfo::IPAddress* ipAddress = + networkInfo->add_ip_addresses(); + ipAddress->set_ip_address("4.3.2.1"); networkInfo->add_groups("public"); Label* networkInfoLabel = networkInfo->mutable_labels()->add_labels(); http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/slave/slave.cpp ---------------------------------------------------------------------- diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp index 5f9b52b..ddeece4 100644 --- a/src/slave/slave.cpp +++ b/src/slave/slave.cpp @@ -2832,7 +2832,13 @@ void Slave::statusUpdate(StatusUpdate update, const UPID& pid) update.mutable_status()->mutable_container_status(); if (containerStatus->network_infos().size() == 0) { NetworkInfo* networkInfo = containerStatus->add_network_infos(); + + // TODO(CD): Deprecated -- Remove after 0.27.0. networkInfo->set_ip_address(stringify(self().address.ip)); + + NetworkInfo::IPAddress* ipAddress = + networkInfo->add_ip_addresses(); + ipAddress->set_ip_address(stringify(self().address.ip)); } TaskStatus status = update.status(); http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/tests/common/http_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/common/http_tests.cpp b/src/tests/common/http_tests.cpp index c2e7704..66a2dca 100644 --- a/src/tests/common/http_tests.cpp +++ b/src/tests/common/http_tests.cpp @@ -188,3 +188,39 @@ TEST(HTTP, ModelRoleResources) ASSERT_SOME(expected); EXPECT_EQ(expected.get(), object); } + +// This test ensures we don't break the API when it comes to JSON +// representation of NetworkInfo. +TEST(HTTP, SerializeNetworkInfo) +{ + NetworkInfo networkInfo; + NetworkInfo::IPAddress* address = networkInfo.add_ip_addresses(); + address->set_protocol(NetworkInfo::IPv4); + address->set_ip_address("10.0.0.1"); + networkInfo.set_protocol(NetworkInfo::IPv6); + networkInfo.set_ip_address("10.0.0.2"); + networkInfo.add_groups("foo"); + networkInfo.add_groups("bar"); + + JSON::Value object = JSON::Protobuf(networkInfo); + + Try<JSON::Value> expected = JSON::parse( + "{" + " \"ip_addresses\":" + " [" + " {" + " \"protocol\": \"IPv4\"," + " \"ip_address\": \"10.0.0.1\"" + " }" + " ]," + " \"protocol\": \"IPv6\"," + " \"ip_address\": \"10.0.0.2\"," + " \"groups\": [" + " \"foo\"," + " \"bar\"" + " ]" + "}"); + + ASSERT_SOME(expected); + EXPECT_EQ(expected.get(), object); +} http://git-wip-us.apache.org/repos/asf/mesos/blob/5520587e/src/tests/hook_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/hook_tests.cpp b/src/tests/hook_tests.cpp index 5a5d019..0b48551 100644 --- a/src/tests/hook_tests.cpp +++ b/src/tests/hook_tests.cpp @@ -572,10 +572,16 @@ TEST_F(HookTest, VerifySlaveTaskStatusDecorator) status.get().container_status().network_infos(0); // The hook module sets up '4.3.2.1' as the IP address and 'public' as the - // network isolation group. + // network isolation group. The `ip_address` field is deprecated, but the + // hook module should continue to set it as well as the new `ip_addresses` + // field for now. EXPECT_TRUE(networkInfo.has_ip_address()); EXPECT_EQ("4.3.2.1", networkInfo.ip_address()); + EXPECT_EQ(1, networkInfo.ip_addresses().size()); + EXPECT_TRUE(networkInfo.ip_addresses(0).has_ip_address()); + EXPECT_EQ("4.3.2.1", networkInfo.ip_addresses(0).ip_address()); + EXPECT_EQ(1, networkInfo.groups().size()); EXPECT_EQ("public", networkInfo.groups(0));
