[mesos] branch master updated: Removed an unconditional .get() in DefaultExecutor.
This is an automated email from the ASF dual-hosted git repository. alexr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 221884c Removed an unconditional .get() in DefaultExecutor. 221884c is described below commit 221884c88447f280dfa61cbbb465d9342f155f61 Author: Benno Evers AuthorDate: Thu Dec 13 10:39:16 2018 +0100 Removed an unconditional .get() in DefaultExecutor. This would fail in some cases where the optional message is `None`, e.g. when the container is in killing state. Review: https://reviews.apache.org/r/66815/ --- src/launcher/default_executor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/launcher/default_executor.cpp b/src/launcher/default_executor.cpp index b89b036..cc7b6b7 100644 --- a/src/launcher/default_executor.cpp +++ b/src/launcher/default_executor.cpp @@ -953,9 +953,9 @@ protected: forward(taskStatus); LOG(INFO) - << "Child container " << container->containerId << " of task '" << taskId - << "' completed in state " << stringify(taskState) - << ": " << message.get(); + << "Child container " << container->containerId << " of task '" + << taskId << "' completed in state " << stringify(taskState) + << (message.isSome() ? ": " + message.get() : ""); // The default restart policy for a task group is to kill all the // remaining child containers if one of them terminated with a
[mesos] branch master updated: Fixed SIGINT handling in parallel test runner.
This is an automated email from the ASF dual-hosted git repository.
bbannier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 3ade731 Fixed SIGINT handling in parallel test runner.
3ade731 is described below
commit 3ade731d0c1772206c4afdf56318cfab6356acee
Author: Andrei Budnik
AuthorDate: Thu Dec 13 13:57:50 2018 +0100
Fixed SIGINT handling in parallel test runner.
Previously, we were changing SIGINT signal handling by setting the
`SIG_IGN` flag in every worker from the worker pool. Hence, sending
SIGINT to `mesos-gtest-runner.py` led to leaks of `mesos-tests`
processes. This patch removes this handler to fix the issue.
This closes #321
---
support/mesos-gtest-runner.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/support/mesos-gtest-runner.py b/support/mesos-gtest-runner.py
index 94cc1a6..88a60c3 100755
--- a/support/mesos-gtest-runner.py
+++ b/support/mesos-gtest-runner.py
@@ -32,7 +32,6 @@ import multiprocessing
import os
import resource
import shlex
-import signal
import subprocess
import sys
@@ -73,8 +72,6 @@ def run_test(opts):
"""
shard, nshards, args = opts
-signal.signal(signal.SIGINT, signal.SIG_IGN)
-
env = os.environ.copy()
env['GTEST_TOTAL_SHARDS'] = str(nshards)
env['GTEST_SHARD_INDEX'] = str(shard)
@@ -88,6 +85,8 @@ def run_test(opts):
print(Bcolors.colorize('.', Bcolors.OKGREEN), end='')
sys.stdout.flush()
return True, output
+except KeyboardInterrupt:
+return False
except subprocess.CalledProcessError as error:
print(Bcolors.colorize('.', Bcolors.FAIL), end='')
sys.stdout.flush()
[mesos] 03/05: Set up the `Resource.DiskInfo.Source.vendor` field in SLRP.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 3c0b147fe3409984e78eb9b43b45253ea6b89883
Author: Chun-Hung Hsiao
AuthorDate: Thu Dec 6 15:28:16 2018 -0800
Set up the `Resource.DiskInfo.Source.vendor` field in SLRP.
For a CSI-backed disk resource, this field is set to the concatenation
of the type and name of its CSI plugin, with a dot in between. This
enables frameworks to identify the storage backend the disk resource.
Review: https://reviews.apache.org/r/69520
---
src/resource_provider/storage/provider.cpp | 33 ++
1 file changed, 25 insertions(+), 8 deletions(-)
diff --git a/src/resource_provider/storage/provider.cpp
b/src/resource_provider/storage/provider.cpp
index 93df4a2..a151ead 100644
--- a/src/resource_provider/storage/provider.cpp
+++ b/src/resource_provider/storage/provider.cpp
@@ -246,32 +246,41 @@ static inline Resource createRawDiskResource(
const ResourceProviderInfo& info,
const Bytes& capacity,
const Option& profile,
+const Option& vendor,
const Option& id = None(),
const Option& metadata = None())
{
CHECK(info.has_id());
+ CHECK(info.has_storage());
Resource resource;
resource.set_name("disk");
resource.set_type(Value::SCALAR);
resource.mutable_scalar()
-->set_value((double) capacity.bytes() / Bytes::MEGABYTES);
+->set_value(static_cast(capacity.bytes()) / Bytes::MEGABYTES);
+
resource.mutable_provider_id()->CopyFrom(info.id()),
resource.mutable_reservations()->CopyFrom(info.default_reservations());
- resource.mutable_disk()->mutable_source()
-->set_type(Resource::DiskInfo::Source::RAW);
+
+ Resource::DiskInfo::Source* source =
+resource.mutable_disk()->mutable_source();
+
+ source->set_type(Resource::DiskInfo::Source::RAW);
if (profile.isSome()) {
-resource.mutable_disk()->mutable_source()->set_profile(profile.get());
+source->set_profile(profile.get());
+ }
+
+ if (vendor.isSome()) {
+source->set_vendor(vendor.get());
}
if (id.isSome()) {
-resource.mutable_disk()->mutable_source()->set_id(id.get());
+source->set_id(id.get());
}
if (metadata.isSome()) {
-resource.mutable_disk()->mutable_source()->mutable_metadata()
- ->CopyFrom(metadata.get());
+source->mutable_metadata()->CopyFrom(metadata.get());
}
return resource;
@@ -296,6 +305,9 @@ public:
metaDir(slave::paths::getMetaRootDir(_workDir)),
contentType(ContentType::PROTOBUF),
info(_info),
+ vendor(
+ info.storage().plugin().type() + "." +
+ info.storage().plugin().name()),
slaveId(_slaveId),
authToken(_authToken),
strict(_strict),
@@ -455,6 +467,7 @@ private:
const string metaDir;
const ContentType contentType;
ResourceProviderInfo info;
+ const string vendor;
const SlaveID slaveId;
const Option authToken;
const bool strict;
@@ -1374,6 +1387,8 @@ ResourceConversion
StorageLocalResourceProviderProcess::reconcileResources(
Bytes(resource.scalar().value() * Bytes::MEGABYTES),
resource.disk().source().has_profile()
? resource.disk().source().profile() : Option::none(),
+resource.disk().source().has_vendor()
+ ? resource.disk().source().vendor() : Option::none(),
resource.disk().source().has_id()
? resource.disk().source().id() : Option::none(),
resource.disk().source().has_metadata()
@@ -2913,6 +2928,7 @@ Future
StorageLocalResourceProviderProcess::listVolumes()
volumesToProfiles.contains(entry.volume().id())
? volumesToProfiles.at(entry.volume().id())
: Option::none(),
+vendor,
entry.volume().id(),
entry.volume().attributes().empty()
? Option::none()
@@ -2960,7 +2976,8 @@ Future
StorageLocalResourceProviderProcess::getCapacities()
return createRawDiskResource(
info,
Bytes(response.available_capacity()),
-profile);
+profile,
+vendor);
})));
}
[mesos] branch 1.7.x updated (4afe10a -> 2b26edd)
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a change to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 4afe10a Added UCR bridge network for Mesos Mini. new 8785ec5 Added MESOS-9340 to the 1.7.1 CHANGELOG. new 5e49943 Added an optional `vendor` field to `Resource.DiskInfo.Source`. new 3c0b147 Set up the `Resource.DiskInfo.Source.vendor` field in SLRP. new 30dbd96 Fixed `AgentResourceProviderConfigApiTest.Update` for `vendor` field. new 2b26edd Added MESOS-9321 to the 1.7.1 CHANGELOG. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 2 + include/mesos/mesos.proto | 28 include/mesos/v1/mesos.proto | 28 src/common/resources.cpp | 36 +++ src/resource_provider/storage/provider.cpp | 33 ++ .../agent_resource_provider_config_api_tests.cpp | 51 -- src/v1/resources.cpp | 36 +++ 7 files changed, 141 insertions(+), 73 deletions(-)
[mesos] branch master updated (3ade731 -> b6fc970)
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 3ade731 Fixed SIGINT handling in parallel test runner. new 65ec2b0 Added an optional `vendor` field to `Resource.DiskInfo.Source`. new 48211c1 Set up the `Resource.DiskInfo.Source.vendor` field in SLRP. new fd5b28b Fixed `AgentResourceProviderConfigApiTest.Update` for `vendor` field. new a3a8b5f Added MESOS-9321 to the 1.7.1 CHANGELOG. new 7c852a9 Tested the `Resource.DiskInfo.Source.vendor` field in related tests. new b6fc970 Added the missing MESOS-7947 to the 1.7.0 CHANGELOG. The 6 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 7 +++ include/mesos/mesos.proto | 28 include/mesos/v1/mesos.proto | 28 src/common/resources.cpp | 36 +++ src/resource_provider/storage/provider.cpp | 33 ++ .../agent_resource_provider_config_api_tests.cpp | 49 +++- src/tests/resources_tests.cpp | 32 - .../storage_local_resource_provider_tests.cpp | 53 +++--- src/v1/resources.cpp | 36 +++ 9 files changed, 213 insertions(+), 89 deletions(-)
[mesos] 06/06: Added the missing MESOS-7947 to the 1.7.0 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit b6fc9706c03c93239128f4a27051997c9647d8ff Author: Joseph Wu AuthorDate: Wed Sep 5 16:52:40 2018 -0700 Added the missing MESOS-7947 to the 1.7.0 CHANGELOG. --- CHANGELOG | 6 ++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 3a517c1..434b0df 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -81,6 +81,11 @@ This release contains the following highlights: along with other cgroups related options (e.g., `cgroups/cpu`), those options will be just ignored. +* [MESOS-7947] - Added a new `--gc_non_executor_container_sandboxes` + option which tells the agent to garbage collect sandboxes created + via the LAUNCH_NESTED_CONTAINER API. The same flag will apply to + standalone container sandboxes in future. + * [MESOS-8327] - Added container-specific cgroups mounts under `/sys/fs/cgroup` to containers with image launched by Mesos containerizer. @@ -299,6 +304,7 @@ All Resolved Issues: ** Improvement * [MESOS-6451] - Add timer and percentile for docker pull latency distribution. * [MESOS-7691] - Support local enabled cgroups subsystems automatically. + * [MESOS-7947] - Add GC capability to nested containers * [MESOS-8064] - Add capability so mesos can programmatically decode .zip, .tar, .gzip, and other common file compression schemes * [MESOS-8106] - Docker fetcher plugin unsupported scheme failure message is not accurate. * [MESOS-8340] - Add a no-enforce option to the `network/ports` isolator.
[mesos] 03/06: Fixed `AgentResourceProviderConfigApiTest.Update` for `vendor` field.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit fd5b28b5176efb03d6b68921f2c264e3c0066ad2
Author: Chun-Hung Hsiao
AuthorDate: Thu Dec 6 20:20:46 2018 -0800
Fixed `AgentResourceProviderConfigApiTest.Update` for `vendor` field.
The introduction of the `Resource.DiskInfo.Source.vendor` changes the
resource comparison and thus breaks this test. This patch fixes the test
and make it more robust.
NOTE: The updated test will fail unless the previous patch (which sets
up the `vendor` field) is also applied.
Review: https://reviews.apache.org/r/69521
---
.../agent_resource_provider_config_api_tests.cpp | 49 --
1 file changed, 37 insertions(+), 12 deletions(-)
diff --git a/src/tests/agent_resource_provider_config_api_tests.cpp
b/src/tests/agent_resource_provider_config_api_tests.cpp
index 15c83b3..7185ac0 100644
--- a/src/tests/agent_resource_provider_config_api_tests.cpp
+++ b/src/tests/agent_resource_provider_config_api_tests.cpp
@@ -151,17 +151,18 @@ public:
// This extra closure is necessary in order to use `ASSERT_*`, as
// these macros require a void return type.
[&] {
- // Randomize the plugin name so we get a clean work directory for
- // each created config.
+ // Randomize the plugin name so every created config uses its own CSI
+ // plugin instance. We use this to check if the config has been updated
in
+ // some tests.
const string testCsiPluginName =
-"test_csi_plugin_" +
-strings::remove(id::UUID::random().toString(), "-");
+"local_" + strings::remove(id::UUID::random().toString(), "-");
const string testCsiPluginPath =
path::join(tests::flags.build_dir, "src", "test-csi-plugin");
const string testCsiPluginWorkDir =
path::join(sandbox.get(), testCsiPluginName);
+
ASSERT_SOME(os::mkdir(testCsiPluginWorkDir));
Try resourceProviderConfig = strings::format(
@@ -579,11 +580,13 @@ TEST_P(AgentResourceProviderConfigApiTest, Update)
Owned detector = master.get()->createDetector();
// Generate a pre-existing config.
+ const string volumes = "volume1:4GB";
const string configPath =
path::join(resourceProviderConfigDir.get(), "test.json");
+
ASSERT_SOME(os::write(
configPath,
- stringify(JSON::protobuf(createResourceProviderInfo("volume1:4GB");
+ stringify(JSON::protobuf(createResourceProviderInfo(volumes);
Future slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
@@ -616,14 +619,22 @@ TEST_P(AgentResourceProviderConfigApiTest, Update)
Future> oldOffers;
EXPECT_CALL(sched, resourceOffers(&driver, OffersHaveAnyResource(
- std::bind(&Resources::hasResourceProvider, lambda::_1
+ &Resources::hasResourceProvider)))
.WillOnce(FutureArg<1>(&oldOffers));
driver.start();
// Wait for an offer having the old provider resource.
AWAIT_READY(oldOffers);
- ASSERT_FALSE(oldOffers->empty());
+ ASSERT_EQ(1u, oldOffers->size());
+
+ Resource oldResource = *Resources(oldOffers->at(0).resources())
+.filter(&Resources::hasResourceProvider)
+.begin();
+
+ ASSERT_TRUE(oldResource.has_disk());
+ ASSERT_TRUE(oldResource.disk().has_source());
+ ASSERT_TRUE(oldResource.disk().source().has_vendor());
Future rescinded;
@@ -636,7 +647,8 @@ TEST_P(AgentResourceProviderConfigApiTest, Update)
std::bind(&Resources::hasResourceProvider, lambda::_1
.WillOnce(FutureArg<1>(&newOffers));
- ResourceProviderInfo info = createResourceProviderInfo("volume1:2GB");
+ // Create a new config that serves the same volumes with a different vendor.
+ ResourceProviderInfo info = createResourceProviderInfo(volumes);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(
http::OK().status,
@@ -655,6 +667,7 @@ TEST_P(AgentResourceProviderConfigApiTest, Update)
Try _info =
::protobuf::parse(json.get());
+
ASSERT_SOME(_info);
EXPECT_EQ(_info.get(), info);
@@ -663,10 +676,22 @@ TEST_P(AgentResourceProviderConfigApiTest, Update)
// Wait for an offer having the new provider resource.
AWAIT_READY(newOffers);
-
- // The new provider resource is smaller than the old provider resource.
- EXPECT_FALSE(Resources(newOffers->at(0).resources()).contains(
- oldOffers->at(0).resources()));
+ ASSERT_EQ(1u, newOffers->size());
+
+ Resource newResource = *Resources(newOffers->at(0).resources())
+.filter(&Resources::hasResourceProvider)
+.begin();
+
+ ASSERT_TRUE(newResource.has_disk());
+ ASSERT_TRUE(newResource.disk().has_source());
+ ASSERT_TRUE(newResource.disk().source().has_vendor());
+
+ // The resource from the new resource provider has the same provider ID but a
+ // different vendor as that from the old one.
+ EXPECT_EQ(oldResource.provider_id(), newResource.provide
[mesos] 02/05: Added an optional `vendor` field to `Resource.DiskInfo.Source`.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 5e49943cfb15684dbea91c3323dfee68ca9b31e0
Author: Chun-Hung Hsiao
AuthorDate: Thu Nov 15 23:06:22 2018 -0800
Added an optional `vendor` field to `Resource.DiskInfo.Source`.
The new `vendor` field together with the `id` field provide the means of
uniquely identifying a CSI volume upon a CSI plugin migration (e.g.,
there is a change in the agent ID).
Review: https://reviews.apache.org/r/69037/
---
include/mesos/mesos.proto| 28
include/mesos/v1/mesos.proto | 28
src/common/resources.cpp | 36 ++--
src/v1/resources.cpp | 36 ++--
4 files changed, 76 insertions(+), 52 deletions(-)
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 043a737..e984541 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1026,7 +1026,7 @@ message CSIPluginContainerInfo {
* Describes a CSI plugin.
*/
message CSIPluginInfo {
- // The type of the CSI service. This uniquely identifies a CSI
+ // The type of the CSI plugin. This uniquely identifies a CSI
// implementation. For instance:
// org.apache.mesos.csi.test
//
@@ -1035,11 +1035,17 @@ message CSIPluginInfo {
// to avoid conflicts on type names.
required string type = 1;
- // The name of the CSI service. There could be mutliple instances of a
- // type of CSI service. The name field is used to distinguish these
- // instances. It should be a legal Java identifier
+ // The name of the CSI plugin. There could be multiple instances of a
+ // type of CSI plugin within a Mesos cluster. The name field is used to
+ // distinguish these instances. It should be a legal Java identifier
// (https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html)
// to avoid conflicts on concatenation of type and name.
+ //
+ // The type and name together provide the means to uniquely identify a
storage
+ // backend and its resources in the cluster, so the operator should ensure
+ // that the concatenation of type and name is unique in the cluster, and it
+ // remains the same if the instance is migrated to another agent (e.g., there
+ // is a change in the agent ID).
required string name = 2;
// A list of container configurations to run CSI plugin components.
@@ -1451,12 +1457,18 @@ message Resource {
optional Path path = 2;
optional Mount mount = 3;
- // An identifier for this source. This field maps onto CSI
- // volume IDs and is not expected to be set by frameworks.
+ // The vendor of this source. If present, this field provides the means
to
+ // uniquely identify the storage backend of this source in the cluster.
+ optional string vendor = 7; // EXPERIMENTAL.
+
+ // The identifier of this source. This field maps onto CSI volume IDs and
+ // is not expected to be set by frameworks. If both `vendor` and `id` are
+ // present, these two fields together provide the means to uniquely
+ // identify this source in the cluster.
optional string id = 4; // EXPERIMENTAL.
- // Additional metadata for this source. This field maps onto CSI
- // volume metadata and is not expected to be set by frameworks.
+ // Additional metadata for this source. This field maps onto CSI volume
+ // attributes and is not expected to be set by frameworks.
optional Labels metadata = 5; // EXPERIMENTAL.
// This field serves as an indirection to a set of storage
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 9830c0b..dac7f51 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1018,7 +1018,7 @@ message CSIPluginContainerInfo {
* Describes a CSI plugin.
*/
message CSIPluginInfo {
- // The type of the CSI service. This uniquely identifies a CSI
+ // The type of the CSI plugin. This uniquely identifies a CSI
// implementation. For instance:
// org.apache.mesos.csi.test
//
@@ -1027,11 +1027,17 @@ message CSIPluginInfo {
// to avoid conflicts on type names.
required string type = 1;
- // The name of the CSI service. There could be mutliple instances of a
- // type of CSI service. The name field is used to distinguish these
- // instances. It should be a legal Java identifier
+ // The name of the CSI plugin. There could be multiple instances of a
+ // type of CSI plugin within a Mesos cluster. The name field is used to
+ // distinguish these instances. It should be a legal Java identifier
// (https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html)
// to avoid conflicts on concatenation of type and name.
+ //
+ // The type and name together provide the means to uniqu
[mesos] 04/06: Added MESOS-9321 to the 1.7.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit a3a8b5f81963c5dd0344b91131defea53fd4f7c3 Author: Chun-Hung Hsiao AuthorDate: Thu Dec 13 14:24:23 2018 -0800 Added MESOS-9321 to the 1.7.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 85bc05a..3a517c1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -39,6 +39,7 @@ Release Notes - Mesos - Version 1.7.1 (WIP) * [MESOS-9249] - Avoid dirtying the DRF sorter when allocating resources. * [MESOS-9255] - Use consistent "totals" across role / framework DRF. * [MESOS-9305] - Create cgoup recursively to workaround systemd deleting cgroups_root. + * [MESOS-9321] - Add an optional `vendor` field in `Resource.DiskInfo.Source`. * [MESOS-9340] - Log all socket errors in libprocess.
[mesos] 02/06: Set up the `Resource.DiskInfo.Source.vendor` field in SLRP.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 48211c1def9cbd8885764fee136a644e29bd7d79
Author: Chun-Hung Hsiao
AuthorDate: Thu Dec 6 15:28:16 2018 -0800
Set up the `Resource.DiskInfo.Source.vendor` field in SLRP.
For a CSI-backed disk resource, this field is set to the concatenation
of the type and name of its CSI plugin, with a dot in between. This
enables frameworks to identify the storage backend the disk resource.
Review: https://reviews.apache.org/r/69520
---
src/resource_provider/storage/provider.cpp | 33 ++
1 file changed, 25 insertions(+), 8 deletions(-)
diff --git a/src/resource_provider/storage/provider.cpp
b/src/resource_provider/storage/provider.cpp
index 955471c..d6e20a5 100644
--- a/src/resource_provider/storage/provider.cpp
+++ b/src/resource_provider/storage/provider.cpp
@@ -246,32 +246,41 @@ static inline Resource createRawDiskResource(
const ResourceProviderInfo& info,
const Bytes& capacity,
const Option& profile,
+const Option& vendor,
const Option& id = None(),
const Option& metadata = None())
{
CHECK(info.has_id());
+ CHECK(info.has_storage());
Resource resource;
resource.set_name("disk");
resource.set_type(Value::SCALAR);
resource.mutable_scalar()
-->set_value((double) capacity.bytes() / Bytes::MEGABYTES);
+->set_value(static_cast(capacity.bytes()) / Bytes::MEGABYTES);
+
resource.mutable_provider_id()->CopyFrom(info.id()),
resource.mutable_reservations()->CopyFrom(info.default_reservations());
- resource.mutable_disk()->mutable_source()
-->set_type(Resource::DiskInfo::Source::RAW);
+
+ Resource::DiskInfo::Source* source =
+resource.mutable_disk()->mutable_source();
+
+ source->set_type(Resource::DiskInfo::Source::RAW);
if (profile.isSome()) {
-resource.mutable_disk()->mutable_source()->set_profile(profile.get());
+source->set_profile(profile.get());
+ }
+
+ if (vendor.isSome()) {
+source->set_vendor(vendor.get());
}
if (id.isSome()) {
-resource.mutable_disk()->mutable_source()->set_id(id.get());
+source->set_id(id.get());
}
if (metadata.isSome()) {
-resource.mutable_disk()->mutable_source()->mutable_metadata()
- ->CopyFrom(metadata.get());
+source->mutable_metadata()->CopyFrom(metadata.get());
}
return resource;
@@ -296,6 +305,9 @@ public:
metaDir(slave::paths::getMetaRootDir(_workDir)),
contentType(ContentType::PROTOBUF),
info(_info),
+ vendor(
+ info.storage().plugin().type() + "." +
+ info.storage().plugin().name()),
slaveId(_slaveId),
authToken(_authToken),
strict(_strict),
@@ -455,6 +467,7 @@ private:
const string metaDir;
const ContentType contentType;
ResourceProviderInfo info;
+ const string vendor;
const SlaveID slaveId;
const Option authToken;
const bool strict;
@@ -1378,6 +1391,8 @@ ResourceConversion
StorageLocalResourceProviderProcess::reconcileResources(
Bytes(resource.scalar().value() * Bytes::MEGABYTES),
resource.disk().source().has_profile()
? resource.disk().source().profile() : Option::none(),
+resource.disk().source().has_vendor()
+ ? resource.disk().source().vendor() : Option::none(),
resource.disk().source().has_id()
? resource.disk().source().id() : Option::none(),
resource.disk().source().has_metadata()
@@ -2917,6 +2932,7 @@ Future
StorageLocalResourceProviderProcess::listVolumes()
volumesToProfiles.contains(entry.volume().id())
? volumesToProfiles.at(entry.volume().id())
: Option::none(),
+vendor,
entry.volume().id(),
entry.volume().attributes().empty()
? Option::none()
@@ -2964,7 +2980,8 @@ Future
StorageLocalResourceProviderProcess::getCapacities()
return createRawDiskResource(
info,
Bytes(response.available_capacity()),
-profile);
+profile,
+vendor);
})));
}
[mesos] 01/06: Added an optional `vendor` field to `Resource.DiskInfo.Source`.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 65ec2b0149970d2790d18f9f3493b247c698baae
Author: Chun-Hung Hsiao
AuthorDate: Thu Nov 15 23:06:22 2018 -0800
Added an optional `vendor` field to `Resource.DiskInfo.Source`.
The new `vendor` field together with the `id` field provide the means of
uniquely identifying a CSI volume upon a CSI plugin migration (e.g.,
there is a change in the agent ID).
Review: https://reviews.apache.org/r/69037/
---
include/mesos/mesos.proto| 28
include/mesos/v1/mesos.proto | 28
src/common/resources.cpp | 36 ++--
src/v1/resources.cpp | 36 ++--
4 files changed, 76 insertions(+), 52 deletions(-)
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index ef4d785..137df1f 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1026,7 +1026,7 @@ message CSIPluginContainerInfo {
* Describes a CSI plugin.
*/
message CSIPluginInfo {
- // The type of the CSI service. This uniquely identifies a CSI
+ // The type of the CSI plugin. This uniquely identifies a CSI
// implementation. For instance:
// org.apache.mesos.csi.test
//
@@ -1035,11 +1035,17 @@ message CSIPluginInfo {
// to avoid conflicts on type names.
required string type = 1;
- // The name of the CSI service. There could be mutliple instances of a
- // type of CSI service. The name field is used to distinguish these
- // instances. It should be a legal Java identifier
+ // The name of the CSI plugin. There could be multiple instances of a
+ // type of CSI plugin within a Mesos cluster. The name field is used to
+ // distinguish these instances. It should be a legal Java identifier
// (https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html)
// to avoid conflicts on concatenation of type and name.
+ //
+ // The type and name together provide the means to uniquely identify a
storage
+ // backend and its resources in the cluster, so the operator should ensure
+ // that the concatenation of type and name is unique in the cluster, and it
+ // remains the same if the instance is migrated to another agent (e.g., there
+ // is a change in the agent ID).
required string name = 2;
// A list of container configurations to run CSI plugin components.
@@ -1452,12 +1458,18 @@ message Resource {
optional Path path = 2;
optional Mount mount = 3;
- // An identifier for this source. This field maps onto CSI
- // volume IDs and is not expected to be set by frameworks.
+ // The vendor of this source. If present, this field provides the means
to
+ // uniquely identify the storage backend of this source in the cluster.
+ optional string vendor = 7; // EXPERIMENTAL.
+
+ // The identifier of this source. This field maps onto CSI volume IDs and
+ // is not expected to be set by frameworks. If both `vendor` and `id` are
+ // present, these two fields together provide the means to uniquely
+ // identify this source in the cluster.
optional string id = 4; // EXPERIMENTAL.
- // Additional metadata for this source. This field maps onto CSI
- // volume metadata and is not expected to be set by frameworks.
+ // Additional metadata for this source. This field maps onto CSI volume
+ // attributes and is not expected to be set by frameworks.
optional Labels metadata = 5; // EXPERIMENTAL.
// This field serves as an indirection to a set of storage
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 4e8f7a1..f121709 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1018,7 +1018,7 @@ message CSIPluginContainerInfo {
* Describes a CSI plugin.
*/
message CSIPluginInfo {
- // The type of the CSI service. This uniquely identifies a CSI
+ // The type of the CSI plugin. This uniquely identifies a CSI
// implementation. For instance:
// org.apache.mesos.csi.test
//
@@ -1027,11 +1027,17 @@ message CSIPluginInfo {
// to avoid conflicts on type names.
required string type = 1;
- // The name of the CSI service. There could be mutliple instances of a
- // type of CSI service. The name field is used to distinguish these
- // instances. It should be a legal Java identifier
+ // The name of the CSI plugin. There could be multiple instances of a
+ // type of CSI plugin within a Mesos cluster. The name field is used to
+ // distinguish these instances. It should be a legal Java identifier
// (https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html)
// to avoid conflicts on concatenation of type and name.
+ //
+ // The type and name together provide the means to uniq
[mesos] 05/06: Tested the `Resource.DiskInfo.Source.vendor` field in related tests.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 7c852a984c61f8ccd60eaf45749603f8f08e3043
Author: Chun-Hung Hsiao
AuthorDate: Thu Dec 6 15:26:34 2018 -0800
Tested the `Resource.DiskInfo.Source.vendor` field in related tests.
Review: https://reviews.apache.org/r/69522
---
src/tests/resources_tests.cpp | 32 -
.../storage_local_resource_provider_tests.cpp | 53 +++---
2 files changed, 68 insertions(+), 17 deletions(-)
diff --git a/src/tests/resources_tests.cpp b/src/tests/resources_tests.cpp
index ab42374..5337a5c 100644
--- a/src/tests/resources_tests.cpp
+++ b/src/tests/resources_tests.cpp
@@ -19,6 +19,7 @@
#include
#include
#include
+#include
#include
@@ -2370,10 +2371,11 @@ TEST(DiskResourcesTest, DiskSourceEquals)
class DiskResourcesSourceTest
: public ::testing::Test,
-public ::testing::WithParamInterface> {};
+bool, // Whether the disk has the `vendor` field set.
+bool, // Whether the disk has the `id` field set.
+bool>> {}; // Whether the disk has the `profile` field set.
INSTANTIATE_TEST_CASE_P(
@@ -2386,11 +2388,14 @@ INSTANTIATE_TEST_CASE_P(
Resource::DiskInfo::Source::PATH,
Resource::DiskInfo::Source::BLOCK,
Resource::DiskInfo::Source::MOUNT),
-// We test the case where the source has identity (i.e., has
-// an `id` set) and where not.
+// We test the cases where the source has a vendor (i.e., has the
+// `vendor` field set) and where not.
::testing::Bool(),
-// We test the case where the source has profile (i.e., has
-// an `profile` set) and where not.
+// We test the cases where the source has an identity (i.e., has the
+// `id` field set) and where not.
+::testing::Bool(),
+// We test the cases where the source has a profile (i.e., has the
+// `profile` field set) and where not.
::testing::Bool()));
@@ -2398,14 +2403,19 @@ TEST_P(DiskResourcesSourceTest, SourceIdentity)
{
auto parameters = GetParam();
- Resource::DiskInfo::Source::Type type = std::tr1::get<0>(parameters);
- bool hasIdentity = std::tr1::get<1>(parameters);
- bool hasProfile = std::tr1::get<2>(parameters);
+ Resource::DiskInfo::Source::Type type = std::get<0>(parameters);
+ bool hasVendor = std::get<1>(parameters);
+ bool hasIdentity = std::get<2>(parameters);
+ bool hasProfile = std::get<3>(parameters);
- // Create a disk, possibly with an id to signify identiy.
+ // Create a disk, possibly with an id to signify identity.
Resource::DiskInfo::Source source;
source.set_type(type);
+ if (hasVendor) {
+source.set_vendor("vendor");
+ }
+
if (hasIdentity) {
source.set_id("id");
}
diff --git a/src/tests/storage_local_resource_provider_tests.cpp
b/src/tests/storage_local_resource_provider_tests.cpp
index 422a152..7887bd5 100644
--- a/src/tests/storage_local_resource_provider_tests.cpp
+++ b/src/tests/storage_local_resource_provider_tests.cpp
@@ -81,6 +81,9 @@ constexpr char URI_DISK_PROFILE_ADAPTOR_NAME[] =
constexpr char TEST_SLRP_TYPE[] = "org.apache.mesos.rp.local.storage";
constexpr char TEST_SLRP_NAME[] = "test";
+constexpr char TEST_CSI_PLUGIN_TYPE[] = "org.apache.mesos.csi.test";
+constexpr char TEST_CSI_PLUGIN_NAME[] = "local";
+constexpr char TEST_CSI_VENDOR[] = "org.apache.mesos.csi.test.local";
class StorageLocalResourceProviderTest
@@ -196,13 +199,10 @@ public:
const Option volumes = None(),
const Option createParameters = None())
{
-const string testCsiPluginName = "test_csi_plugin";
-
const string testCsiPluginPath =
path::join(tests::flags.build_dir, "src", "test-csi-plugin");
-const string testCsiPluginWorkDir =
- path::join(sandbox.get(), testCsiPluginName);
+const string testCsiPluginWorkDir = path::join(sandbox.get(), "storage");
ASSERT_SOME(os::mkdir(testCsiPluginWorkDir));
Try resourceProviderConfig = strings::format(
@@ -218,7 +218,7 @@ public:
],
"storage": {
"plugin": {
- "type": "org.apache.mesos.csi.test",
+ "type": "%s",
"name": "%s",
"containers": [
{
@@ -261,7 +261,8 @@ public:
)~",
TEST_SLRP_TYPE,
TEST_SLRP_NAME,
-testCsiPluginName,
+TEST_CSI_PLUGIN_TYPE,
+TEST_CSI_PLUGIN_NAME,
testCsiPluginPath,
testCsiPluginPath,
stringify(capacity),
@@ -516,6 +517,8 @@ TEST_F(StorageLocalResourceProviderTest, DISABLED_SmallDisk)
return r.has_disk() &&
r.disk().has_source() &&
r.disk().source().type() == Resource::DiskInfo::Source::RAW &&
+ r.disk().source().has_vendor() &&
+ r.disk().source().vendor() ==
[mesos] 01/05: Added MESOS-9340 to the 1.7.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 8785ec52349728474e577dbfd5256d6dc224270c Author: Benjamin Mahler AuthorDate: Wed Oct 24 17:36:38 2018 -0700 Added MESOS-9340 to the 1.7.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index d530cb7..2dffc67 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -39,6 +39,7 @@ Release Notes - Mesos - Version 1.7.1 (WIP) * [MESOS-9249] - Avoid dirtying the DRF sorter when allocating resources. * [MESOS-9255] - Use consistent "totals" across role / framework DRF. * [MESOS-9305] - Create cgoup recursively to workaround systemd deleting cgroups_root. + * [MESOS-9340] - Log all socket errors in libprocess. Release Notes - Mesos - Version 1.7.0
[mesos] 05/05: Added MESOS-9321 to the 1.7.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 2b26edd6560fbd21aed9bf42652efe18d6e36433 Author: Chun-Hung Hsiao AuthorDate: Thu Dec 13 14:24:23 2018 -0800 Added MESOS-9321 to the 1.7.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 2dffc67..6f102a2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -39,6 +39,7 @@ Release Notes - Mesos - Version 1.7.1 (WIP) * [MESOS-9249] - Avoid dirtying the DRF sorter when allocating resources. * [MESOS-9255] - Use consistent "totals" across role / framework DRF. * [MESOS-9305] - Create cgoup recursively to workaround systemd deleting cgroups_root. + * [MESOS-9321] - Add an optional `vendor` field in `Resource.DiskInfo.Source`. * [MESOS-9340] - Log all socket errors in libprocess.
[mesos] 04/05: Fixed `AgentResourceProviderConfigApiTest.Update` for `vendor` field.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 30dbd9644a9f8b7078b7e81618c87f1a82c02971
Author: Chun-Hung Hsiao
AuthorDate: Thu Dec 6 20:20:46 2018 -0800
Fixed `AgentResourceProviderConfigApiTest.Update` for `vendor` field.
The introduction of the `Resource.DiskInfo.Source.vendor` changes the
resource comparison and thus breaks this test. This patch fixes the test
and make it more robust.
NOTE: The updated test will fail unless the previous patch (which sets
up the `vendor` field) is also applied.
Review: https://reviews.apache.org/r/69521
---
.../agent_resource_provider_config_api_tests.cpp | 51 --
1 file changed, 38 insertions(+), 13 deletions(-)
diff --git a/src/tests/agent_resource_provider_config_api_tests.cpp
b/src/tests/agent_resource_provider_config_api_tests.cpp
index e6a68ba..f55953e 100644
--- a/src/tests/agent_resource_provider_config_api_tests.cpp
+++ b/src/tests/agent_resource_provider_config_api_tests.cpp
@@ -85,18 +85,19 @@ public:
// This extra closure is necessary in order to use `ASSERT_*`, as
// these macros require a void return type.
-[&]() {
- // Randomize the plugin name so we get a clean work directory for
- // each created config.
+[&] {
+ // Randomize the plugin name so every created config uses its own CSI
+ // plugin instance. We use this to check if the config has been updated
in
+ // some tests.
const string testCsiPluginName =
-"test_csi_plugin_" +
-strings::remove(id::UUID::random().toString(), "-");
+"local_" + strings::remove(id::UUID::random().toString(), "-");
const string testCsiPluginPath =
path::join(tests::flags.build_dir, "src", "test-csi-plugin");
const string testCsiPluginWorkDir =
path::join(sandbox.get(), testCsiPluginName);
+
ASSERT_SOME(os::mkdir(testCsiPluginWorkDir));
Try resourceProviderConfig = strings::format(
@@ -508,10 +509,12 @@ TEST_P(AgentResourceProviderConfigApiTest, ROOT_Update)
slaveFlags.resource_provider_config_dir = resourceProviderConfigDir;
// Generate a pre-existing config.
+ const string volumes = "volume1:4GB";
const string configPath = path::join(resourceProviderConfigDir, "test.json");
+
ASSERT_SOME(os::write(
configPath,
- stringify(JSON::protobuf(createResourceProviderInfo("volume1:4GB");
+ stringify(JSON::protobuf(createResourceProviderInfo(volumes);
Future slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
@@ -544,14 +547,22 @@ TEST_P(AgentResourceProviderConfigApiTest, ROOT_Update)
Future> oldOffers;
EXPECT_CALL(sched, resourceOffers(&driver, OffersHaveAnyResource(
- std::bind(&Resources::hasResourceProvider, lambda::_1
+ &Resources::hasResourceProvider)))
.WillOnce(FutureArg<1>(&oldOffers));
driver.start();
// Wait for an offer having the old provider resource.
AWAIT_READY(oldOffers);
- ASSERT_FALSE(oldOffers->empty());
+ ASSERT_EQ(1u, oldOffers->size());
+
+ Resource oldResource = *Resources(oldOffers->at(0).resources())
+.filter(&Resources::hasResourceProvider)
+.begin();
+
+ ASSERT_TRUE(oldResource.has_disk());
+ ASSERT_TRUE(oldResource.disk().has_source());
+ ASSERT_TRUE(oldResource.disk().source().has_vendor());
Future rescinded;
@@ -564,7 +575,8 @@ TEST_P(AgentResourceProviderConfigApiTest, ROOT_Update)
std::bind(&Resources::hasResourceProvider, lambda::_1
.WillOnce(FutureArg<1>(&newOffers));
- ResourceProviderInfo info = createResourceProviderInfo("volume1:2GB");
+ // Create a new config that serves the same volumes with a different vendor.
+ ResourceProviderInfo info = createResourceProviderInfo(volumes);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(
http::OK().status,
@@ -585,6 +597,7 @@ TEST_P(AgentResourceProviderConfigApiTest, ROOT_Update)
Try _info =
::protobuf::parse(json.get());
+
ASSERT_SOME(_info);
EXPECT_EQ(_info.get(), info);
@@ -593,10 +606,22 @@ TEST_P(AgentResourceProviderConfigApiTest, ROOT_Update)
// Wait for an offer having the new provider resource.
AWAIT_READY(newOffers);
-
- // The new provider resource is smaller than the old provider resource.
- EXPECT_FALSE(Resources(newOffers->at(0).resources()).contains(
- oldOffers->at(0).resources()));
+ ASSERT_EQ(1u, newOffers->size());
+
+ Resource newResource = *Resources(newOffers->at(0).resources())
+.filter(&Resources::hasResourceProvider)
+.begin();
+
+ ASSERT_TRUE(newResource.has_disk());
+ ASSERT_TRUE(newResource.disk().has_source());
+ ASSERT_TRUE(newResource.disk().source().has_vendor());
+
+ // The resource from the new resource provider has the same provider ID but a
+ // different vendor as that from the old one.
+ EXPECT_EQ(
