[mesos] 01/02: Added a test to ensure correct headroom accounting.
This is an automated email from the ASF dual-hosted git repository. mzhu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit a0170bb17656089ce07b6ccb30e7d23062d729d1 Author: Meng Zhu AuthorDate: Wed Apr 3 12:31:27 2019 -0700 Added a test to ensure correct headroom accounting. This test verifies that the headroom accounting is correct in the presence of subrole allocation. In particular, we need to ensure that a subrole's allocation is accounted only once (not multiple times in itself as well as all of its ancestors) when calculating available the quota headroom. Review: https://reviews.apache.org/r/70390 --- src/tests/hierarchical_allocator_tests.cpp | 97 ++ 1 file changed, 97 insertions(+) diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp index 30c0c3e..fb22319 100644 --- a/src/tests/hierarchical_allocator_tests.cpp +++ b/src/tests/hierarchical_allocator_tests.cpp @@ -6187,6 +6187,103 @@ TEST_F(HierarchicalAllocatorTest, QuotaWithNestedRoleReservation) } +// This test ensures that quota headroom is calculated correctly +// in the presence of subrole's allocations. +TEST_F(HierarchicalAllocatorTest, QuotaHeadroomWithNestedRoleAllocation) +{ + // Setup: + // agents: 2 * R + //roles: "a" --> allocated R + // "a/b" --> allocated R + // "quota-role" --> guarantee R w/ no framework + // + // Test: Add 1 more agent with R. + // Ensure agent is held back for "quota-role". + // Add 1 more agent with R. + // Ensure only 1 of the two extra agents goes to "a" or "a/b" + // (since there is enough headroom for "quota-role") + + Clock::pause(); + + initialize(); + + const string PARENT_ROLE = "a"; + const string CHILD_ROLE = "a/b"; + + // Add framework1 under the parent role "a/b". + FrameworkInfo framework1 = createFrameworkInfo({CHILD_ROLE}); + allocator->addFramework(framework1.id(), framework1, {}, true, {}); + + SlaveInfo agent1 = createSlaveInfo("cpus:1;mem:100"); + allocator->addSlave( + agent1.id(), + agent1, + AGENT_CAPABILITIES(), + None(), + agent1.resources(), + {}); + + // All the resources of agent1 are offered to framework1. + Allocation expected = Allocation( + framework1.id(), {{CHILD_ROLE, {{agent1.id(), agent1.resources()); + + AWAIT_EXPECT_EQ(expected, allocations.get()); + + // Add framework2 under the child role "a". + FrameworkInfo framework2 = createFrameworkInfo({PARENT_ROLE}); + allocator->addFramework(framework2.id(), framework2, {}, true, {}); + + // Add `agent2` which will be allocated to `framework2`. + SlaveInfo agent2 = createSlaveInfo("cpus:1;mem:100"); + allocator->addSlave( + agent2.id(), + agent2, + AGENT_CAPABILITIES(), + None(), + agent2.resources(), + {}); + + // All the resources of agent2 are offered to framework2. + expected = Allocation( + framework2.id(), {{PARENT_ROLE, {{agent2.id(), agent2.resources()); + + AWAIT_EXPECT_EQ(expected, allocations.get()); + + const string QUOTA_ROLE{"quota-role"}; + + Quota quota = createQuota(QUOTA_ROLE, "cpus:1;mem:100"); + allocator->setQuota(QUOTA_ROLE, quota); + + SlaveInfo agent3 = createSlaveInfo("cpus:1;mem:100"); + allocator->addSlave( + agent3.id(), + agent3, + AGENT_CAPABILITIES(), + None(), + agent3.resources(), + {}); + + Future allocation = allocations.get(); + EXPECT_TRUE(allocation.isPending()); + + SlaveInfo agent4 = createSlaveInfo("cpus:1;mem:100"); + allocator->addSlave( + agent4.id(), + agent4, + AGENT_CAPABILITIES(), + None(), + agent4.resources(), + {}); + + Clock::settle(); + + // There should be one allocation made. Either agent3 or agent4 is + // allocated and the other agent is set aside for the quota headroom. + AWAIT_READY(allocation); + EXPECT_TRUE(allocations.get().isPending()); +} + + // This test checks that quota guarantees work as expected when a // nested role is created as a child of an existing quota'd role. //
[mesos] branch master updated (3d1c435 -> 281039c)
This is an automated email from the ASF dual-hosted git repository. mzhu pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 3d1c435 Updated Mesos version to 1.9.0. new a0170bb Added a test to ensure correct headroom accounting. new 281039c Added a test to ensure correct role consumed quota accounting. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/tests/hierarchical_allocator_tests.cpp | 174 + 1 file changed, 174 insertions(+)
[mesos] 02/02: Added a test to ensure correct role consumed quota accounting.
This is an automated email from the ASF dual-hosted git repository. mzhu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 281039cb1b64ea916da11a9009f2895125242fc6 Author: Meng Zhu AuthorDate: Thu Apr 4 16:18:19 2019 -0700 Added a test to ensure correct role consumed quota accounting. This test ensures that a nested role's allocation is accounted to the top-level role's consumed quota. Review: https://reviews.apache.org/r/70396 --- src/tests/hierarchical_allocator_tests.cpp | 77 ++ 1 file changed, 77 insertions(+) diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp index fb22319..7221a64 100644 --- a/src/tests/hierarchical_allocator_tests.cpp +++ b/src/tests/hierarchical_allocator_tests.cpp @@ -6186,6 +6186,83 @@ TEST_F(HierarchicalAllocatorTest, QuotaWithNestedRoleReservation) AWAIT_EXPECT_EQ(expected, allocations.get()); } +// This test ensures that nested role's allocation is accounted +// to top-level role's consumed quota. +TEST_F(HierarchicalAllocatorTest, QuotaWithNestedRoleAllocation) +{ + // Setup: + // agent1: R + // Roles: + // "a" --> guarantee with R w/ no framework + // "a/b" --> has `framework1`, allocated R on agent1 + // + // Test: + // Add `framework2` under "a" + // Add agent2 with R + // Ensure: + // `agent2` is allocated to `framework1` under `a/b`, even though, role + // `a` and `framework2` have lower shares. This is because role `a` has + // reached its quota limit (due to its subrole's allocation). Also, + // currently, subrole's allocations are not constrained by top-level + // role's quota (though they are tracked post factum). + // + // TODO(mzhu): Once we finish support for hierarchical quota, no allocation + // should be made since "a/b" will also be bound by the quota of "a". + + // --- SET UP --- + + Clock::pause(); + + initialize(); + + const string PARENT_ROLE{"a"}; + const string CHILD_ROLE{"a/b"}; + + // Add framework1 under the child role "a/b". + FrameworkInfo framework1 = createFrameworkInfo({CHILD_ROLE}); + allocator->addFramework(framework1.id(), framework1, {}, true, {}); + + SlaveInfo agent1 = createSlaveInfo("cpus:1;mem:1024"); + allocator->addSlave( + agent1.id(), + agent1, + AGENT_CAPABILITIES(), + None(), + agent1.resources(), + {}); + + Allocation expected = Allocation( + framework1.id(), {{CHILD_ROLE, {{agent1.id(), agent1.resources()); + + AWAIT_EXPECT_EQ(expected, allocations.get()); + + Quota quota = createQuota(PARENT_ROLE, "cpus:1;mem:1024"); + allocator->setQuota(PARENT_ROLE, quota); + + // --- TEST --- + + // Add framework2 under the parent role "a". + FrameworkInfo framework2 = createFrameworkInfo({PARENT_ROLE}); + allocator->addFramework(framework2.id(), framework2, {}, true, {}); + + SlaveInfo agent2 = createSlaveInfo("cpus:1;mem:1024"); + allocator->addSlave( + agent2.id(), + agent2, + AGENT_CAPABILITIES(), + None(), + agent2.resources(), + {}); + + // Process all events. + Clock::settle(); + + expected = Allocation( + framework1.id(), {{CHILD_ROLE, {{agent2.id(), agent2.resources()); + + AWAIT_EXPECT_EQ(expected, allocations.get()); +} + // This test ensures that quota headroom is calculated correctly // in the presence of subrole's allocations.
[mesos-site] branch asf-site updated: Updated the website built from mesos SHA: 3d1c435.
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/mesos-site.git The following commit(s) were added to refs/heads/asf-site by this push: new ac1ca18 Updated the website built from mesos SHA: 3d1c435. ac1ca18 is described below commit ac1ca181c8dd13a4d28cb3ef39eec50b2eadc054 Author: jenkins AuthorDate: Fri Apr 5 21:49:49 2019 + Updated the website built from mesos SHA: 3d1c435. --- content/api/latest/java/constant-values.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/api/latest/java/constant-values.html b/content/api/latest/java/constant-values.html index b9d8538..915f51d 100644 --- a/content/api/latest/java/constant-values.html +++ b/content/api/latest/java/constant-values.html @@ -92,7 +92,7 @@ publicstaticfinaljava.lang.String VERSION -"1.8.0" +"1.9.0"
[mesos] 01/01: Updated CHANGELOG for 1.8.0.
This is an automated email from the ASF dual-hosted git repository. bennoe pushed a commit to branch 1.8.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 2c7cee0fded247da6f8f66e75fc759f0b11be42f Author: Benno Evers AuthorDate: Fri Apr 5 22:58:51 2019 +0200 Updated CHANGELOG for 1.8.0. --- CHANGELOG | 295 +- 1 file changed, 293 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 9d11de1..e0eb881 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,5 @@ -Release Notes - Mesos - Version 1.8.0 (WIP) +Release Notes - Mesos - Version 1.8.0 +- This release contains the following highlights: * Performance Improvements: @@ -13,6 +13,297 @@ This release contains the following highlights: the scheduler re-subscribes each time it wants to mutate the minimum resource quantity offer filter information, see MESOS-7258. +Unresolved Critical Issues: + * [MESOS-9697] Release RPMs are not uploaded to bintray + * [MESOS-9672] Docker containerizer should ignore pids of executors that do not pass the connection check. + * [MESOS-9667] Check failure when executor for task using resource provider resources subscribes before agent is registered + * [MESOS-9654] `PUBLISH_RESOURCES` should fail if the resource version changes. + * [MESOS-9619] Mesos Master Crashes with Launch Group when using Port Resources + * [MESOS-9616] `Filters.refuse_seconds` declines resources not in offers. + * [MESOS-9609] Master check failure when marking agent unreachable + * [MESOS-9579] ExecutorHttpApiTest.HeartbeatCalls is flaky. + * [MESOS-9560] ContentType/AgentAPITest.MarkResourceProviderGone/1 is flaky + * [MESOS-9536] Nested container launched with non-root user may not be able to write to its sandbox via the environment variable + * [MESOS-9520] IOTest.Read hangs on Windows + * [MESOS-9500] spark submit with docker image on mesos cluster fails. + * [MESOS-9426] ZK master detection can become forever pending. + * [MESOS-9393] Fetcher crashes extracting archives with non-ASCII filenames. + * [MESOS-9365] Windows - GET_CONTAINERS API call causes the Mesos agent to fail + * [MESOS-9355] Persistence volume does not unmount correctly with wrong artifact URI + * [MESOS-9352] Data in persistent volume deleted accidentally when using Docker container and Persistent volume + * [MESOS-9306] Mesos containerizer can get stuck during cgroup cleanup + * [MESOS-9180] tasks get stuck in TASK_KILLING on the default executor + * [MESOS-9053] Network ports isolator can falsely trigger while destroying containers. + * [MESOS-9006] The agent's GET_AGENT leaks resource information when using authorization + * [MESOS-8946] CURL 7.58 causes Mesos to fail decoding raw responses. + * [MESOS-8840] `cpu.cfs_quota_us` may be accidentally set for command task using docker during agent recovery. + * [MESOS-8803] Libprocess deadlocks in a test. + * [MESOS-8769] Agent crashes when CNI config not defined + * [MESOS-8679] If the first KILL stuck in the default executor, all other KILLs will be ignored. + * [MESOS-8608] RmdirContinueOnErrorTest.RemoveWithContinueOnError fails. + * [MESOS-8467] Destroyed executors might be used after `Slave::publishResource()`. + * [MESOS-8257] Unified Containerizer "leaks" a target container mount path to the host FS when the target resolves to an absolute path + * [MESOS-8256] Libprocess can silently deadlock due to worker thread exhaustion. + * [MESOS-8096] Enqueueing events in MockHTTPScheduler can lead to segfaults. + * [MESOS-8038] Launching GPU task sporadically fails. + * [MESOS-7971] PersistentVolumeEndpointsTest.EndpointCreateThenOfferRemove test is flaky + * [MESOS-7911] Non-checkpointing framework's tasks should not be marked LOST when agent disconnects. + * [MESOS-7748] Slow subscribers of streaming APIs can lead to Mesos OOMing. + * [MESOS-7721] Master's agent removal rate limit also applies to agent unreachability. + * [MESOS-7566] Master crash due to failed check in DRFSorter::remove + * [MESOS-7386] Executor not cleaning up existing running docker containers if external logrotate/logger processes die/killed + * [MESOS-5989] Libevent SSL Socket downgrade code accesses uninitialized memory / assumes single peek is sufficient. + * [MESOS-5754] CommandInfo.user not honored in docker containerizer + * [MESOS-2842] Master crashes when framework changes principal on re-registration + +All Resolved Issues: +** Bug + * [MESOS-5048] - MesosContainerizerSlaveRecoveryTest.ResourceStatistics is flaky + * [MESOS-5189] - SSLTest.ProtocolMismatch is slow + * [MESOS-6874] - Agent silently ignores FS isolation when protobuf is malformed + * [MESOS-6949] - SchedulerTest.MasterFailover is flaky + * [MESOS-6990] - PartitionTest.TaskCompletedOnPartitionedAgent is flaky. + *
[mesos] branch 1.8.x created (now 2c7cee0)
This is an automated email from the ASF dual-hosted git repository. bennoe pushed a change to branch 1.8.x in repository https://gitbox.apache.org/repos/asf/mesos.git. at 2c7cee0 Updated CHANGELOG for 1.8.0. This branch includes the following new commits: new 2c7cee0 Updated CHANGELOG for 1.8.0. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[mesos] branch master updated: Updated Mesos version to 1.9.0.
This is an automated email from the ASF dual-hosted git repository. bennoe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 3d1c435 Updated Mesos version to 1.9.0. 3d1c435 is described below commit 3d1c43599471a0717b7ee80bd2623f40bb526965 Author: Benno Evers AuthorDate: Fri Apr 5 23:18:59 2019 +0200 Updated Mesos version to 1.9.0. --- CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2885fa..65bfbac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ endif () project(Mesos) set(MESOS_MAJOR_VERSION 1) -set(MESOS_MINOR_VERSION 8) +set(MESOS_MINOR_VERSION 9) set(MESOS_PATCH_VERSION 0) set(PACKAGE_VERSION ${MESOS_MAJOR_VERSION}.${MESOS_MINOR_VERSION}.${MESOS_PATCH_VERSION}) diff --git a/configure.ac b/configure.ac index a367c28..459713a 100644 --- a/configure.ac +++ b/configure.ac @@ -18,7 +18,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([mesos], [1.8.0]) +AC_INIT([mesos], [1.9.0]) # Have autoconf setup some variables related to the system. AC_CANONICAL_HOST
[mesos] branch master updated (5ca5625 -> dd48ac6)
This is an automated email from the ASF dual-hosted git repository. bmahler pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 5ca5625 Clarified a comment for `SeccompInfo` in mesos.proto. new b9fc493 Disabled additional hierarchical quota tests. new 1156719 Added a NOTE/TODO to track reservations in the allocator's roles map. new 4c97c7f Fixed incorrect quota headroom and consumption calculations. new 31206cc Enhanced HierarchicalAllocatorTest.QuotaWithNestedRoleReservation. new 6abe32f Updated the sorters to track the root node allocation. new 0db8a97 Added an root node overload for Sorter::allocationScalarQuantities. new af3f189 Added tests for Sorter::allocationScalarQuantities. new dd48ac6 Avoid looping over role allocations during headroom computation. The 16916 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/master/allocator/mesos/hierarchical.cpp | 49 + src/master/allocator/mesos/hierarchical.hpp | 6 +++ src/master/allocator/sorter/drf/sorter.cpp| 43 -- src/master/allocator/sorter/drf/sorter.hpp| 3 ++ src/master/allocator/sorter/random/sorter.cpp | 40 - src/master/allocator/sorter/random/sorter.hpp | 2 + src/master/allocator/sorter/sorter.hpp| 3 +- src/tests/hierarchical_allocator_tests.cpp| 52 +++--- src/tests/sorter_tests.cpp| 63 +++ 9 files changed, 168 insertions(+), 93 deletions(-)
[mesos-site] branch asf-site updated: Updated the website built from mesos SHA: 5ca5625.
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/mesos-site.git The following commit(s) were added to refs/heads/asf-site by this push: new a332110 Updated the website built from mesos SHA: 5ca5625. a332110 is described below commit a3321100d3f1efaa4f27e88842f0f445ced27511 Author: jenkins AuthorDate: Fri Apr 5 14:59:51 2019 + Updated the website built from mesos SHA: 5ca5625. --- .../java/org/apache/mesos/Protos.LinuxInfo.Builder.html | 8 .../latest/java/org/apache/mesos/Protos.LinuxInfo.html | 3 +++ .../java/org/apache/mesos/Protos.LinuxInfoOrBuilder.html | 3 +++ .../org/apache/mesos/Protos.SeccompInfo.Builder.html | 16 .../latest/java/org/apache/mesos/Protos.SeccompInfo.html | 8 ++-- .../org/apache/mesos/Protos.SeccompInfoOrBuilder.html| 8 ++-- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.Builder.html b/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.Builder.html index 76437dd..15c8b5c 100644 --- a/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.Builder.html +++ b/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.Builder.html @@ -1161,6 +1161,7 @@ public Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1179,6 +1180,7 @@ publicProtos.SeccompInfogetSeccomp() Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1197,6 +1199,7 @@ publicProtos.LinuxInfo.BuildersetSeccomp(Protos.SeccompInfovalue) Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1211,6 +1214,7 @@ publicProtos.LinuxInfo.BuildersetSeccomp(Protos.SeccompInfo.BuilderbuilderForValue) Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1225,6 +1229,7 @@ publicProtos.LinuxInfo.BuildermergeSeccomp(Protos.SeccompInfovalue) Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1239,6 +1244,7 @@ publicProtos.LinuxInfo.BuilderclearSeccomp() Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1253,6 +1259,7 @@ publicProtos.SeccompInfo.BuildergetSeccompBuilder() Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -1267,6 +1274,7 @@ publicProtos.SeccompInfoOrBuildergetSeccompOrBuilder() Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; diff --git a/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.html b/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.html index 9f05c67..5cea75d 100644 --- a/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.html +++ b/content/api/latest/java/org/apache/mesos/Protos.LinuxInfo.html @@ -790,6 +790,7 @@ public Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -808,6 +809,7 @@ publicProtos.SeccompInfogetSeccomp() Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; @@ -826,6 +828,7 @@ publicProtos.SeccompInfoOrBuildergetSeccompOrBuilder() Represents Seccomp configuration, which is used for syscall filtering. + This field is used to override the agent's default Seccomp configuration. optional .mesos.SeccompInfo seccomp = 5; diff --git a/content/api/latest/java/org/apache/mesos/Protos.LinuxInfoOrBuilder.html b/content/api/latest/java/org/apache/mesos/Protos.LinuxInfoOrBuilder.html index b933326..5fe2e0d 100644 --- a/content/api/latest/java/org/apache/mesos/Protos.LinuxInfoOrBuilder.html +++ b/content/api/latest/java/org/apache/mesos/Protos.LinuxInfoOrBuilder.html @@ -428,6 +428,7 @@ booleanhasCapabilityInfo()
[mesos] branch master updated: Clarified a comment for `SeccompInfo` in mesos.proto.
This is an automated email from the ASF dual-hosted git repository. abudnik pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 5ca5625 Clarified a comment for `SeccompInfo` in mesos.proto. 5ca5625 is described below commit 5ca56257e9377c36c595cc83ce2eeb98a166945e Author: Andrei Budnik AuthorDate: Fri Apr 5 16:14:04 2019 +0200 Clarified a comment for `SeccompInfo` in mesos.proto. Review: https://reviews.apache.org/r/70407 --- include/mesos/mesos.proto| 5 - include/mesos/v1/mesos.proto | 5 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto index 7aa7d5b..dc6a87f 100644 --- a/include/mesos/mesos.proto +++ b/include/mesos/mesos.proto @@ -3228,7 +3228,9 @@ message SeccompInfo { // If not set or set to `false`, the container is launched with // the profile specified in the `profile_name` field. // - // NOTE: `profile_name` should not be specified if `unconfined` set to `true`. + // NOTE: `profile_name` must not be specified if `unconfined` set to `true`. + // `profile_name` must be specified if `unconfined` is not set or + // is set to `false`. optional bool unconfined = 2; } @@ -3258,6 +3260,7 @@ message LinuxInfo { optional bool share_pid_namespace = 4; // Represents Seccomp configuration, which is used for syscall filtering. + // This field is used to override the agent's default Seccomp configuration. optional SeccompInfo seccomp = 5; } diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto index d66f29c..3507909 100644 --- a/include/mesos/v1/mesos.proto +++ b/include/mesos/v1/mesos.proto @@ -3221,7 +3221,9 @@ message SeccompInfo { // If not set or set to `false`, the container is launched with // the profile specified in the `profile_name` field. // - // NOTE: `profile_name` should not be specified if `unconfined` set to `true`. + // NOTE: `profile_name` must not be specified if `unconfined` set to `true`. + // `profile_name` must be specified if `unconfined` is not set or + // is set to `false`. optional bool unconfined = 2; } @@ -3251,6 +3253,7 @@ message LinuxInfo { optional bool share_pid_namespace = 4; // Represents Seccomp configuration, which is used for syscall filtering. + // This field is used to override the agent's default Seccomp configuration. optional SeccompInfo seccomp = 5; }
[mesos] branch master updated: Fixed use-after-free bug in Docker provisioner store.
This is an automated email from the ASF dual-hosted git repository. abudnik pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new db917f6 Fixed use-after-free bug in Docker provisioner store. db917f6 is described below commit db917f639e2d05fcf493e87649f42ddd2abfeae0 Author: Andrei Budnik AuthorDate: Fri Apr 5 13:06:49 2019 +0200 Fixed use-after-free bug in Docker provisioner store. Deferred lambda callback of the `moveLayers()` to the `StoreProcess` to prevent use-after-free of the process object since the callback refers to the `StoreProcess` class variable `flags`. Review: https://reviews.apache.org/r/70405 --- src/slave/containerizer/mesos/provisioner/docker/store.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp index 909364d..36b2c7d 100644 --- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp +++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp @@ -446,7 +446,7 @@ Future StoreProcess::moveLayers( } return collect(futures) -.then([=]() -> Future { +.then(defer(self(), [=]() -> Future { if (image.has_config_digest()) { const string configSource = path::join(staging, image.config_digest()); const string configTarget = paths::getImageLayerPath( @@ -464,7 +464,7 @@ Future StoreProcess::moveLayers( } return image; -}); +})); }
[mesos] branch master updated: Quoted chmod arguments.
This is an automated email from the ASF dual-hosted git repository. bennoe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new f5c6d96 Quoted chmod arguments. f5c6d96 is described below commit f5c6d96c2789da688e16a88b68b156452c24dd8b Author: Benno Evers AuthorDate: Fri Apr 5 12:55:53 2019 +0200 Quoted chmod arguments. Quoted arguments passed to the `chmod` command in `mesos-build.sh` script. Review: https://reviews.apache.org/r/70386 --- support/mesos-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/mesos-build.sh b/support/mesos-build.sh index c2d121d..ee8ef8b 100755 --- a/support/mesos-build.sh +++ b/support/mesos-build.sh @@ -36,7 +36,7 @@ fi # NOTE: We chmod the directory here so that the docker containter can # copy out the test report xml files from the container file system. -chmod 777 ${MESOS_DIR} +chmod 777 "${MESOS_DIR}" docker run \ --rm \
[mesos] branch master updated (e6a3f0d -> 6e73de1)
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from e6a3f0d Added a unit test for Mesos containerizer image force pulling. new 3da5496 Bundled CSI spec 1.1.0. new 6ef64a3 Added spec inclusion header and type helpers for CSI v1. new 2b1336d Added the `mesos::csi::v1::Client` wrapper. new 6e73de1 Extended `CSIClientTest` to test the CSI v1 client wrapper. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: 3rdparty/CMakeLists.txt| 24 ++- 3rdparty/Makefile.am | 19 +- 3rdparty/cmake/Versions.cmake | 2 + 3rdparty/csi-1.1.0.tar.gz | Bin 0 -> 173338 bytes 3rdparty/versions.am | 1 + configure.ac | 8 +- include/mesos/csi/{v0.hpp => v1.hpp} | 24 +-- src/CMakeLists.txt | 3 + src/Makefile.am| 29 ++- src/cmake/MesosProtobuf.cmake | 6 + src/csi/{v0.cpp => v1.cpp} | 6 +- src/csi/{v0_client.cpp => v1_client.cpp} | 78 +++- src/csi/{v0_client.hpp => v1_client.hpp} | 34 +++- src/tests/csi_client_tests.cpp | 31 +++ src/tests/mock_csi_plugin.cpp | 208 ++--- src/tests/mock_csi_plugin.hpp | 137 +- .../storage_local_resource_provider_tests.cpp | 8 +- 17 files changed, 539 insertions(+), 79 deletions(-) create mode 100644 3rdparty/csi-1.1.0.tar.gz copy include/mesos/csi/{v0.hpp => v1.hpp} (89%) copy src/csi/{v0.cpp => v1.cpp} (94%) copy src/csi/{v0_client.cpp => v1_client.cpp} (75%) copy src/csi/{v0_client.hpp => v1_client.hpp} (77%)
[mesos] 02/04: Added spec inclusion header and type helpers for CSI v1.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 6ef64a3a6ff34975d58abbb0b78e2b402d39873c Author: Chun-Hung Hsiao AuthorDate: Thu Mar 28 22:14:32 2019 -0700 Added spec inclusion header and type helpers for CSI v1. Review: https://reviews.apache.org/r/70361 --- include/mesos/csi/v1.hpp | 103 +++ src/CMakeLists.txt | 1 + src/Makefile.am | 4 +- src/csi/v1.cpp | 32 +++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/include/mesos/csi/v1.hpp b/include/mesos/csi/v1.hpp new file mode 100644 index 000..d4ebe42 --- /dev/null +++ b/include/mesos/csi/v1.hpp @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __MESOS_CSI_V1_HPP__ +#define __MESOS_CSI_V1_HPP__ + +#include +#include + +// ONLY USEFUL AFTER RUNNING PROTOC. +#include + +// ONLY USEFUL AFTER RUNNING PROTOC WITH GRPC CPP PLUGIN. +#include + +#include + +#include +#include + +#include + +namespace mesos { +namespace csi { +namespace v1 { + +using namespace ::csi::v1; + +} // namespace v1 { +} // namespace csi { +} // namespace mesos { + + +namespace csi { +namespace v1 { + +// Default implementation for comparing protobuf messages in namespace +// `csi::v1`. Note that any non-template overloading of the equality operator +// would take precedence over this function template. +template < +typename Message, +typename std::enable_if::value, int>::type = 0> +bool operator==(const Message& left, const Message& right) +{ + // NOTE: `MessageDifferencer::Equivalent` would ignore unknown fields and load + // default values for unset fields (which are indistinguishable in proto3). + return google::protobuf::util::MessageDifferencer::Equivalent(left, right); +} + + +template < +typename Message, +typename std::enable_if::value, int>::type = 0> +bool operator!=(const Message& left, const Message& right) +{ + return !(left == right); +} + + +std::ostream& operator<<( +std::ostream& stream, +const ControllerServiceCapability::RPC::Type& type); + + +// Default implementation for output protobuf messages in namespace `csi::v1`. +// Note that any non-template overloading of the output operator would take +// precedence over this function template. +template < +typename Message, +typename std::enable_if::value, int>::type = 0> +std::ostream& operator<<(std::ostream& stream, const Message& message) +{ + // NOTE: We use Google's JSON utility functions for proto3. + std::string output; + google::protobuf::util::Status status = +google::protobuf::util::MessageToJsonString(message, ); + + CHECK(status.ok()) +<< "Could not convert messages to string: " << status.error_message(); + + return stream << output; +} + +} // namespace v1 { +} // namespace csi { + +#endif // __MESOS_CSI_V1_HPP__ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ffeccd..0512044 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -241,6 +241,7 @@ set(CSI_SRC csi/v0_client.cpp csi/v0_utils.cpp csi/v0_volume_manager.cpp + csi/v1.cpp csi/volume_manager.cpp) set(DOCKER_SRC diff --git a/src/Makefile.am b/src/Makefile.am index 6782e4b..2f25a84 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -710,7 +710,8 @@ csidir = $(pkgincludedir)/csi csi_HEADERS = \ $(top_srcdir)/include/mesos/csi/types.hpp\ $(top_srcdir)/include/mesos/csi/types.proto \ - $(top_srcdir)/include/mesos/csi/v0.hpp + $(top_srcdir)/include/mesos/csi/v0.hpp \ + $(top_srcdir)/include/mesos/csi/v1.hpp nodist_csi_HEADERS = \ ../include/mesos/csi/types.pb.h @@ -1594,6 +1595,7 @@ libcsi_la_SOURCES = \ csi/v0_volume_manager.cpp\ csi/v0_volume_manager.hpp\
[mesos] 03/04: Added the `mesos::csi::v1::Client` wrapper.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 2b1336d307380768176db072c1dcb70bcbb2f891 Author: Chun-Hung Hsiao AuthorDate: Fri Mar 29 16:33:18 2019 -0700 Added the `mesos::csi::v1::Client` wrapper. The differences between CSI v0 and v1 client wrappers are summarized as follows: * Added snapshot related functions: `createSnapshot`, `deleteSnapshot` and `listSnapshots`. * Added volume expansion related functions: `controllerExpandVolume` and `nodeExpandVolume`. * Added volume statistics related functions: `nodeGetVolumeStats`. * Replaced `nodeGetId` with `nodeGetInfo`. Review: https://reviews.apache.org/r/70362 --- src/CMakeLists.txt| 1 + src/Makefile.am | 2 + src/csi/v1_client.cpp | 282 ++ src/csi/v1_client.hpp | 119 + 4 files changed, 404 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0512044..51c7a04 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -242,6 +242,7 @@ set(CSI_SRC csi/v0_utils.cpp csi/v0_volume_manager.cpp csi/v1.cpp + csi/v1_client.cpp csi/volume_manager.cpp) set(DOCKER_SRC diff --git a/src/Makefile.am b/src/Makefile.am index 2f25a84..9c0180f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1596,6 +1596,8 @@ libcsi_la_SOURCES = \ csi/v0_volume_manager.hpp\ csi/v0_volume_manager_process.hpp\ csi/v1.cpp \ + csi/v1_client.cpp\ + csi/v1_client.hpp\ csi/volume_manager.cpp \ csi/volume_manager.hpp diff --git a/src/csi/v1_client.cpp b/src/csi/v1_client.cpp new file mode 100644 index 000..a19eb55 --- /dev/null +++ b/src/csi/v1_client.cpp @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "csi/v1_client.hpp" + +#include + +using process::Future; + +using process::grpc::client::CallOptions; + +namespace mesos { +namespace csi { +namespace v1 { + +Future> +Client::getPluginInfo(GetPluginInfoRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Identity, GetPluginInfo), + std::move(request), + CallOptions()); +} + + +Future> +Client::getPluginCapabilities(GetPluginCapabilitiesRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Identity, GetPluginCapabilities), + std::move(request), + CallOptions()); +} + + +Future> Client::probe(ProbeRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Identity, Probe), + std::move(request), + CallOptions()); +} + + +Future> +Client::createVolume(CreateVolumeRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Controller, CreateVolume), + std::move(request), + CallOptions()); +} + + +Future> +Client::deleteVolume(DeleteVolumeRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Controller, DeleteVolume), + std::move(request), + CallOptions()); +} + + +Future> +Client::controllerPublishVolume(ControllerPublishVolumeRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Controller, ControllerPublishVolume), + std::move(request), + CallOptions()); +} + + +Future> +Client::controllerUnpublishVolume(ControllerUnpublishVolumeRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Controller, ControllerUnpublishVolume), + std::move(request), + CallOptions()); +} + + +Future> +Client::validateVolumeCapabilities(ValidateVolumeCapabilitiesRequest request) +{ + return runtime.call( + connection, + GRPC_CLIENT_METHOD(Controller, ValidateVolumeCapabilities), +
[mesos] 01/04: Bundled CSI spec 1.1.0.
This is an automated email from the ASF dual-hosted git repository. chhsiao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 3da54965d02a6bf0e4806bf2d4acebb3310d60f7 Author: Chun-Hung Hsiao AuthorDate: Thu Mar 28 21:26:04 2019 -0700 Bundled CSI spec 1.1.0. Since the CSI v1 spec proto file depends on certain proto files in the Protobuf library, we have to ensure the Protobuf library's include path is in the proto paths of the `protoc` command when compiling the CSI spec proto file. Specifically in Autotools, this path is passed through the `PROTOBUF_PROTOCFLAGS` variable when building with an unbundled protobuf library. Review: https://reviews.apache.org/r/70360 --- 3rdparty/CMakeLists.txt | 24 ++-- 3rdparty/Makefile.am | 19 --- 3rdparty/cmake/Versions.cmake | 2 ++ 3rdparty/csi-1.1.0.tar.gz | Bin 0 -> 173338 bytes 3rdparty/versions.am | 1 + configure.ac | 8 +++- src/CMakeLists.txt| 1 + src/Makefile.am | 23 --- src/cmake/MesosProtobuf.cmake | 6 ++ 9 files changed, 75 insertions(+), 9 deletions(-) diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index 7f70407..53396fb 100644 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -27,6 +27,7 @@ set(BOOST_URL ${FETCH_URL}/boost-${BOOST_VERSION}.tar.gz) set(BZIP2_URL ${FETCH_URL}/bzip2-${BZIP2_VERSION}.tar.gz) set(CONCURRENTQUEUE_URL ${FETCH_URL}/concurrentqueue-${CONCURRENTQUEUE_VERSION}.tar.gz) set(CSI_V0_URL ${FETCH_URL}/csi-${CSI_V0_VERSION}.tar.gz) +set(CSI_V1_URL ${FETCH_URL}/csi-${CSI_V1_VERSION}.tar.gz) set(ELFIO_URL ${FETCH_URL}/elfio-${ELFIO_VERSION}.tar.gz) set(GLOG_URL${FETCH_URL}/glog-${GLOG_VERSION}.tar.gz) set(GOOGLETEST_URL ${FETCH_URL}/googletest-release-${GOOGLETEST_VERSION}.tar.gz) @@ -299,12 +300,21 @@ add_library(csi_v0 INTERFACE) add_dependencies(csi_v0 ${CSI_V0_TARGET}) target_include_directories(csi_v0 INTERFACE ${CSI_V0_ROOT}) -# NOTE: To support multiple CSI versions, we move the CSI proto file to a -# version-qualified path so `protoc` can find it. +EXTERNAL(csi_v1 ${CSI_V1_VERSION} ${CMAKE_CURRENT_BINARY_DIR}) +add_library(csi_v1 INTERFACE) +add_dependencies(csi_v1 ${CSI_V1_TARGET}) +target_include_directories(csi_v1 INTERFACE ${CSI_V1_ROOT}) + +# NOTE: To support multiple CSI versions, we move the CSI proto files to +# version-qualified paths so `protoc` can find them. set(CSI_V0_CONFIG_CMD ${CMAKE_COMMAND} -E make_directory ${CSI_V0_ROOT}/csi/v0 && ${CMAKE_COMMAND} -E rename ${CSI_V0_ROOT}/csi.proto ${CSI_V0_ROOT}/csi/v0/csi.proto) +set(CSI_V1_CONFIG_CMD + ${CMAKE_COMMAND} -E make_directory ${CSI_V1_ROOT}/csi/v1 && + ${CMAKE_COMMAND} -E rename ${CSI_V1_ROOT}/csi.proto ${CSI_V1_ROOT}/csi/v1/csi.proto) + ExternalProject_Add( ${CSI_V0_TARGET} PREFIX${CSI_V0_CMAKE_ROOT} @@ -315,6 +325,16 @@ ExternalProject_Add( URL ${CSI_V0_URL} URL_HASH ${CSI_V0_HASH}) +ExternalProject_Add( + ${CSI_V1_TARGET} + PREFIX${CSI_V1_CMAKE_ROOT} + BUILD_BYPRODUCTS ${CSI_V1_ROOT}/csi/v1/csi.proto + CONFIGURE_COMMAND ${CSI_V1_CONFIG_CMD} + BUILD_COMMAND ${CMAKE_NOOP} + INSTALL_COMMAND ${CMAKE_NOOP} + URL ${CSI_V1_URL} + URL_HASH ${CSI_V1_HASH}) + # ELFIO: library for reading and generating ELF files. # http://elfio.sourceforge.net diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am index adbbaf5..99815da 100644 --- a/3rdparty/Makefile.am +++ b/3rdparty/Makefile.am @@ -51,6 +51,7 @@ include versions.am BOOST = boost-$(BOOST_VERSION) CONCURRENTQUEUE = concurrentqueue-$(CONCURRENTQUEUE_VERSION) CSI_V0 = csi-$(CSI_V0_VERSION) +CSI_V1 = csi-$(CSI_V1_VERSION) ELFIO = elfio-$(ELFIO_VERSION) GLOG = glog-$(GLOG_VERSION) GMOCK = $(GOOGLETEST)/googlemock @@ -78,6 +79,7 @@ EXTRA_DIST = \ $(BOOST).tar.gz \ $(CONCURRENTQUEUE).tar.gz\ $(CSI_V0).tar.gz \ + $(CSI_V1).tar.gz \ $(ELFIO).tar.gz \ $(GLOG).tar.gz \ $(GOOGLETEST).tar.gz \ @@ -140,6 +142,7 @@ CLEAN_EXTRACTED = \ $(BOOST) \ $(CONCURRENTQUEUE) \ $(CSI_V0)\ + $(CSI_V1)\ $(ELFIO) \ $(GLOG) \ $(GOOGLETEST)\ @@ -175,7 +178,7 @@ CLEAN_EXTRACTED = \ # # NOTE: Since GNU make 3.81 does not support shortest-stem pattern matching, we # explicitly specify a static pattern rule here. -CSI_STAMPS = $(CSI_V0)-stamp +CSI_STAMPS = $(CSI_V0)-stamp $(CSI_V1)-stamp $(CSI_STAMPS): csi-%-stamp: csi-%.tar.gz $(MKDIR_P) csi-$* gzip -d -c $^ | (cd csi-$* && tar xf -)