[mesos] branch master updated: Use override on overriding methods - found with clang-tidy.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 44b172f Use override on overriding methods - found with clang-tidy.
44b172f is described below
commit 44b172f33de2f2dd14e575f54758e66b56a851d0
Author: Charles-Francois Natali
AuthorDate: Sat Jul 24 14:29:35 2021 +0100
Use override on overriding methods - found with clang-tidy.
---
src/tests/containerizer/routing_tests.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/tests/containerizer/routing_tests.cpp
b/src/tests/containerizer/routing_tests.cpp
index f2e4622..a030668 100644
--- a/src/tests/containerizer/routing_tests.cpp
+++ b/src/tests/containerizer/routing_tests.cpp
@@ -239,7 +239,7 @@ constexpr char TEST_PEER_LINK[] = "veth-peer";
class RoutingVethTest : public RoutingAdvancedTest
{
protected:
- virtual void SetUp()
+ void SetUp() override
{
RoutingAdvancedTest::SetUp();
@@ -251,7 +251,7 @@ protected:
ASSERT_SOME_FALSE(link::exists(TEST_PEER_LINK));
}
- virtual void TearDown()
+ void TearDown() override
{
link::remove(TEST_VETH_LINK);
}
[mesos] branch master updated: Fixed a LevelDBStorage bug where positions would overflow.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 50b7299 Fixed a LevelDBStorage bug where positions would overflow.
50b7299 is described below
commit 50b729904e4b4ed7964cd5f2ac8177bca5a5a72c
Author: Charles-Francois Natali
AuthorDate: Sun May 16 00:07:09 2021 +0100
Fixed a LevelDBStorage bug where positions would overflow.
Positions were encoded with a width of 10, which limited the
maximum position to 9'999'999'999. And actually the code suffered
from overflow above INT32_MAX.
In order to be backward compatible, we still encode positions up to
9'999'999'999 using a width of 10, however for positions above
that, we switch to a width of 20 with the twist that we prepend 'A'
in order to preserve lexicographic ordering. Here's what it looks like:
00
...
98
99
A0100
A0101
The reason this works is because the only property which is required
by the encoding function is that it is strictly monotonically
increasing, i.e. if i < j, then encode(i) < encode(j) in lexicographic
order.
Closes #10216.
---
src/log/leveldb.cpp | 39 -
src/tests/log_tests.cpp | 148
2 files changed, 185 insertions(+), 2 deletions(-)
diff --git a/src/log/leveldb.cpp b/src/log/leveldb.cpp
index 72eeffb..7cb84bc 100644
--- a/src/log/leveldb.cpp
+++ b/src/log/leveldb.cpp
@@ -88,7 +88,10 @@ public:
static string encode(uint64_t position, bool adjust = true)
{
// Adjusted stringified represenation is plus 1 of actual position.
- position = adjust ? position + 1 : position;
+ if (adjust) {
+CHECK_LT(position, UINT64_MAX);
+position += 1;
+ }
// TODO(benh): Use varint encoding for VarInt64Comparator!
// string s;
@@ -98,7 +101,31 @@ static string encode(uint64_t position, bool adjust = true)
// stream.WriteVarint64(position);
// return s;
- Try s = strings::format("%.*d", 10, position);
+ // Historically, positions were encoded with a width of 10, which limited the
+ // maximum position to 9'999'999'999. And actually the code suffered from
+ // overflow above INT32_MAX, see MESOS-10186.
+ // In order to be backward compatible, we still encode positions up to
+ // 9'999'999'999 using a width of 10, however for positions above that, we
+ // switch to a width of 20 with the twist that we prepend 'A' in order to
+ // preserve lexicographic ordering. Here's what it looks like:
+ //
+ // 00
+ // ...
+ // 98
+ // 99
+ // A0100
+ // A0101
+ //
+ // The reason this works is because the only property which is required
+ // by the encoding function is that it is strictly monotonically increasing,
+ // i.e. if i < j, then encode(i) < encode(j) in lexicographic order.
+ Try s = "";
+ if (position <= 99ull) {
+s = strings::format("%.*llu", 10, position);
+ } else {
+s = strings::format("A%.*llu", 20, position);
+ }
+
CHECK_SOME(s);
return s.get();
}
@@ -160,6 +187,14 @@ Try LevelDBStorage::restore(const string&
path)
CHECK(leveldb::BytewiseComparator()->Compare(one, ten) < 0);
CHECK(leveldb::BytewiseComparator()->Compare(ten, two) > 0);
CHECK(leveldb::BytewiseComparator()->Compare(ten, ten) == 0);
+ CHECK(leveldb::BytewiseComparator()->Compare(
+ encode(98), encode(99)) < 0);
+ CHECK(leveldb::BytewiseComparator()->Compare(
+ encode(99), encode(100)) < 0);
+ CHECK(leveldb::BytewiseComparator()->Compare(
+ encode(100), encode(UINT64_MAX - 2)) < 0);
+ CHECK(leveldb::BytewiseComparator()->Compare(
+ encode(UINT64_MAX - 2), encode(UINT64_MAX - 1)) < 0);
Stopwatch stopwatch;
stopwatch.start();
diff --git a/src/tests/log_tests.cpp b/src/tests/log_tests.cpp
index a8980e3..94fca13 100644
--- a/src/tests/log_tests.cpp
+++ b/src/tests/log_tests.cpp
@@ -321,6 +321,154 @@ TYPED_TEST(LogStorageTest, TruncateWithManyHoles)
}
+TYPED_TEST(LogStorageTest, Overflow)
+{
+ // This test checks that the log storage correctly handles positions
+ // up to UINT64_MAX, see MESOS-10216.
+ // Specifically we check some values which would overflow when `encode`
+ // formatted them as signed integer, and collide with each other.
+ TypeParam storage;
+
+ Try state = storage.restore(os::getcwd() + "/.log");
+ ASSERT_SOME(state);
+
+ EXPECT_EQ(Metadata::EMPTY, state->metadata.status());
+ EXPECT_EQ(0u, state->metadata.promised());
+ EXPECT_EQ(0u, state->begin);
+ EXPECT_EQ(0u, state->end);
+
+ // Note that positi
[mesos] branch master updated: Fixed compilation against Linux 5.9+ capability.h.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 85981d7 Fixed compilation against Linux 5.9+ capability.h.
85981d7 is described below
commit 85981d7c728798783b2e1e7cbed4f27a1497ccb2
Author: Charles-Francois Natali
AuthorDate: Sun May 16 20:07:03 2021 +0100
Fixed compilation against Linux 5.9+ capability.h.
They should be defined without the "CAP_" prefix to avoid clashing with
the corresponding definitions in `linux/capability.h` - and be
consistent with other capabilities.
---
include/mesos/mesos.proto| 6 +++---
include/mesos/v1/mesos.proto | 6 +++---
src/linux/capabilities.cpp | 6 +++---
src/linux/capabilities.hpp | 6 +++---
4 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 3cbdc5c..f153de4 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -3378,9 +3378,9 @@ message CapabilityInfo {
WAKE_ALARM = 1035;
BLOCK_SUSPEND = 1036;
AUDIT_READ = 1037;
-CAP_PERFMON = 1038;
-CAP_BPF = 1039;
-CAP_CHECKPOINT_RESTORE = 1040;
+PERFMON = 1038;
+BPF = 1039;
+CHECKPOINT_RESTORE = 1040;
}
repeated Capability capabilities = 1;
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 0a3f8c0..f99cff3 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -3367,9 +3367,9 @@ message CapabilityInfo {
WAKE_ALARM = 1035;
BLOCK_SUSPEND = 1036;
AUDIT_READ = 1037;
-CAP_PERFMON = 1038;
-CAP_BPF = 1039;
-CAP_CHECKPOINT_RESTORE = 1040;
+PERFMON = 1038;
+BPF = 1039;
+CHECKPOINT_RESTORE = 1040;
}
repeated Capability capabilities = 1;
diff --git a/src/linux/capabilities.cpp b/src/linux/capabilities.cpp
index b161e64..41bf811 100644
--- a/src/linux/capabilities.cpp
+++ b/src/linux/capabilities.cpp
@@ -494,9 +494,9 @@ ostream& operator<<(ostream& stream, const Capability&
capability)
case WAKE_ALARM: return stream << "WAKE_ALARM";
case BLOCK_SUSPEND: return stream << "BLOCK_SUSPEND";
case AUDIT_READ: return stream << "AUDIT_READ";
-case CAP_PERFMON: return stream << "CAP_PERFMON";
-case CAP_BPF: return stream << "CAP_BPF";
-case CAP_CHECKPOINT_RESTORE: return stream << "CAP_CHECKPOINT_RESTORE";
+case PERFMON: return stream << "PERFMON";
+case BPF: return stream << "BPF";
+case CHECKPOINT_RESTORE: return stream << "CHECKPOINT_RESTORE";
case MAX_CAPABILITY: UNREACHABLE();
}
diff --git a/src/linux/capabilities.hpp b/src/linux/capabilities.hpp
index 4f41f49..6e11e4d 100644
--- a/src/linux/capabilities.hpp
+++ b/src/linux/capabilities.hpp
@@ -72,9 +72,9 @@ enum Capability : int
WAKE_ALARM = 35,
BLOCK_SUSPEND = 36,
AUDIT_READ = 37,
- CAP_PERFMON= 38,
- CAP_BPF= 39,
- CAP_CHECKPOINT_RESTORE = 40,
+ PERFMON= 38,
+ BPF= 39,
+ CHECKPOINT_RESTORE = 40,
MAX_CAPABILITY = 41,
};
[mesos] branch master updated: Revert "Fixed read of uninitialized variables detected with UBSAN."
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 62d44d1 Revert "Fixed read of uninitialized variables detected with
UBSAN."
62d44d1 is described below
commit 62d44d167220caff5fc0c42270671fb99a6a9fc8
Author: Andrei Sekretenko
AuthorDate: Sun May 16 09:53:25 2021 +0200
Revert "Fixed read of uninitialized variables detected with UBSAN."
This reverts commit add9f1de771693884548095703ed2bd4bc6cfc16.
As it turns out, fixing this UB this way exposes some other issue
in containerizer tests - for example,
`MesosContainerizerTest.StatusWithContainerID` - which results
in failures to wait for cgroup destruction on some platforms
in the affected tests. Temporarily reverting this pending further
investigation. See https://github.com/apache/mesos/pull/385
---
src/tests/cluster.hpp | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/tests/cluster.hpp b/src/tests/cluster.hpp
index eae6bcf..b36b94f 100644
--- a/src/tests/cluster.hpp
+++ b/src/tests/cluster.hpp
@@ -106,8 +106,7 @@ public:
void setAuthorizationCallbacks(Authorizer* authorizer);
private:
- Master() : files(master::READONLY_HTTP_AUTHENTICATION_REALM),
- authorizationCallbacksSet(false) {}
+ Master() : files(master::READONLY_HTTP_AUTHENTICATION_REALM) {};
// Not copyable, not assignable.
Master(const Master&) = delete;
@@ -204,8 +203,7 @@ public:
void setAuthorizationCallbacks(Authorizer* authorizer);
private:
- Slave() : files(slave::READONLY_HTTP_AUTHENTICATION_REALM),
-authorizationCallbacksSet(false) {}
+ Slave() : files(slave::READONLY_HTTP_AUTHENTICATION_REALM) {};
// Not copyable, not assignable.
Slave(const Slave&) = delete;
[mesos] branch master updated: Fixed read of uninitialized variables detected with UBSAN.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new add9f1d Fixed read of uninitialized variables detected with UBSAN.
add9f1d is described below
commit add9f1de771693884548095703ed2bd4bc6cfc16
Author: Charles-Francois Natali
AuthorDate: Sat May 15 12:39:18 2021 +0100
Fixed read of uninitialized variables detected with UBSAN.
---
src/tests/cluster.hpp | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/tests/cluster.hpp b/src/tests/cluster.hpp
index b36b94f..eae6bcf 100644
--- a/src/tests/cluster.hpp
+++ b/src/tests/cluster.hpp
@@ -106,7 +106,8 @@ public:
void setAuthorizationCallbacks(Authorizer* authorizer);
private:
- Master() : files(master::READONLY_HTTP_AUTHENTICATION_REALM) {};
+ Master() : files(master::READONLY_HTTP_AUTHENTICATION_REALM),
+ authorizationCallbacksSet(false) {}
// Not copyable, not assignable.
Master(const Master&) = delete;
@@ -203,7 +204,8 @@ public:
void setAuthorizationCallbacks(Authorizer* authorizer);
private:
- Slave() : files(slave::READONLY_HTTP_AUTHENTICATION_REALM) {};
+ Slave() : files(slave::READONLY_HTTP_AUTHENTICATION_REALM),
+authorizationCallbacksSet(false) {}
// Not copyable, not assignable.
Slave(const Slave&) = delete;
[mesos] branch master updated: Fixed a bug preventing agent recovery when executor GC is interrupted.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new f30f10d Fixed a bug preventing agent recovery when executor GC is
interrupted.
f30f10d is described below
commit f30f10d03e7291b60beedc9742f163d393f94d88
Author: Charles-Francois Natali
AuthorDate: Sat Jan 30 18:41:37 2021 +
Fixed a bug preventing agent recovery when executor GC is interrupted.
If the agent is interrupted after garbage collecting the executor's
latest run meta directory but before garbage collecting the top-level
executor meta directory, the "latest" symlink will dangle, which would
cause the agent executor recovery to fail.
Instead, we can simply ignore if the "latest" symlink dangles, since
it's always created after the latest run directory it points to, and
never deleted until the top-level executor meta directory is garbage
collected.
---
src/slave/state.cpp| 14 +++--
src/tests/slave_recovery_tests.cpp | 124 +
2 files changed, 133 insertions(+), 5 deletions(-)
diff --git a/src/slave/state.cpp b/src/slave/state.cpp
index 96f22b3..6d065d0 100644
--- a/src/slave/state.cpp
+++ b/src/slave/state.cpp
@@ -417,13 +417,17 @@ Try ExecutorState::recover(
foreach (const string& path, runs.get()) {
if (Path(path).basename() == paths::LATEST_SYMLINK) {
const Result latest = os::realpath(path);
- if (!latest.isSome()) {
+ if (latest.isNone()) {
+// This can happen if the slave died between garbage collecting the
+// executor latest run and garbage collecting the top level executor
+// meta directory, containing the "latest" symlink.
+LOG(WARNING) << "Dangling 'latest' run symlink of executor '"
+ << executorId << "'";
+continue;
+ } else if (latest.isError()) {
return Error(
"Failed to find latest run of executor '" +
-executorId.value() + "': " +
-(latest.isError()
- ? latest.error()
- : "No such file or directory"));
+executorId.value() + "': " + latest.error());
}
// Store the ContainerID of the latest executor run.
diff --git a/src/tests/slave_recovery_tests.cpp
b/src/tests/slave_recovery_tests.cpp
index 1c5177d..3980114 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -3301,6 +3301,130 @@ TYPED_TEST(SlaveRecoveryTest, GCExecutor)
}
+// When the slave is down we remove the latest run directory
+// but not the "latest" symlink, to simulate a situation where the
+// slave died in the middle of gc'ing the run meta directory.
+TYPED_TEST(SlaveRecoveryTest, ExecutorDanglingLatestSymlink)
+{
+ Try> master = this->StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = this->CreateSlaveFlags();
+ flags.strict = true;
+
+ Fetcher fetcher(flags);
+
+ Try _containerizer = TypeParam::create(flags, true, );
+ ASSERT_SOME(_containerizer);
+ Owned containerizer(_containerizer.get());
+
+ Owned detector = master.get()->createDetector();
+
+ Try> slave =
+this->StartSlave(detector.get(), containerizer.get(), flags);
+ ASSERT_SOME(slave);
+
+ // Enable checkpointing for the framework.
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.set_checkpoint(true);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ , frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(_, _, _));
+
+ Future> offers1;
+ EXPECT_CALL(sched, resourceOffers(_, _))
+.WillOnce(FutureArg<1>()).WillRepeatedly(Return());
+
+ driver.start();
+
+ AWAIT_READY(offers1);
+ ASSERT_FALSE(offers1->empty());
+
+ TaskInfo task = createTask(offers1.get()[0], SLEEP_COMMAND(1000));
+
+ // Capture the slave and framework ids.
+ SlaveID slaveId = offers1.get()[0].slave_id();
+ FrameworkID frameworkId = offers1.get()[0].framework_id();
+
+ Future registerExecutor =
+FUTURE_PROTOBUF(RegisterExecutorMessage(), _, _);
+
+ Future status;
+ EXPECT_CALL(sched, statusUpdate(_, _))
+.WillOnce(FutureSatisfy())
+.WillRepeatedly(Return()); // Ignore subsequent updates.
+
+ driver.launchTasks(offers1.get()[0].id(), {task});
+
+ // Capture the executor id.
+ AWAIT_READY(registerExecutor);
+ ExecutorID executorId = registerExecutor->executor_id();
+
+ // Wait for TASK_RUNNING update.
+ AWAIT_READY(status);
+
+ // Terminate the slave.
+ slave.get()->terminate();
+
+ // The "latest" symlink should exist.
+ const string latestPath = paths::getExecutorLatestRunPath(
+ paths::getMeta
[mesos] branch master updated: Fixed NNP isolator test on systems with POSIX-compliant /bin/sh.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 96339ef Fixed NNP isolator test on systems with POSIX-compliant
/bin/sh.
96339ef is described below
commit 96339efb53f7cdf1126ead7755d2b83b435e3263
Author: Charles-Francois Natali
AuthorDate: Sun Jan 31 10:08:36 2021 +
Fixed NNP isolator test on systems with POSIX-compliant /bin/sh.
The test used some non-POSIX features such as arrays when parsing
/proc/self/status, which breaks on systems where /bin/sh is
POSIX-compliant, e.g. on Debian which uses dash.
---
src/tests/containerizer/linux_nnp_isolator_tests.cpp | 20 ++--
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/src/tests/containerizer/linux_nnp_isolator_tests.cpp
b/src/tests/containerizer/linux_nnp_isolator_tests.cpp
index 4d8bce1..4661373 100644
--- a/src/tests/containerizer/linux_nnp_isolator_tests.cpp
+++ b/src/tests/containerizer/linux_nnp_isolator_tests.cpp
@@ -102,18 +102,26 @@ TEST_F(LinuxNNPIsolatorTest, ROOT_CheckNoNewPrivileges)
containerId.set_value(id::UUID::random().toString());
// Test that the child process inherits the PR_NO_NEW_PRIVS flag.
- // Using parameter expansion to parse the process status file
+ // Using convoluted way to parse the process status file
// due to minimal docker image. The child process should inherit
// the PR_NO_NEW_PRIVS flag. Parse the process status file and
// determine if "NoNewPrivs: 1" is found.
ExecutorInfo executor = createExecutorInfo(
"test_executor",
R"~(
- #!/bin/bash
- x=$(cat /proc/self/status);
- y=${x##*NoNewPrivs:};
- read -a a <<< $y;
- if [ ${a[0]} == "1" ]; then exit 0; else exit 1; fi
+ nnp_seen="false"
+ for word in $(cat /proc/self/status); do
+if [ "$word" = "NoNewPrivs:" ]; then
+ nnp_seen="true"
+elif [ "$nnp_seen" = "true" ]; then
+ if [ "$word" = "1" ]; then
+exit 0
+ else
+exit 1
+ fi
+fi
+ done
+ exit 1
)~");
executor.mutable_container()->CopyFrom(createContainerInfo("test_image"));
[mesos] 02/03: Made the `latest_stable` in the releases list point to 1.11.0.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit a05f3654938ae09a7c1d1fc6f37eeb8b36df4199 Author: Andrei Sekretenko AuthorDate: Tue Nov 24 16:36:16 2020 +0100 Made the `latest_stable` in the releases list point to 1.11.0. --- site/data/releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/data/releases.yml b/site/data/releases.yml index 8ca54ba..35705c3 100644 --- a/site/data/releases.yml +++ b/site/data/releases.yml @@ -3,8 +3,8 @@ # `versions` below. # * When publishing a new blog post, we need to append an item to `news` below. latest_stable: - version: 1.10.0 - jira_version: 12346271 + version: 1.11.0 + jira_version: 12348332 news: ## * If the news is used to announce a release version, the item structure is ## - title: (required)
[mesos] 01/03: Linked the 1.10.0 release blogpost to the release list.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit bf7250ead5feb9fa4de6a3e8f5f6d4d56baebe7a Author: Andrei Sekretenko AuthorDate: Tue Nov 24 16:30:51 2020 +0100 Linked the 1.10.0 release blogpost to the release list. --- site/data/releases.yml | 4 1 file changed, 4 insertions(+) diff --git a/site/data/releases.yml b/site/data/releases.yml index 49ebfe4..8ca54ba 100644 --- a/site/data/releases.yml +++ b/site/data/releases.yml @@ -15,6 +15,10 @@ news: ## - title: (required) ## date: (required) ## blog: (required) + - title: Mesos 1.10.0 is released! +date: October 20, 2019 +target_version: 1.10.0 +blog: mesos-1-10-0-released - title: Mesos 1.9.0 is released! date: September 5, 2019 target_version: 1.9.0
[mesos] 03/03: Updated `building.md` for the 1.11.0 release.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 206da612c0aada0b1d86beb63660d9083b774894 Author: Andrei Sekretenko AuthorDate: Tue Nov 24 16:40:17 2020 +0100 Updated `building.md` for the 1.11.0 release. --- docs/building.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/building.md b/docs/building.md index b805b38..6541182 100644 --- a/docs/building.md +++ b/docs/building.md @@ -11,8 +11,8 @@ There are different ways you can get Mesos: 1\. Download the latest stable release from [Apache](http://mesos.apache.org/downloads/) (***Recommended***) -$ wget http://www.apache.org/dist/mesos/1.10.0/mesos-1.10.0.tar.gz -$ tar -zxf mesos-1.10.0.tar.gz +$ wget https://downloads.apache.org/mesos/1.11.0/mesos-1.11.0.tar.gz +$ tar -zxf mesos-1.11.0.tar.gz 2\. Clone the Mesos git [repository](https://gitbox.apache.org/repos/asf/mesos.git) (***Advanced Users Only***)
[mesos] branch master updated (2fbdb30 -> 206da61)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 2fbdb30 Added the `--offer_constraints_re2_max*` flags to the documentation. new bf7250e Linked the 1.10.0 release blogpost to the release list. new a05f365 Made the `latest_stable` in the releases list point to 1.11.0. new 206da61 Updated `building.md` for the 1.11.0 release. The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/building.md | 4 ++-- site/data/releases.yml | 8 ++-- 2 files changed, 8 insertions(+), 4 deletions(-)
[mesos] branch 1.11.x updated: Updated Mesos version to 1.11.1.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.11.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/1.11.x by this push:
new b01e71d Updated Mesos version to 1.11.1.
b01e71d is described below
commit b01e71d4f2c78a6972769dd359063341c9d0b408
Author: Andrei Sekretenko
AuthorDate: Tue Nov 24 16:44:25 2020 +0100
Updated Mesos version to 1.11.1.
---
CMakeLists.txt | 2 +-
configure.ac | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4390aa8..dd3a873 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,7 +25,7 @@ endif ()
project(Mesos)
set(MESOS_MAJOR_VERSION 1)
set(MESOS_MINOR_VERSION 11)
-set(MESOS_PATCH_VERSION 0)
+set(MESOS_PATCH_VERSION 1)
set(PACKAGE_VERSION
${MESOS_MAJOR_VERSION}.${MESOS_MINOR_VERSION}.${MESOS_PATCH_VERSION})
diff --git a/configure.ac b/configure.ac
index b19440a..7d46d2f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -18,7 +18,7 @@
# Process this file with autoconf to produce a configure script.
AC_PREREQ([2.61])
-AC_INIT([mesos], [1.11.0])
+AC_INIT([mesos], [1.11.1])
# Have autoconf setup some variables related to the system.
AC_CANONICAL_HOST
[mesos] annotated tag 1.11.0 created (now c21c1c9)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to annotated tag 1.11.0 in repository https://gitbox.apache.org/repos/asf/mesos.git. at c21c1c9 (tag) tagging 213739cbb069b2e87aa6042b22db5cfdf78cd51e (tag) length 169 bytes by Andrei Sekretenko on Tue Nov 24 16:12:42 2020 +0100 - Log - Tagging Mesos 1.11.0 --- No new revisions were added by this update.
svn commit: r44666 - in /release/mesos/1.11.0: ./ mesos-1.11.0.tar.gz mesos-1.11.0.tar.gz.asc mesos-1.11.0.tar.gz.sha512
Author: asekretenko Date: Tue Nov 24 15:12:42 2020 New Revision: 44666 Log: Adding mesos-1.11.0. Added: release/mesos/1.11.0/ release/mesos/1.11.0/mesos-1.11.0.tar.gz (with props) release/mesos/1.11.0/mesos-1.11.0.tar.gz.asc release/mesos/1.11.0/mesos-1.11.0.tar.gz.sha512 Added: release/mesos/1.11.0/mesos-1.11.0.tar.gz == Binary file - no diff available. Propchange: release/mesos/1.11.0/mesos-1.11.0.tar.gz -- svn:mime-type = application/octet-stream Added: release/mesos/1.11.0/mesos-1.11.0.tar.gz.asc == --- release/mesos/1.11.0/mesos-1.11.0.tar.gz.asc (added) +++ release/mesos/1.11.0/mesos-1.11.0.tar.gz.asc Tue Nov 24 15:12:42 2020 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEQFDbyFC8813QvwPzxKfLkrITBaQFAl+z3WcACgkQxKfLkrIT +BaTjzQ/7B6c0H4HXjoK4PN+rJuO0cbIM4Adc665BVBr/+ksf6szDw+cTvABIYOch +8tMfYhU0zqRAE3dg+4ZAJg6RVvf/RutdPls0mgPjuvYxUmPGZI12OjqWll+obEtO +YI/UtaYwei9DUlmQWH5Z+TJowBbW/WEQBfaG6Kp9wJZGNh7SHrXyk4gG9vdIM9ec +Q1tJHRI+uPEmE6PKh//a8yuorA0zn01FhPvrRryxB57krknRN60O83CLFVRWMhlc ++BCeOOlJ23MV+zlVKN3Oh5hx3pci2xv2lCBrMj7N9g6QEk/hSzC066oyM9Eim4B1 +rZt+b3Z7uEqCwsrx/o9NCTGTmgzKEduHFOtEwvKFepK4Vilja8rbTFYPeU2YHLXZ +sllYW4Y200z5P8u+5h1yDm9YHH3L+OnU9mqNmY8IMbVHMfq60HOXV868+KZyShtS +CFt4Gp7qSBsjDMndN3ca01H6lqEwa5CLXAFdDA/+JCeiZJa6PTUFBB9uXbq0s6xZ +sxbpCU8PPNNrC8RSNz/wjETMungDo4QAvuLieX4l8Zpz7UkYF7pLDnYiwx6E5prB +xpoFso1SUkClhhyr34+V5xjsmekLdE9KDo/aknQSy9c7PzjfEHxCTc0McioG55lO +WtwjI1+7kFRCkF0AkPH0BlIJ/hav+RwNsg2STbqYYMkUqWcasVo= +=e0Y9 +-END PGP SIGNATURE- Added: release/mesos/1.11.0/mesos-1.11.0.tar.gz.sha512 == --- release/mesos/1.11.0/mesos-1.11.0.tar.gz.sha512 (added) +++ release/mesos/1.11.0/mesos-1.11.0.tar.gz.sha512 Tue Nov 24 15:12:42 2020 @@ -0,0 +1 @@ +48ecc9457f8f86fc3a7f769a43539cfbb1e2178069aaea19ccaa78917612a2ddaefa033a5ace40ccf6f483be4c49e154008c85bc9c715b07415448e2b93ebdba mesos-1.11.0.tar.gz
svn commit: r44543 - in /dev/mesos/1.11.0-rc1: ./ mesos-1.11.0.tar.gz mesos-1.11.0.tar.gz.asc mesos-1.11.0.tar.gz.sha512
Author: asekretenko Date: Tue Nov 17 14:27:17 2020 New Revision: 44543 Log: Adding mesos-1.11.0-rc1. Added: dev/mesos/1.11.0-rc1/ dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz (with props) dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.asc dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.sha512 Added: dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz == Binary file - no diff available. Propchange: dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz -- svn:mime-type = application/octet-stream Added: dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.asc == --- dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.asc (added) +++ dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.asc Tue Nov 17 14:27:17 2020 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEQFDbyFC8813QvwPzxKfLkrITBaQFAl+z3WcACgkQxKfLkrIT +BaTjzQ/7B6c0H4HXjoK4PN+rJuO0cbIM4Adc665BVBr/+ksf6szDw+cTvABIYOch +8tMfYhU0zqRAE3dg+4ZAJg6RVvf/RutdPls0mgPjuvYxUmPGZI12OjqWll+obEtO +YI/UtaYwei9DUlmQWH5Z+TJowBbW/WEQBfaG6Kp9wJZGNh7SHrXyk4gG9vdIM9ec +Q1tJHRI+uPEmE6PKh//a8yuorA0zn01FhPvrRryxB57krknRN60O83CLFVRWMhlc ++BCeOOlJ23MV+zlVKN3Oh5hx3pci2xv2lCBrMj7N9g6QEk/hSzC066oyM9Eim4B1 +rZt+b3Z7uEqCwsrx/o9NCTGTmgzKEduHFOtEwvKFepK4Vilja8rbTFYPeU2YHLXZ +sllYW4Y200z5P8u+5h1yDm9YHH3L+OnU9mqNmY8IMbVHMfq60HOXV868+KZyShtS +CFt4Gp7qSBsjDMndN3ca01H6lqEwa5CLXAFdDA/+JCeiZJa6PTUFBB9uXbq0s6xZ +sxbpCU8PPNNrC8RSNz/wjETMungDo4QAvuLieX4l8Zpz7UkYF7pLDnYiwx6E5prB +xpoFso1SUkClhhyr34+V5xjsmekLdE9KDo/aknQSy9c7PzjfEHxCTc0McioG55lO +WtwjI1+7kFRCkF0AkPH0BlIJ/hav+RwNsg2STbqYYMkUqWcasVo= +=e0Y9 +-END PGP SIGNATURE- Added: dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.sha512 == --- dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.sha512 (added) +++ dev/mesos/1.11.0-rc1/mesos-1.11.0.tar.gz.sha512 Tue Nov 17 14:27:17 2020 @@ -0,0 +1 @@ +48ecc9457f8f86fc3a7f769a43539cfbb1e2178069aaea19ccaa78917612a2ddaefa033a5ace40ccf6f483be4c49e154008c85bc9c715b07415448e2b93ebdba mesos-1.11.0.tar.gz
[mesos] annotated tag 1.11.0-rc1 created (now 213739c)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to annotated tag 1.11.0-rc1 in repository https://gitbox.apache.org/repos/asf/mesos.git. at 213739c (tag) tagging 04bc7ee1984d86fbb0bd782ccc69ce44c13ae4b9 (commit) by Andrei Sekretenko on Tue Nov 17 14:47:18 2020 +0100 - Log - Tagging Mesos 1.11.0-rc1. --- No new revisions were added by this update.
[mesos] branch 1.11.x updated: Added the `--offer_constraints_re2_max*` flags to the documentation.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.11.x in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/1.11.x by this push: new 04bc7ee Added the `--offer_constraints_re2_max*` flags to the documentation. 04bc7ee is described below commit 04bc7ee1984d86fbb0bd782ccc69ce44c13ae4b9 Author: Andrei Sekretenko AuthorDate: Mon Nov 16 16:42:11 2020 +0100 Added the `--offer_constraints_re2_max*` flags to the documentation. --- docs/configuration/master.md | 26 ++ 1 file changed, 26 insertions(+) diff --git a/docs/configuration/master.md b/docs/configuration/master.md index 6c68b8b..e8c8442 100644 --- a/docs/configuration/master.md +++ b/docs/configuration/master.md @@ -485,6 +485,32 @@ If not set, offers do not timeout. + + +--offer_constraints_re2_max_mem=VALUE + + +Limit on the memory usage of each RE2 regular expression in +framework's offer constraints. If `OfferConstraints` contain a regex +from which a RE2 object cannot be constructed without exceeding this +limit, then framework's attempt to subscribe or update subscription +with these `OfferConstraints` will fail. (default: 4KB) + + + + + +--offer_constraints_re2_max_program_size=VALUE + + +Limit on the RE2 program size of each regular expression in +framework's offer constraints. If `OfferConstraints` contain a regex +which results in a RE2 object exceeding this limit, +then framework's attempt to subscribe or update subscription +with these `OfferConstraints` will fail. (default: 100) + + + --[no-]publish_per_framework_metrics
[mesos] branch master updated: Added the `--offer_constraints_re2_max*` flags to the documentation.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 2fbdb30 Added the `--offer_constraints_re2_max*` flags to the documentation. 2fbdb30 is described below commit 2fbdb30a355caeeb1b9e038d9de7f66693180c8a Author: Andrei Sekretenko AuthorDate: Mon Nov 16 16:42:11 2020 +0100 Added the `--offer_constraints_re2_max*` flags to the documentation. --- docs/configuration/master.md | 26 ++ 1 file changed, 26 insertions(+) diff --git a/docs/configuration/master.md b/docs/configuration/master.md index 6c68b8b..e8c8442 100644 --- a/docs/configuration/master.md +++ b/docs/configuration/master.md @@ -485,6 +485,32 @@ If not set, offers do not timeout. + + +--offer_constraints_re2_max_mem=VALUE + + +Limit on the memory usage of each RE2 regular expression in +framework's offer constraints. If `OfferConstraints` contain a regex +from which a RE2 object cannot be constructed without exceeding this +limit, then framework's attempt to subscribe or update subscription +with these `OfferConstraints` will fail. (default: 4KB) + + + + + +--offer_constraints_re2_max_program_size=VALUE + + +Limit on the RE2 program size of each regular expression in +framework's offer constraints. If `OfferConstraints` contain a regex +which results in a RE2 object exceeding this limit, +then framework's attempt to subscribe or update subscription +with these `OfferConstraints` will fail. (default: 100) + + + --[no-]publish_per_framework_metrics
[mesos] branch 1.11.x updated: Updated CHANGELOG for 1.11.0.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.11.x in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/1.11.x by this push: new ca49ec4 Updated CHANGELOG for 1.11.0. ca49ec4 is described below commit ca49ec4a30d5ee46f695df1a7bf4aad5cf2d7a16 Author: Andrei Sekretenko AuthorDate: Fri Nov 13 22:21:52 2020 +0100 Updated CHANGELOG for 1.11.0. --- CHANGELOG | 99 ++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 951d1af..98dc044 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,107 @@ -Release Notes - Mesos - Version 1.11.0 (WIP) +Release Notes - Mesos - Version 1.11.0 --- +This release contains the following highlights: + + * Mesos Containerizer now supports using pre-provisioned external CSI storage +volumes by means of the new `volume/csi` isolator; the latter significantly +extends the range of compatible 3rd party CSI plugins compared to the +already existing SLRP-based solution (MESOS-10141). + + * The Scheduler API adds an interface allowing frameworks to put constraints +on agent attributes in resource offers to help "picky" frameworks +significantly reduce scheduling latency when close to being out of quota +(MESOS-10161). + + * The CMake build becomes usable for deploying in production (MESOS-898). Additional API Changes: * **Breaking change** Deprecated authentication credential text format support. +Unresolved Critical Issues: + * [MESOS-10194] - Mesos master failure "Check failed: 'get_(role)' Must be SOME" + * [MESOS-10186] - Segmentation fault while running mesos in SSL mode + * [MESOS-10146] - Removing task from slave when framework is disconnected causes master to crash + * [MESOS-10066] - mesos-docker-executor process dies when agent stops. Recovery fails when agent returns + * [MESOS-10011] - Operation feedback with stale agent ID crashes the master + * [MESOS-9967] - Authorization header is missing when using a default registry + * [MESOS-9579] - ExecutorHttpApiTest.HeartbeatCalls is flaky. + * [MESOS-9536] - Nested container launched with non-root user may not be able to write to its sandbox via the environment variable `MESOS_SANDBOX` + * [MESOS-9500] - spark submit with docker image on mesos cluster fails. + * [MESOS-9426] - ZK master detection can become forever pending. + * [MESOS-9393] - Fetcher crashes extracting archives with non-ASCII filenames. + * [MESOS-9365] - Windows - GET_CONTAINERS API call causes the Mesos agent to fail + * [MESOS-9355] - Persistence volume does not unmount correctly with wrong artifact URI + * [MESOS-9352] - Data in persistent volume deleted accidentally when using Docker container and Persistent volume + * [MESOS-9053] - Network ports isolator can falsely trigger while destroying containers. + * [MESOS-9006] - The agent's GET_AGENT leaks resource information when using authorization + * [MESOS-8840] - `cpu.cfs_quota_us` may be accidentally set for command task using docker during agent recovery. + * [MESOS-8803] - Libprocess deadlocks in a test. + * [MESOS-8679] - "If the first KILL stuck in the default executor, all other KILLs will be ignored." + * [MESOS-8608] - RmdirContinueOnErrorTest.RemoveWithContinueOnError fails. + * [MESOS-8257] - "Unified Containerizer ""leaks"" a target container mount path to the host FS when the target resolves to an absolute path" + * [MESOS-8256] - Libprocess can silently deadlock due to worker thread exhaustion. + * [MESOS-8096] - Enqueueing events in MockHTTPScheduler can lead to segfaults. + * [MESOS-8038] - Launching GPU task sporadically fails. + * [MESOS-7971] - PersistentVolumeEndpointsTest.EndpointCreateThenOfferRemove test is flaky + * [MESOS-7911] - Non-checkpointing framework's tasks should not be marked LOST when agent disconnects. + * [MESOS-7748] - Slow subscribers of streaming APIs can lead to Mesos OOMing. + * [MESOS-7721] - Master's agent removal rate limit also applies to agent unreachability. + * [MESOS-7566] - Master crash due to failed check in DRFSorter::remove + * [MESOS-7386] - Executor not cleaning up existing running docker containers if external logrotate/logger processes die/killed + * [MESOS-6285] - Agents may OOM during recovery if there are too many tasks or executors + * [MESOS-5989] - Libevent SSL Socket downgrade code accesses uninitialized memory / assumes single peek is sufficient. + +All Resolved Issues: + +** Bug +* [MESOS-7485] - Add verbose logging for curl commands used in fetcher/puller +* [MESOS-7834] - CMake does not set default --launcher_dir correctly +* [MESOS-9609] - Master check failure when marking agent unreachable. +* [MESOS
[mesos] branch master updated: Updated CHANGELOG for 1.11.0.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git The following commit(s) were added to refs/heads/master by this push: new 934f6d9 Updated CHANGELOG for 1.11.0. 934f6d9 is described below commit 934f6d90af933ea75252ea825a5cce2718741570 Author: Andrei Sekretenko AuthorDate: Fri Nov 13 22:21:52 2020 +0100 Updated CHANGELOG for 1.11.0. --- CHANGELOG | 99 ++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 951d1af..98dc044 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,107 @@ -Release Notes - Mesos - Version 1.11.0 (WIP) +Release Notes - Mesos - Version 1.11.0 --- +This release contains the following highlights: + + * Mesos Containerizer now supports using pre-provisioned external CSI storage +volumes by means of the new `volume/csi` isolator; the latter significantly +extends the range of compatible 3rd party CSI plugins compared to the +already existing SLRP-based solution (MESOS-10141). + + * The Scheduler API adds an interface allowing frameworks to put constraints +on agent attributes in resource offers to help "picky" frameworks +significantly reduce scheduling latency when close to being out of quota +(MESOS-10161). + + * The CMake build becomes usable for deploying in production (MESOS-898). Additional API Changes: * **Breaking change** Deprecated authentication credential text format support. +Unresolved Critical Issues: + * [MESOS-10194] - Mesos master failure "Check failed: 'get_(role)' Must be SOME" + * [MESOS-10186] - Segmentation fault while running mesos in SSL mode + * [MESOS-10146] - Removing task from slave when framework is disconnected causes master to crash + * [MESOS-10066] - mesos-docker-executor process dies when agent stops. Recovery fails when agent returns + * [MESOS-10011] - Operation feedback with stale agent ID crashes the master + * [MESOS-9967] - Authorization header is missing when using a default registry + * [MESOS-9579] - ExecutorHttpApiTest.HeartbeatCalls is flaky. + * [MESOS-9536] - Nested container launched with non-root user may not be able to write to its sandbox via the environment variable `MESOS_SANDBOX` + * [MESOS-9500] - spark submit with docker image on mesos cluster fails. + * [MESOS-9426] - ZK master detection can become forever pending. + * [MESOS-9393] - Fetcher crashes extracting archives with non-ASCII filenames. + * [MESOS-9365] - Windows - GET_CONTAINERS API call causes the Mesos agent to fail + * [MESOS-9355] - Persistence volume does not unmount correctly with wrong artifact URI + * [MESOS-9352] - Data in persistent volume deleted accidentally when using Docker container and Persistent volume + * [MESOS-9053] - Network ports isolator can falsely trigger while destroying containers. + * [MESOS-9006] - The agent's GET_AGENT leaks resource information when using authorization + * [MESOS-8840] - `cpu.cfs_quota_us` may be accidentally set for command task using docker during agent recovery. + * [MESOS-8803] - Libprocess deadlocks in a test. + * [MESOS-8679] - "If the first KILL stuck in the default executor, all other KILLs will be ignored." + * [MESOS-8608] - RmdirContinueOnErrorTest.RemoveWithContinueOnError fails. + * [MESOS-8257] - "Unified Containerizer ""leaks"" a target container mount path to the host FS when the target resolves to an absolute path" + * [MESOS-8256] - Libprocess can silently deadlock due to worker thread exhaustion. + * [MESOS-8096] - Enqueueing events in MockHTTPScheduler can lead to segfaults. + * [MESOS-8038] - Launching GPU task sporadically fails. + * [MESOS-7971] - PersistentVolumeEndpointsTest.EndpointCreateThenOfferRemove test is flaky + * [MESOS-7911] - Non-checkpointing framework's tasks should not be marked LOST when agent disconnects. + * [MESOS-7748] - Slow subscribers of streaming APIs can lead to Mesos OOMing. + * [MESOS-7721] - Master's agent removal rate limit also applies to agent unreachability. + * [MESOS-7566] - Master crash due to failed check in DRFSorter::remove + * [MESOS-7386] - Executor not cleaning up existing running docker containers if external logrotate/logger processes die/killed + * [MESOS-6285] - Agents may OOM during recovery if there are too many tasks or executors + * [MESOS-5989] - Libevent SSL Socket downgrade code accesses uninitialized memory / assumes single peek is sufficient. + +All Resolved Issues: + +** Bug +* [MESOS-7485] - Add verbose logging for curl commands used in fetcher/puller +* [MESOS-7834] - CMake does not set default --launcher_dir correctly +* [MESOS-9609] - Master check failure when marking agent unreachable. +* [MESOS
[mesos] branch 1.11.x updated: Documented setting offer constraints via the scheduler API.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.11.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/1.11.x by this push:
new 8540dc8 Documented setting offer constraints via the scheduler API.
8540dc8 is described below
commit 8540dc85d4c6a2627125f272cfdb8922ac68b30d
Author: Andrei Sekretenko
AuthorDate: Tue Nov 3 21:19:54 2020 +0100
Documented setting offer constraints via the scheduler API.
Review: https://reviews.apache.org/r/73004
---
docs/app-framework-development-guide.md | 6
docs/scheduler-http-api.md | 60 -
2 files changed, 57 insertions(+), 9 deletions(-)
diff --git a/docs/app-framework-development-guide.md
b/docs/app-framework-development-guide.md
index 8f30045..4653e1a 100644
--- a/docs/app-framework-development-guide.md
+++ b/docs/app-framework-development-guide.md
@@ -66,6 +66,12 @@ schedulers in the same Mesos cluster:
up-to-date with the minimum desired offer shape for each role will ensure
that
the sccheduler gets a better chance to receive offers sized with sufficient
resources.
+6. Consider specifying **offer constraints** via `SUBSCRIBE`/`UPDATE_FRAMEWORK`
+ calls so that the framework role's quota is not consumed by offers that the
+ scheduler will have to decline anyway based on agent attributes.
+ See [MESOS-10161](https://issues.apache.org/jira/browse/MESOS-10161])
+ and
[scheduler.proto](https://github.com/apache/mesos/blob/master/include/mesos/v1/scheduler/scheduler.proto)
+ for more details.
Operationally, the following can be done to ensure that schedulers get the
resources
they need when co-existing with other schedulers:
diff --git a/docs/scheduler-http-api.md b/docs/scheduler-http-api.md
index e447ffb..6f4a804 100644
--- a/docs/scheduler-http-api.md
+++ b/docs/scheduler-http-api.md
@@ -55,7 +55,9 @@ The response returned from the `SUBSCRIBE` call (see
[below](#subscribe)) is enc
This is the first step in the communication process between the scheduler and
the master. This is also to be considered as subscription to the "/scheduler"
event stream.
-To subscribe with the master, the scheduler sends an HTTP POST with a
`SUBSCRIBE` message including the required FrameworkInfo and the list of
initially suppressed roles (which must be a subset of roles in FrameworkInfo,
see the section for `SUPPRESS` call). Note that if
"subscribe.framework_info.id" and "FrameworkID" are not set, the master
considers the scheduler as a new one and subscribes it by assigning it a
FrameworkID. The HTTP response is a stream in RecordIO format; the event st
[...]
+To subscribe with the master, the scheduler sends an HTTP POST with a
`SUBSCRIBE` message including the required FrameworkInfo, the list of initially
suppressed roles and the initial offer constraints. The initially suppressed
roles, as well as roles for which offer constraints are specified, must be
contained in the set of roles in FrameworkInfo. Note that Mesos 1.11.0 simply
ignores constraints for invalid roles, but this might change in the future.
+
+Note that if "subscribe.framework_info.id" and "FrameworkID" are not set, the
master considers the scheduler as a new one and subscribes it by assigning it a
FrameworkID. The HTTP response is a stream in RecordIO format; the event stream
begins with either a `SUBSCRIBED` event or an `ERROR` event (see details in
**Events** section). The response also includes the `Mesos-Stream-Id` header,
which is used by the master to uniquely identify the subscribed scheduler
instance. This stream ID h [...]
```
SUBSCRIBE Request (JSON):
@@ -76,8 +78,20 @@ Connection: close
"roles": ["test1", "test2"],
"capabilities" : [{"type": "MULTI_ROLE"}]
},
- "suppressed_roles" : ["test2"]
- }
+ "suppressed_roles" : ["test2"],
+ "offer_constraints" : {
+"role_constraints": {
+ "test1": {
+"groups": [{
+ "attribute_constraints": [{
+"selector": {"attribute_name": "foo"},
+"predicate": {"exists": {}}
+ }]
+}]
+ }
+}
+ }
+ }
}
SUBSCRIBE Response Event (JSON):
@@ -103,10 +117,11 @@ with a `SUBSCRIBED` event. For further details, see the
**Disconnections** secti
NOTE: In the old version of the API, (re-)registered callbacks also included
MasterInfo, which contained information about the master the driver currently
connected to. With the new API, since schedulers explicitly subscribe with the
leading master (see details below in **Master De
[mesos] branch master updated: Documented setting offer constraints via the scheduler API.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new ab272be Documented setting offer constraints via the scheduler API.
ab272be is described below
commit ab272be96edc40b69db5503e56b994198939db40
Author: Andrei Sekretenko
AuthorDate: Tue Nov 3 21:19:54 2020 +0100
Documented setting offer constraints via the scheduler API.
Review: https://reviews.apache.org/r/73004
---
docs/app-framework-development-guide.md | 6
docs/scheduler-http-api.md | 60 -
2 files changed, 57 insertions(+), 9 deletions(-)
diff --git a/docs/app-framework-development-guide.md
b/docs/app-framework-development-guide.md
index 8f30045..4653e1a 100644
--- a/docs/app-framework-development-guide.md
+++ b/docs/app-framework-development-guide.md
@@ -66,6 +66,12 @@ schedulers in the same Mesos cluster:
up-to-date with the minimum desired offer shape for each role will ensure
that
the sccheduler gets a better chance to receive offers sized with sufficient
resources.
+6. Consider specifying **offer constraints** via `SUBSCRIBE`/`UPDATE_FRAMEWORK`
+ calls so that the framework role's quota is not consumed by offers that the
+ scheduler will have to decline anyway based on agent attributes.
+ See [MESOS-10161](https://issues.apache.org/jira/browse/MESOS-10161])
+ and
[scheduler.proto](https://github.com/apache/mesos/blob/master/include/mesos/v1/scheduler/scheduler.proto)
+ for more details.
Operationally, the following can be done to ensure that schedulers get the
resources
they need when co-existing with other schedulers:
diff --git a/docs/scheduler-http-api.md b/docs/scheduler-http-api.md
index e447ffb..6f4a804 100644
--- a/docs/scheduler-http-api.md
+++ b/docs/scheduler-http-api.md
@@ -55,7 +55,9 @@ The response returned from the `SUBSCRIBE` call (see
[below](#subscribe)) is enc
This is the first step in the communication process between the scheduler and
the master. This is also to be considered as subscription to the "/scheduler"
event stream.
-To subscribe with the master, the scheduler sends an HTTP POST with a
`SUBSCRIBE` message including the required FrameworkInfo and the list of
initially suppressed roles (which must be a subset of roles in FrameworkInfo,
see the section for `SUPPRESS` call). Note that if
"subscribe.framework_info.id" and "FrameworkID" are not set, the master
considers the scheduler as a new one and subscribes it by assigning it a
FrameworkID. The HTTP response is a stream in RecordIO format; the event st
[...]
+To subscribe with the master, the scheduler sends an HTTP POST with a
`SUBSCRIBE` message including the required FrameworkInfo, the list of initially
suppressed roles and the initial offer constraints. The initially suppressed
roles, as well as roles for which offer constraints are specified, must be
contained in the set of roles in FrameworkInfo. Note that Mesos 1.11.0 simply
ignores constraints for invalid roles, but this might change in the future.
+
+Note that if "subscribe.framework_info.id" and "FrameworkID" are not set, the
master considers the scheduler as a new one and subscribes it by assigning it a
FrameworkID. The HTTP response is a stream in RecordIO format; the event stream
begins with either a `SUBSCRIBED` event or an `ERROR` event (see details in
**Events** section). The response also includes the `Mesos-Stream-Id` header,
which is used by the master to uniquely identify the subscribed scheduler
instance. This stream ID h [...]
```
SUBSCRIBE Request (JSON):
@@ -76,8 +78,20 @@ Connection: close
"roles": ["test1", "test2"],
"capabilities" : [{"type": "MULTI_ROLE"}]
},
- "suppressed_roles" : ["test2"]
- }
+ "suppressed_roles" : ["test2"],
+ "offer_constraints" : {
+"role_constraints": {
+ "test1": {
+"groups": [{
+ "attribute_constraints": [{
+"selector": {"attribute_name": "foo"},
+"predicate": {"exists": {}}
+ }]
+}]
+ }
+}
+ }
+ }
}
SUBSCRIBE Response Event (JSON):
@@ -103,10 +117,11 @@ with a `SUBSCRIBED` event. For further details, see the
**Disconnections** secti
NOTE: In the old version of the API, (re-)registered callbacks also included
MasterInfo, which contained information about the master the driver currently
connected to. With the new API, since schedulers explicitly subscribe with the
leading master (see details below in **Master De
[mesos] branch master updated (3a81cc9 -> c1e7160)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 3a81cc9 Updated Postgres URL in CentOS 6 Dockerfile. new 4150559 Moved failover timeout validation to stateless FrameworkInfo validation. new 3fcc148 Consolidated creation and validation of `allocator::Framework` options. new c1e7160 Added validation that offer constraints are set only for existing roles. The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/master/master.cpp | 191 ++-- src/master/master.hpp | 13 +- src/master/validation.cpp | 59 - src/master/validation.hpp | 8 ++ src/tests/master/update_framework_tests.cpp | 87 + 5 files changed, 253 insertions(+), 105 deletions(-)
[mesos] 03/03: Added validation that offer constraints are set only for existing roles.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c1e716054d8dead61074c8619ffa7b33f3064152
Author: Andrei Sekretenko
AuthorDate: Mon Oct 12 22:06:51 2020 +0200
Added validation that offer constraints are set only for existing roles.
This patch makes SUBSCRIBE/UPDATE_FRAMEWORK calls validate that
the framework does not specify offer constraints for roles to which
it is not going to be subscribed.
Review: https://reviews.apache.org/r/72956
---
src/master/master.cpp | 8 ++-
src/master/validation.cpp | 19 +++
src/master/validation.hpp | 4 ++
src/tests/master/update_framework_tests.cpp | 87 +
4 files changed, 117 insertions(+), 1 deletion(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 531b971..164720a 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -2619,7 +2619,13 @@ static Try
createAllocatorFrameworkOptions(
return *error;
}
- // TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
+ error = validation::framework::validateOfferConstraintsRoles(
+ validFrameworkRoles, offerConstraints);
+
+ if (error.isSome()) {
+return *error;
+ }
+
Try filter = OfferConstraintsFilter::create(
filterOptions, std::move(offerConstraints));
diff --git a/src/master/validation.cpp b/src/master/validation.cpp
index aafffd5..6bdab54 100644
--- a/src/master/validation.cpp
+++ b/src/master/validation.cpp
@@ -59,6 +59,8 @@ using std::vector;
using google::protobuf::RepeatedPtrField;
+using mesos::scheduler::OfferConstraints;
+
namespace mesos {
namespace internal {
namespace master {
@@ -697,6 +699,23 @@ Option validateSuppressedRoles(
}
+Option validateOfferConstraintsRoles(
+const set& validFrameworkRoles,
+const OfferConstraints& offerConstraints)
+{
+ for (const auto& pair : offerConstraints.role_constraints()) {
+const string& role = pair.first;
+if (validFrameworkRoles.count(role) < 1) {
+ return Error(
+ "Offer constraints specify `role_constraints` for a role '" + role +
+ "'not contained in the set of roles");
+}
+ }
+
+ return None();
+}
+
+
} // namespace framework {
diff --git a/src/master/validation.hpp b/src/master/validation.hpp
index 6239492..17652e3 100644
--- a/src/master/validation.hpp
+++ b/src/master/validation.hpp
@@ -134,6 +134,10 @@ Option validateSuppressedRoles(
const std::set& validFrameworkRoles,
const std::set& suppressedRoles);
+Option validateOfferConstraintsRoles(
+const std::set& validFrameworkRoles,
+const scheduler::OfferConstraints& offerConstraints);
+
} // namespace framework {
diff --git a/src/tests/master/update_framework_tests.cpp
b/src/tests/master/update_framework_tests.cpp
index 3f86573..ce5a51c 100644
--- a/src/tests/master/update_framework_tests.cpp
+++ b/src/tests/master/update_framework_tests.cpp
@@ -981,6 +981,93 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
}
+// This test ensures that an UPDATE_FRAMEWORK call trying to set offer
+// constraints for a role to which the framework will not be subscribed
+// fails validation and is rejected as a whole.
+TEST_F(UpdateFrameworkTest, OfferConstraintsForUnsubscribedRole)
+{
+ using ::mesos::v1::scheduler::OfferConstraints;
+
+ const Try constraintsJson = JSON::parse(
+ R"~(
+{
+ "role_constraints": {
+")~" + DEFAULT_FRAMEWORK_INFO.roles(0) + R"~(": {
+ "groups": [{
+"attribute_constraints": [{
+ "selector": {"attribute_name": "foo"},
+ "predicate": {"exists": {}}
+}]
+ }]
+}
+ }
+})~");
+
+ ASSERT_SOME(constraintsJson);
+
+ const Try constraints =
+::protobuf::parse(*constraintsJson);
+
+ ASSERT_SOME(constraints);
+
+ Try> master = StartMaster(CreateMasterFlags());
+ ASSERT_SOME(master);
+
+ auto scheduler = std::make_shared();
+
+ Future connected;
+ EXPECT_CALL(*scheduler, connected(_))
+.WillOnce(Invoke([constraints](Mesos* mesos) {
+ Call call;
+ call.set_type(Call::SUBSCRIBE);
+ *call.mutable_subscribe()->mutable_framework_info() =
+DEFAULT_FRAMEWORK_INFO;
+
+ *call.mutable_subscribe()->mutable_offer_constraints() = *constraints;
+
+ mesos->send(call);
+}));
+
+ EXPECT_CALL(*scheduler, heartbeat(_))
+.WillRepeatedly(Return()); // Ignore heartbeats.
+
+ Future subscribed;
+
+ EXPECT_CALL(*scheduler, subscribed(_, _))
+.WillOnce(FutureArg<1>());
+
+ TestMesos mesos(master->get()->pid
[mesos] 02/03: Consolidated creation and validation of `allocator::Framework` options.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 3fcc148d5d28405f4256e4cfa7005786de10b4f9
Author: Andrei Sekretenko
AuthorDate: Mon Oct 12 20:47:01 2020 +0200
Consolidated creation and validation of `allocator::Framework` options.
This merges three near-identical pieces of scattered code in SUBSCRIBE
and UPDATE_FRAMEWORK execution paths in the Master that validate
and construct parts of `allocator::FrameworkOptions` (the set of
suppressed roles and the offer constraints filter) into a single
function.
This is a prerequisite to adding validation of offer constraint roles.
Review: https://reviews.apache.org/r/72955
---
src/master/master.cpp | 172 +++---
src/master/master.hpp | 13 ++--
src/master/validation.cpp | 20 ++
src/master/validation.hpp | 4 ++
4 files changed, 118 insertions(+), 91 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 6c0523d..531b971 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -2551,8 +2551,7 @@ void Master::reregisterFramework(
Option Master::validateFramework(
- const FrameworkInfo& frameworkInfo,
- const google::protobuf::RepeatedPtrField& suppressedRoles) const
+const FrameworkInfo& frameworkInfo) const
{
Option validationError =
validation::framework::validate(frameworkInfo);
@@ -2583,17 +2582,6 @@ Option Master::validateFramework(
" are not present in the master's --roles");
}
- // Ensure each of the suppressed role is contained in the list of roles.
- set frameworkRoles = protobuf::framework::getRoles(frameworkInfo);
- // The suppressed roles must be contained within the list of all
- // roles for the framwork.
- foreach (const string& role, suppressedRoles) {
-if (!frameworkRoles.count(role)) {
- return Error("Suppressed role '" + role +
- "' is not contained in the list of roles");
-}
- }
-
// TODO(vinod): Deprecate this in favor of authorization.
if (frameworkInfo.user() == "root" && !flags.root_submissions) {
return Error("User 'root' is not allowed to run frameworks"
@@ -2614,6 +2602,37 @@ Option Master::validateFramework(
}
+static Try createAllocatorFrameworkOptions(
+const set& validFrameworkRoles,
+const OfferConstraintsFilter::Options filterOptions,
+google::protobuf::RepeatedPtrField&& suppressedRoles,
+OfferConstraints offerConstraints)
+{
+ set suppressedRolesSet(
+ make_move_iterator(suppressedRoles.begin()),
+ make_move_iterator(suppressedRoles.end()));
+
+ Option error = validation::framework::validateSuppressedRoles(
+ validFrameworkRoles, suppressedRolesSet);
+
+ if (error.isSome()) {
+return *error;
+ }
+
+ // TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
+ Try filter = OfferConstraintsFilter::create(
+ filterOptions, std::move(offerConstraints));
+
+ if (filter.isError()) {
+return Error(
+"Offer constraints are not valid: " + std::move(filter.error()));
+ }
+
+ return allocator::FrameworkOptions{
+std::move(suppressedRolesSet), std::move(*filter)};
+}
+
+
// Returns None if the framework object approvers are ready and the scheduler
// trying to SUBSCRIBE is authorized to do so with provided framework info.
// Otherwise, returns an error to be sent to the scheduler trying to subscribe.
@@ -2665,40 +2684,34 @@ void Master::subscribe(
LOG(INFO) << "Received subscription request for"
<< " HTTP framework '" << frameworkInfo.name() << "'";
- Option validationError =
-validateFramework(frameworkInfo, subscribe.suppressed_roles());
-
- allocator::FrameworkOptions allocatorOptions;
-
- // TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
- if (validationError.isNone()) {
-Try filter = OfferConstraintsFilter::create(
-offerConstraintsFilterOptions,
-OfferConstraints(subscribe.offer_constraints()));
-
-if (filter.isError()) {
- validationError = Error(std::move(filter.error()));
-} else {
- allocatorOptions.offerConstraintsFilter = std::move(*filter);
-}
- }
-
- if (validationError.isSome()) {
+ auto refuseSubscription = [&](const string& error) {
LOG(INFO) << "Refusing subscription of framework"
- << " '" << frameworkInfo.name() << "': "
- << validationError->message;
+ << " '" << frameworkInfo.name() << "': " << error;
FrameworkErrorMessage message;
-message.set_message(validationErro
[mesos] 01/03: Moved failover timeout validation to stateless FrameworkInfo validation.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 4150559ab29e37c8b5d65024ce3a728b359de2f1
Author: Andrei Sekretenko
AuthorDate: Wed Oct 14 17:46:46 2020 +0200
Moved failover timeout validation to stateless FrameworkInfo validation.
This turns the validation of the failover timeout in `FrameworkInfo`
into a part of `validation::framework::validate()` that performs
all the other validations that depend on `FrameworkInfo` only.
Review: https://reviews.apache.org/r/72964
---
src/master/master.cpp | 13 -
src/master/validation.cpp | 20 +++-
2 files changed, 19 insertions(+), 14 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index d6d3ea7..6c0523d 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -157,8 +157,6 @@ using mesos::master::detector::MasterDetector;
using mesos::scheduler::OfferConstraints;
-static bool isValidFailoverTimeout(const FrameworkInfo& frameworkInfo);
-
class SlaveObserver : public ProtobufProcess
{
@@ -2612,11 +2610,6 @@ Option Master::validateFramework(
return Error("Framework has been removed");
}
- if (!isValidFailoverTimeout(frameworkInfo)) {
-return Error("The framework failover_timeout (" +
- stringify(frameworkInfo.failover_timeout()) +
- ") is invalid");
- }
return Option::none();
}
@@ -12319,12 +12312,6 @@ double Master::_resources_revocable_percent(const
string& name)
}
-static bool isValidFailoverTimeout(const FrameworkInfo& frameworkInfo)
-{
- return Duration::create(frameworkInfo.failover_timeout()).isSome();
-}
-
-
void Master::Subscribers::send(
const mesos::master::Event& event,
const Option& frameworkInfo,
diff --git a/src/master/validation.cpp b/src/master/validation.cpp
index 5b1bcb5..feeea8e 100644
--- a/src/master/validation.cpp
+++ b/src/master/validation.cpp
@@ -560,13 +560,25 @@ Option validateOfferFilters(const FrameworkInfo&
frameworkInfo)
return None();
}
+
+Option validateFailoverTimeout(const FrameworkInfo& frameworkInfo)
+{
+ if (Duration::create(frameworkInfo.failover_timeout()).isSome()) {
+return None();
+ }
+
+ return Error(
+ "The framework failover_timeout (" +
+ stringify(frameworkInfo.failover_timeout()) + ") is invalid");
+}
+
} // namespace internal {
Option validate(const mesos::FrameworkInfo& frameworkInfo)
{
// TODO(jay_guo): This currently only validates the role(s),
- // framework ID and offer filters, validate more fields!
+ // framework ID, offer filters and failover timeout, validate more fields!
Option error = internal::validateRoles(frameworkInfo);
if (error.isSome()) {
@@ -585,6 +597,12 @@ Option validate(const mesos::FrameworkInfo&
frameworkInfo)
return error;
}
+ error = internal::validateFailoverTimeout(frameworkInfo);
+
+ if(error.isSome()) {
+return error;
+ }
+
return None();
}
[mesos] branch 1.11.x updated: Fixed javadoc error `type arguments not allowed here`.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.11.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/1.11.x by this push:
new cb6cfe9 Fixed javadoc error `type arguments not allowed here`.
cb6cfe9 is described below
commit cb6cfe9b122d1b60a8264b28b6abb38a3c8417b4
Author: Andrei Sekretenko
AuthorDate: Wed Oct 21 21:52:49 2020 +0200
Fixed javadoc error `type arguments not allowed here`.
This fixes a javadoc build failure introduced by
c28fd3a93e0d9d9a868aec2380abd1dd338304ef that has been occurring
on platforms that use older versions of javadoc.
---
src/java/src/org/apache/mesos/MesosSchedulerDriver.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
index 9be0ec1..5bde34d 100644
--- a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
+++ b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
@@ -410,7 +410,7 @@ public class MesosSchedulerDriver implements
SchedulerDriver {
/**
* @deprecated Replaced by
- * {@link #updateFramework(FrameworkInfo, Collection,
OfferConstraints)}
+ * {@link #updateFramework(FrameworkInfo, Collection, OfferConstraints)}
*
* NOTE: The underlying JNI method exists only to maintain compatibility
* of newer versions of libmesos-java.so with older versions of mesos.jar
[mesos] branch master updated: Fixed javadoc error `type arguments not allowed here`.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 550bb4e Fixed javadoc error `type arguments not allowed here`.
550bb4e is described below
commit 550bb4e3d3301cc87937a30344d8bee74bd53534
Author: Andrei Sekretenko
AuthorDate: Wed Oct 21 21:52:49 2020 +0200
Fixed javadoc error `type arguments not allowed here`.
This fixes a javadoc build failure introduced by
c28fd3a93e0d9d9a868aec2380abd1dd338304ef that has been occurring
on platforms that use older versions of javadoc.
---
src/java/src/org/apache/mesos/MesosSchedulerDriver.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
index 9be0ec1..5bde34d 100644
--- a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
+++ b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
@@ -410,7 +410,7 @@ public class MesosSchedulerDriver implements
SchedulerDriver {
/**
* @deprecated Replaced by
- * {@link #updateFramework(FrameworkInfo, Collection,
OfferConstraints)}
+ * {@link #updateFramework(FrameworkInfo, Collection, OfferConstraints)}
*
* NOTE: The underlying JNI method exists only to maintain compatibility
* of newer versions of libmesos-java.so with older versions of mesos.jar
[mesos] branch master updated: Updated Mesos version to 1.12.0.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 3045f57 Updated Mesos version to 1.12.0.
3045f57 is described below
commit 3045f578ed48e72fcf3aeb7e97baf92260c53708
Author: Andrei Sekretenko
AuthorDate: Thu Oct 15 16:19:46 2020 +0200
Updated Mesos version to 1.12.0.
---
CMakeLists.txt | 2 +-
configure.ac | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4390aa8..f18c5a4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,7 +24,7 @@ endif ()
project(Mesos)
set(MESOS_MAJOR_VERSION 1)
-set(MESOS_MINOR_VERSION 11)
+set(MESOS_MINOR_VERSION 12)
set(MESOS_PATCH_VERSION 0)
set(PACKAGE_VERSION
${MESOS_MAJOR_VERSION}.${MESOS_MINOR_VERSION}.${MESOS_PATCH_VERSION})
diff --git a/configure.ac b/configure.ac
index b19440a..b346208 100644
--- a/configure.ac
+++ b/configure.ac
@@ -18,7 +18,7 @@
# Process this file with autoconf to produce a configure script.
AC_PREREQ([2.61])
-AC_INIT([mesos], [1.11.0])
+AC_INIT([mesos], [1.12.0])
# Have autoconf setup some variables related to the system.
AC_CANONICAL_HOST
[mesos] branch 1.11.x created (now d4678d3)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.11.x in repository https://gitbox.apache.org/repos/asf/mesos.git. at d4678d3 Added suppressed roles to `allocator/offer_constraints_debug` endpoint. No new revisions were added by this update.
[mesos] 02/02: Added suppressed roles to `allocator/offer_constraints_debug` endpoint.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit d4678d33b223fec5d48007f8246f1ed1cda5e90d
Author: Andrei Sekretenko
AuthorDate: Mon Oct 12 15:42:50 2020 +0200
Added suppressed roles to `allocator/offer_constraints_debug` endpoint.
This simplifies debugging frameworks that use offer constraints.
For example, in case a framework is not receiving offers for a role,
getting a suppressed roles snapshot simultaneously with agent
filtering results helps to figure out whether the framework is
mis-specifying offer constraints, or just wrongly suppresses a role.
Review: https://reviews.apache.org/r/72954
---
src/master/allocator/mesos/hierarchical.cpp | 16 ++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/master/allocator/mesos/hierarchical.cpp
b/src/master/allocator/mesos/hierarchical.cpp
index 41a4924..b7cb240 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -3231,26 +3231,32 @@ static string OFFER_CONSTRAINTS_DEBUG_HELP()
"Evaluates current framework offer constraints and returns results."),
process::DESCRIPTION(
"This endpoint evaluates for each role of each framework a list",
-"of agents excluded from allocation by offer constraints.",
+"of agents excluded from allocation by offer constraints,",
+"and also reports the current set of framework's suppressed roles",
"",
"Example:",
"```",
"{\"frameworks\": {",
" \"0f4c63a9-be1e-4a90-9e11-d7bf0aa6c8ad-0017\": {",
+" \"suppressed_roles\": [\"role2\"]",
" \"excluded_by_attribute_constraints\": {",
" \"role1\": [",
" \"0b1e7d60-dfbc-44c9-8222-48b57eca8637-S123\",",
" \"654af69c-80f7-45ad-bcb3-c7c917f1811b-S045\"],",
" \"role2\": [] }},",
" \"b0377da6-090d-4338-9e2e-bf6cf0f309b7-0011\": {",
+" \"suppressed_roles\": []",
" \"excluded_by_attribute_constraints\": { \"role1\": [] }}",
"}}",
"```",
"In this example, two agents are excluded from allocation",
"to the first framework (-0017) under the role \"role1\", no agents",
"are excluded from allocation to this framework under \"role2\".",
+"In addition, this framework has \"role2\" suppressed.",
"The second framework (-0011) is also subscribed to \"role1\",",
-"but has no agents excluded from allocation to \"role1\"."),
+"but has no agents excluded from allocation to \"role1\";",
+"this framework has no roles currently suppressed."
+),
process::AUTHENTICATION(true),
process::AUTHORIZATION(
"This endpoint skips frameworks for which the user is not authorized"
@@ -3300,6 +3306,12 @@ process::http::Response
HierarchicalAllocatorProcess::offerConstraintsDebug_(
auto writeFrameworks = [&](JSON::ObjectWriter* writer) {
for (const Framework* framework : approvedFrameworks) {
auto writeFramework = [&](JSON::ObjectWriter* writer) {
+writer->field("suppressed_roles", [&](JSON::ArrayWriter* writer) {
+ for (const string& role : framework->suppressedRoles) {
+writer->element(role);
+ }
+});
+
writer->field(
"excluded_by_attribute_constraints",
[&](JSON::ObjectWriter* writer) {
[mesos] branch master updated (301902b -> d4678d3)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 301902b Ignored the directoy `/dev/nvidia-caps` when globing Nvidia GPU devices. new 8d3a76f Made the offer constraints filter non-optional inside the allocator. new d4678d3 Added suppressed roles to `allocator/offer_constraints_debug` endpoint. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/master/allocator/mesos/hierarchical.cpp | 33 + src/master/allocator/mesos/hierarchical.hpp | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-)
[mesos] 01/02: Made the offer constraints filter non-optional inside the allocator.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8d3a76f4261f8d249504de5330ce86ba07a17f56
Author: Andrei Sekretenko
AuthorDate: Mon Oct 12 15:42:50 2020 +0200
Made the offer constraints filter non-optional inside the allocator.
Now that the master always constructs an offer constraints filter
for a framework (potentially a no-op one) and always passes this filter
into an allocator, storing the filter as an `Option` inside the
hierarchical allocator is no longer necessary.
Review: https://reviews.apache.org/r/72953
---
src/master/allocator/mesos/hierarchical.cpp | 23 ---
src/master/allocator/mesos/hierarchical.hpp | 2 +-
2 files changed, 9 insertions(+), 16 deletions(-)
diff --git a/src/master/allocator/mesos/hierarchical.cpp
b/src/master/allocator/mesos/hierarchical.cpp
index 35264b9..41a4924 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -2178,8 +2178,7 @@ void HierarchicalAllocatorProcess::__generateOffers()
const Framework& framework = *CHECK_NOTNONE(getFramework(frameworkId));
CHECK(framework.active) << frameworkId;
-if (framework.offerConstraintsFilter.isSome() &&
-framework.offerConstraintsFilter->isAgentExcluded(
+if (framework.offerConstraintsFilter.isAgentExcluded(
role, slave.info)) {
// Framework filters the agent regardless of remaining resources.
continue;
@@ -2414,8 +2413,7 @@ void HierarchicalAllocatorProcess::__generateOffers()
const Framework& framework = *CHECK_NOTNONE(getFramework(frameworkId));
-if (framework.offerConstraintsFilter.isSome() &&
-framework.offerConstraintsFilter->isAgentExcluded(
+if (framework.offerConstraintsFilter.isAgentExcluded(
role, slave.info)) {
// Framework filters the agent regardless of remaining resources.
continue;
@@ -3244,14 +3242,15 @@ static string OFFER_CONSTRAINTS_DEBUG_HELP()
" \"0b1e7d60-dfbc-44c9-8222-48b57eca8637-S123\",",
" \"654af69c-80f7-45ad-bcb3-c7c917f1811b-S045\"],",
" \"role2\": [] }},",
-" \"b0377da6-090d-4338-9e2e-bf6cf0f309b7-0011\": {}",
+" \"b0377da6-090d-4338-9e2e-bf6cf0f309b7-0011\": {",
+" \"excluded_by_attribute_constraints\": { \"role1\": [] }}",
"}}",
"```",
"In this example, two agents are excluded from allocation",
"to the first framework (-0017) under the role \"role1\", no agents",
-"are excluded from allocation to this framework under \"role2\",",
-"and the second framework (-0011) has no offer constraints set."
-),
+"are excluded from allocation to this framework under \"role2\".",
+"The second framework (-0011) is also subscribed to \"role1\",",
+"but has no agents excluded from allocation to \"role1\"."),
process::AUTHENTICATION(true),
process::AUTHORIZATION(
"This endpoint skips frameworks for which the user is not authorized"
@@ -3301,19 +3300,13 @@ process::http::Response
HierarchicalAllocatorProcess::offerConstraintsDebug_(
auto writeFrameworks = [&](JSON::ObjectWriter* writer) {
for (const Framework* framework : approvedFrameworks) {
auto writeFramework = [&](JSON::ObjectWriter* writer) {
-if (framework->offerConstraintsFilter.isNone()) {
- // For an authorized frameworks without offer constraints,
- // an empty object is written.
- return;
-}
-
writer->field(
"excluded_by_attribute_constraints",
[&](JSON::ObjectWriter* writer) {
for (const string& role : framework->roles) {
writer->field(role, [&](JSON::ArrayWriter* writer) {
foreachvalue (const Slave& slave, slaves) {
-if (framework->offerConstraintsFilter->isAgentExcluded(
+if (framework->offerConstraintsFilter.isAgentExcluded(
role, slave.info)) {
writer->element(stringify(slave.id));
}
diff --git a/src/master/allocator/mesos/hierarchical.hpp
b/src/master/allocator/mesos/hierarchical.hpp
index 4ec15b6..99b9c24 100644
--- a/src/master/allocator/mesos/hierarchical.hpp
+++ b/src/master/allocator/mesos/hierarchical.hpp
@@ -117,7 +117,7 @@ struct Framework
// offer constraints.
hashmap>
minAllocatableResources;
- Option<::mesos::allocator::OfferConstraintsFilter> offerConstraintsFilter;
+ ::mesos::allocator::OfferConstraintsFilter offerConstraintsFilter;
};
[mesos] branch master updated: Re-added the obsolete `updateFramework` signature into libmesos-java.so.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new c28fd3a Re-added the obsolete `updateFramework` signature into
libmesos-java.so.
c28fd3a is described below
commit c28fd3a93e0d9d9a868aec2380abd1dd338304ef
Author: Andrei Sekretenko
AuthorDate: Thu Oct 1 14:06:59 2020 +0200
Re-added the obsolete `updateFramework` signature into libmesos-java.so.
This patch converts the implementation of the obsolete 2-parameter
`SchedulerDriver.updateFramework(frameworkInfo, suppressedRoles)`
from a wrapper around the new signature back into a JNI method
that 930c7e98d17e71192dae1d49b4b2217cc2dbd8b2 attempted to remove.
This is needed to keep compatibility between older versions of
`mesos.jar` and newer versions of `libmesos-java.so`.
Review: https://reviews.apache.org/r/72922
---
.../jni/org_apache_mesos_MesosSchedulerDriver.cpp | 42 +-
.../src/org/apache/mesos/MesosSchedulerDriver.java | 28 ---
2 files changed, 56 insertions(+), 14 deletions(-)
diff --git a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
index 4efde30..afdaa3b 100644
--- a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
+++ b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
@@ -21,6 +21,7 @@
#include
#include
+#include
#include
#include
@@ -1066,17 +1067,12 @@
Java_org_apache_mesos_MesosSchedulerDriver_reconcileTasks(
return convert(env, status);
}
-/* Class: org_apache_mesos_MesosSchedulerDriver
- * Method:updateFramework
- * Signature:
(Lorg/apache/mesos/Protos/FrameworkInfo;Ljava/util/Collection;Lorg/apache/mesos/scheduler/Protos/OfferConstraints;)Lorg/apache/mesos/Protos/Status;
- */
-JNIEXPORT jobject JNICALL
-Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
+static jobject updateFramework(
JNIEnv* env,
jobject thiz,
jobject jframeworkInfo,
jobject jsuppressedRoles,
-jobject jofferConstraints)
+Option jofferConstraints)
{
using ::mesos::scheduler::OfferConstraints;
@@ -1086,8 +1082,9 @@
Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
const vector suppressedRoles =
constructFromIterable(env, jsuppressedRoles);
- ::mesos::scheduler::OfferConstraints offerConstraints =
-construct(env, jofferConstraints);
+ OfferConstraints offerConstraints = jofferConstraints.isSome()
+ ? construct(env, *jofferConstraints)
+ : OfferConstraints();
jclass clazz = env->GetObjectClass(thiz);
@@ -1101,4 +1098,31 @@
Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
return convert(env, status);
}
+/* Class: org_apache_mesos_MesosSchedulerDriver
+ * Method:updateFramework
+ * Signature:
(Lorg/apache/mesos/Protos/FrameworkInfo;Ljava/util/Collection;)Lorg/apache/mesos/Protos/Status;
+ */
+JNIEXPORT jobject JNICALL
+Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
+JNIEnv* env, jobject thiz, jobject jframeworkInfo, jobject
jsuppressedRoles)
+{
+ return updateFramework(env, thiz, jframeworkInfo, jsuppressedRoles, None());
+}
+
+/* Class: org_apache_mesos_MesosSchedulerDriver
+ * Method:updateFrameworkWithConstraints
+ * Signature:
(Lorg/apache/mesos/Protos/FrameworkInfo;Ljava/util/Collection;Lorg/apache/mesos/scheduler/Protos/OfferConstraints;)Lorg/apache/mesos/Protos/Status;
+ */
+JNIEXPORT jobject JNICALL
+Java_org_apache_mesos_MesosSchedulerDriver_updateFrameworkWithConstraints(
+JNIEnv* env,
+jobject thiz,
+jobject jframeworkInfo,
+jobject jsuppressedRoles,
+jobject jofferConstraints)
+{
+ return updateFramework(
+ env, thiz, jframeworkInfo, jsuppressedRoles, jofferConstraints);
+}
+
} // extern "C" {
diff --git a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
index 4fdae33..9be0ec1 100644
--- a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
+++ b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
@@ -408,16 +408,34 @@ public class MesosSchedulerDriver implements
SchedulerDriver {
public native Status reconcileTasks(Collection statuses);
+ /**
+ * @deprecated Replaced by
+ * {@link #updateFramework(FrameworkInfo, Collection,
OfferConstraints)}
+ *
+ * NOTE: The underlying JNI method exists only to maintain compatibility
+ * of newer versions of libmesos-java.so with older versions of mesos.jar
+ */
+ @Deprecated
public native Status updateFramework(FrameworkInfo frameworkInfo,
- Collection suppressedRoles,
- OfferConstraints offerConstraints);
+ Collection suppressedRoles);
[mesos] branch master updated: Fixed compilation failure due to -Wmissing-braces.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 77d1c7e Fixed compilation failure due to -Wmissing-braces.
77d1c7e is described below
commit 77d1c7e1067272330d489868a493430439488e4b
Author: Andrei Sekretenko
AuthorDate: Fri Sep 25 21:52:24 2020 +0200
Fixed compilation failure due to -Wmissing-braces.
This fixes compilation failure on some versions of clang/gcc
introduced in 28ff20d4990ec2097852a5cd16966628d89c31d3 caused by
missing braces in the `OfferConstraintsFilter()` constructor code.
---
src/master/allocator/mesos/offer_constraints_filter.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/master/allocator/mesos/offer_constraints_filter.cpp
b/src/master/allocator/mesos/offer_constraints_filter.cpp
index 45199ca..05cb0f2 100644
--- a/src/master/allocator/mesos/offer_constraints_filter.cpp
+++ b/src/master/allocator/mesos/offer_constraints_filter.cpp
@@ -474,7 +474,7 @@ OfferConstraintsFilter::OfferConstraintsFilter(
OfferConstraintsFilter::OfferConstraintsFilter()
: OfferConstraintsFilter(
-CHECK_NOTERROR(OfferConstraintsFilterImpl::create({Bytes(0), 0}, {})))
+CHECK_NOTERROR(OfferConstraintsFilterImpl::create({{Bytes(0), 0}},
{})))
{}
[mesos] branch master updated (098315f -> 930c7e9)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 098315f Exposed hierarchical allocator recovery parameters as master flags. new 739a704 Added `OfferConstraints` validity criteria into protobuf comments. new 242f49e Added a test for creating a filter from default `OfferConstraints`. new 28ff20d Made offer constraints filter and protobuf non-optional inside the code. new fcfa327 Added offer constraints to `MesosSchedulerDriver::updateFramework()`. new 930c7e9 Added offer constraints to `updateFramework()` in the Java bindings. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: include/mesos/allocator/allocator.hpp | 9 +++- include/mesos/scheduler.hpp| 22 ++ include/mesos/scheduler/scheduler.proto| 9 include/mesos/v1/scheduler/scheduler.proto | 9 src/examples/java/TestFramework.java | 7 ++- src/java/jni/construct.cpp | 10 + .../jni/org_apache_mesos_MesosSchedulerDriver.cpp | 16 +-- .../src/org/apache/mesos/MesosSchedulerDriver.java | 10 - src/java/src/org/apache/mesos/SchedulerDriver.java | 37 +++- .../allocator/mesos/offer_constraints_filter.cpp | 6 +++ src/master/framework.cpp | 10 ++--- src/master/http.cpp| 4 +- src/master/master.cpp | 50 -- src/master/master.hpp | 18 src/master/readonly_handler.cpp| 6 +-- .../scheduler/mesos_scheduler_driver_impl.cpp | 4 +- src/sched/sched.cpp| 25 --- .../master/offer_constraints_filter_tests.cpp | 13 ++ src/tests/master/update_framework_tests.cpp| 18 src/tests/scheduler_driver_tests.cpp | 2 +- 20 files changed, 191 insertions(+), 94 deletions(-)
[mesos] 03/05: Made offer constraints filter and protobuf non-optional inside the code.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 28ff20d4990ec2097852a5cd16966628d89c31d3
Author: Andrei Sekretenko
AuthorDate: Thu Sep 24 16:12:46 2020 +0200
Made offer constraints filter and protobuf non-optional inside the code.
Given that Mesos now provides a guarantee that specifying no offer
constraint in UPDATE_FRAMEWORK/SUBSCRIBE call is equivalent to
specifying default-constructed `OfferConstraints`, and that we are
intending to make the V0 scheduler driver always require offer
constraints as an argument to the `updateFramework()`, it no longer
makes sense to keep `OfferConstraints`/`OfferConstraintsFilter`
optional inside the Mesos code base.
This patch replaces a non-set `Option` with
default-constructed `OfferConstraints`, and a non-set
`Option` with a default-constructed filter.
Review: https://reviews.apache.org/r/72897
---
include/mesos/allocator/allocator.hpp | 9 +++-
.../allocator/mesos/offer_constraints_filter.cpp | 6 +++
src/master/framework.cpp | 10 ++---
src/master/http.cpp| 4 +-
src/master/master.cpp | 50 --
src/master/master.hpp | 18
src/master/readonly_handler.cpp| 6 +--
7 files changed, 49 insertions(+), 54 deletions(-)
diff --git a/include/mesos/allocator/allocator.hpp
b/include/mesos/allocator/allocator.hpp
index b0a5d6a..c378f78 100644
--- a/include/mesos/allocator/allocator.hpp
+++ b/include/mesos/allocator/allocator.hpp
@@ -110,7 +110,12 @@ public:
const Options& options,
scheduler::OfferConstraints&& constraints);
- OfferConstraintsFilter() = delete;
+ /*
+ * Constructs a no-op filter that does not exclude any agents/resources from
+ * being offered. This is equivalent to passing default-constructed
+ * `OfferConstraints` to the factory method `create()`.
+ */
+ OfferConstraintsFilter();
// Definitions of these need `OfferConstraintsFilterImpl` to be a complete
// type.
@@ -149,7 +154,7 @@ struct FrameworkOptions
/**
* The internal representation of framework's offer constraints.
*/
- Option offerConstraintsFilter;
+ OfferConstraintsFilter offerConstraintsFilter;
};
diff --git a/src/master/allocator/mesos/offer_constraints_filter.cpp
b/src/master/allocator/mesos/offer_constraints_filter.cpp
index 441ebc1..45199ca 100644
--- a/src/master/allocator/mesos/offer_constraints_filter.cpp
+++ b/src/master/allocator/mesos/offer_constraints_filter.cpp
@@ -472,6 +472,12 @@ OfferConstraintsFilter::OfferConstraintsFilter(
{}
+OfferConstraintsFilter::OfferConstraintsFilter()
+ : OfferConstraintsFilter(
+CHECK_NOTERROR(OfferConstraintsFilterImpl::create({Bytes(0), 0}, {})))
+{}
+
+
OfferConstraintsFilter::OfferConstraintsFilter(OfferConstraintsFilter&&) =
default;
diff --git a/src/master/framework.cpp b/src/master/framework.cpp
index 980828e..a93172a 100644
--- a/src/master/framework.cpp
+++ b/src/master/framework.cpp
@@ -35,7 +35,7 @@ Framework::Framework(
Master* const master,
const Flags& masterFlags,
const FrameworkInfo& info,
-Option&& offerConstraints,
+OfferConstraints&& offerConstraints,
const process::UPID& pid,
const Owned& approvers,
const process::Time& time)
@@ -57,7 +57,7 @@ Framework::Framework(
Master* const master,
const Flags& masterFlags,
const FrameworkInfo& info,
-Option&& offerConstraints,
+OfferConstraints&& offerConstraints,
const StreamingHttpConnection& http,
const Owned& approvers,
const process::Time& time)
@@ -83,7 +83,7 @@ Framework::Framework(
master,
masterFlags,
info,
-None(),
+OfferConstraints{},
RECOVERED,
false,
nullptr,
@@ -95,7 +95,7 @@ Framework::Framework(
Master* const _master,
const Flags& masterFlags,
const FrameworkInfo& _info,
-Option&& offerConstraints,
+OfferConstraints&& offerConstraints,
State state,
bool active_,
const Owned& approvers,
@@ -538,7 +538,7 @@ void Framework::removeOperation(Operation* operation)
void Framework::update(
const FrameworkInfo& newInfo,
-Option&& offerConstraints)
+OfferConstraints&& offerConstraints)
{
// We only merge 'info' from the same framework 'id'.
CHECK_EQ(info.id(), newInfo.id());
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 3d8e709..c4595cc 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -1301,9 +1301,7 @@ mesos::master::Response::GetFrameworks
[mesos] 04/05: Added offer constraints to `MesosSchedulerDriver::updateFramework()`.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit fcfa3271c0c4fc2d8c6ae7cfd88cd82d26d2e1d1
Author: Andrei Sekretenko
AuthorDate: Wed Sep 16 12:25:19 2020 +0200
Added offer constraints to `MesosSchedulerDriver::updateFramework()`.
This patch adds an ability to set V0 framework's offer constraints
via the C++ V0 scheduler driver.
Review: https://reviews.apache.org/r/72874
---
include/mesos/scheduler.hpp| 22 ---
.../jni/org_apache_mesos_MesosSchedulerDriver.cpp | 3 ++-
.../scheduler/mesos_scheduler_driver_impl.cpp | 4 +++-
src/sched/sched.cpp| 25 --
src/tests/master/update_framework_tests.cpp| 18
src/tests/scheduler_driver_tests.cpp | 2 +-
6 files changed, 48 insertions(+), 26 deletions(-)
diff --git a/include/mesos/scheduler.hpp b/include/mesos/scheduler.hpp
index 61cc846..1401c34 100644
--- a/include/mesos/scheduler.hpp
+++ b/include/mesos/scheduler.hpp
@@ -23,6 +23,7 @@
#include
#include
+#include
// Mesos scheduler interface and scheduler driver. A scheduler is used
// to interact with Mesos in order to run distributed computations.
@@ -344,14 +345,16 @@ public:
virtual Status reconcileTasks(
const std::vector& statuses) = 0;
- // Inform Mesos master about changes to the `FrameworkInfo` and
- // the set of suppressed roles. The driver will store the new
- // `FrameworkInfo` and the new set of suppressed roles, and all
- // subsequent re-registrations will use them.
+ // Requests Mesos master to change the `FrameworkInfo`, the set of suppressed
+ // roles and the offer constraints. The driver will store the new
+ // `FrameworkInfo`, the new set of suppressed roles and the new offer
+ // constraints, and all subsequent re-registrations will use them.
//
// NOTE: If the supplied info is invalid or fails authorization,
- // the `error()` callback will be invoked asynchronously (after
- // the master replies with a `FrameworkErrorMessage`).
+ // or the supplied offer constraints are not valid, the `error()` callback
+ // will be invoked asynchronously (after the master replies with a
+ // `FrameworkErrorMessage`). Note that validity of non-empty (i.e.
+ // not default-constructed) offer constraints may depend on master flags.
//
// NOTE: This must be called after initial registration with the
// master completes and the `FrameworkID` is assigned. The assigned
@@ -362,7 +365,8 @@ public:
// during driver initialization.
virtual Status updateFramework(
const FrameworkInfo& frameworkInfo,
- const std::vector& suppressedRoles) = 0;
+ const std::vector& suppressedRoles,
+ ::mesos::scheduler::OfferConstraints&& offerConstraints) = 0;
};
@@ -524,7 +528,9 @@ public:
Status updateFramework(
const FrameworkInfo& frameworkInfo,
- const std::vector& suppressedRoles) override;
+ const std::vector& suppressedRoles,
+ ::mesos::scheduler::OfferConstraints&& offerConstraints)
+override;
protected:
// Used to detect (i.e., choose) the master.
diff --git a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
index 1817bba..4d71765 100644
--- a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
+++ b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
@@ -1086,7 +1086,8 @@
Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
MesosSchedulerDriver* driver =
(MesosSchedulerDriver*) env->GetLongField(thiz, __driver);
- Status status = driver->updateFramework(frameworkInfo, suppressedRoles);
+ Status status = driver->updateFramework(
+ frameworkInfo, suppressedRoles, ::mesos::scheduler::OfferConstraints());
return convert(env, status);
}
diff --git
a/src/python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp
b/src/python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp
index 256632a..17260cd 100644
--- a/src/python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp
+++ b/src/python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp
@@ -859,7 +859,9 @@ PyObject* MesosSchedulerDriverImpl_updateFramework(
return nullptr;
}
- Status status = self->driver->updateFramework(framework, *suppressedRoles);
+ Status status = self->driver->updateFramework(
+ framework, *suppressedRoles, ::mesos::scheduler::OfferConstraints{});
+
return PyInt_FromLong(status);
}
diff --git a/src/sched/sched.cpp b/src/sched/sched.cpp
index 768ce7d..119a062 100644
--- a/src/sched/sched.cpp
+++ b/src/sched/sched.cpp
@@ -105,6 +105,8 @@ using namespace mesos::scheduler;
using google::protobuf::RepeatedPtrF
[mesos] 01/05: Added `OfferConstraints` validity criteria into protobuf comments.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 739a704a11f6cfc68ab6e035c7e945d8f5774303
Author: Andrei Sekretenko
AuthorDate: Tue Sep 22 14:55:13 2020 +0200
Added `OfferConstraints` validity criteria into protobuf comments.
Review: https://reviews.apache.org/r/72898
---
include/mesos/scheduler/scheduler.proto| 9 +
include/mesos/v1/scheduler/scheduler.proto | 9 +
2 files changed, 18 insertions(+)
diff --git a/include/mesos/scheduler/scheduler.proto
b/include/mesos/scheduler/scheduler.proto
index f70738c..f70d379 100644
--- a/include/mesos/scheduler/scheduler.proto
+++ b/include/mesos/scheduler/scheduler.proto
@@ -443,6 +443,15 @@ message AttributeConstraint {
// NOTE: Even if a particular resource matches offer constraints of a
// framework's role, there is still no guarantee that it will ever be offered
// to this role of the framework.
+//
+// NOTE: Each `Group` must contain at least one constraint, and
+// `RoleConstraints` must contain at least one `Group`, otherwise
+// the message is not valid.
+//
+// NOTE: Frameworks are allowed to use default-constructed
+// `OfferConstraints` (instead of not setting the `offer_constraints` field)
+// in the SUBSCRIBE/UPDATE_FRAMEWORK calls to indicate subscription without
+// any offer constraints.
message OfferConstraints {
message RoleConstraints {
message Group {
diff --git a/include/mesos/v1/scheduler/scheduler.proto
b/include/mesos/v1/scheduler/scheduler.proto
index 364d2c8..a8596b6 100644
--- a/include/mesos/v1/scheduler/scheduler.proto
+++ b/include/mesos/v1/scheduler/scheduler.proto
@@ -441,6 +441,15 @@ message AttributeConstraint {
// NOTE: Even if a particular resource matches offer constraints of a
// framework's role, there is still no guarantee that it will ever be offered
// to this role of the framework.
+//
+// NOTE: Each `Group` must contain at least one constraint, and
+// `RoleConstraints` must contain at least one `Group`, otherwise
+// the message is not valid.
+//
+// NOTE: Frameworks are allowed to use default-constructed
+// `OfferConstraints` (instead of not setting the `offer_constraints` field)
+// in the SUBSCRIBE/UPDATE_FRAMEWORK calls to indicate subscription without
+// any offer constraints.
message OfferConstraints {
message RoleConstraints {
message Group {
[mesos] 02/05: Added a test for creating a filter from default `OfferConstraints`.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 242f49ee95e0bcac68d67e0d07f0ce5f3a133e87
Author: Andrei Sekretenko
AuthorDate: Tue Sep 22 14:55:43 2020 +0200
Added a test for creating a filter from default `OfferConstraints`.
Review: https://reviews.apache.org/r/72899
---
src/tests/master/offer_constraints_filter_tests.cpp | 13 +
1 file changed, 13 insertions(+)
diff --git a/src/tests/master/offer_constraints_filter_tests.cpp
b/src/tests/master/offer_constraints_filter_tests.cpp
index f80d56c..64b6f12 100644
--- a/src/tests/master/offer_constraints_filter_tests.cpp
+++ b/src/tests/master/offer_constraints_filter_tests.cpp
@@ -581,3 +581,16 @@ TEST(OfferConstraintsFilter, TwoRoles)
EXPECT_TRUE(
filter->isAgentExcluded("roleB", slaveInfoWithAttributes("foo:123")));
}
+
+
+// Tests that using default-constructed `OfferConstraints` to construct
+// a filter results in a no-op filter that does not exclude any agents
+// (and not, for example, in an error).
+TEST(OfferConstraintsFilter, DefaultOfferConstraints)
+{
+ const Try filter = createFilter(OfferConstraints{});
+
+ ASSERT_SOME(filter);
+
+ EXPECT_FALSE(filter->isAgentExcluded("role", SlaveInfo{}));
+}
[mesos] 05/05: Added offer constraints to `updateFramework()` in the Java bindings.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 930c7e98d17e71192dae1d49b4b2217cc2dbd8b2
Author: Andrei Sekretenko
AuthorDate: Wed Sep 16 12:28:24 2020 +0200
Added offer constraints to `updateFramework()` in the Java bindings.
Review: https://reviews.apache.org/r/72900
---
src/examples/java/TestFramework.java | 7 +++-
src/java/jni/construct.cpp | 10 ++
.../jni/org_apache_mesos_MesosSchedulerDriver.cpp | 15 +++--
.../src/org/apache/mesos/MesosSchedulerDriver.java | 10 +-
src/java/src/org/apache/mesos/SchedulerDriver.java | 37 --
5 files changed, 64 insertions(+), 15 deletions(-)
diff --git a/src/examples/java/TestFramework.java
b/src/examples/java/TestFramework.java
index 1dea79c..1a84ca3 100644
--- a/src/examples/java/TestFramework.java
+++ b/src/examples/java/TestFramework.java
@@ -28,6 +28,8 @@ import com.google.protobuf.ByteString;
import org.apache.mesos.*;
import org.apache.mesos.Protos.*;
+import org.apache.mesos.scheduler.Protos.OfferConstraints;
+import org.apache.mesos.scheduler.Protos.AttributeConstraint;
public class TestFramework {
static class TestScheduler implements Scheduler {
@@ -50,7 +52,10 @@ public class TestFramework {
// Clear suppressed roles.
FrameworkInfo.Builder builder = framework.toBuilder();
builder.setId(frameworkId);
- driver.updateFramework(builder.build(), new ArrayList());
+ driver.updateFramework(
+ builder.build(),
+ new ArrayList(),
+ OfferConstraints.getDefaultInstance());
}
@Override
diff --git a/src/java/jni/construct.cpp b/src/java/jni/construct.cpp
index a48ca07..d45ed6c 100644
--- a/src/java/jni/construct.cpp
+++ b/src/java/jni/construct.cpp
@@ -25,6 +25,8 @@
#include
+#include
+
#include
#include
@@ -147,6 +149,14 @@ FrameworkInfo construct(JNIEnv* env, jobject jobj)
return constructViaProtobufSerialization(env, jobj);
}
+template <>
+::mesos::scheduler::OfferConstraints construct(JNIEnv* env, jobject jobj)
+{
+ return constructViaProtobufSerialization<
+ ::mesos::scheduler::OfferConstraints>(env, jobj);
+}
+
+
template <>
Credential construct(JNIEnv* env, jobject jobj)
diff --git a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
index 4d71765..4efde30 100644
--- a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
+++ b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
@@ -1068,18 +1068,27 @@
Java_org_apache_mesos_MesosSchedulerDriver_reconcileTasks(
/* Class: org_apache_mesos_MesosSchedulerDriver
* Method:updateFramework
- * Signature:
(Lorg/apache/mesos/Protos/FrameworkInfo;Ljava/util/Collection;)Lorg/apache/mesos/Protos/Status;
+ * Signature:
(Lorg/apache/mesos/Protos/FrameworkInfo;Ljava/util/Collection;Lorg/apache/mesos/scheduler/Protos/OfferConstraints;)Lorg/apache/mesos/Protos/Status;
*/
JNIEXPORT jobject JNICALL
Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
-JNIEnv* env, jobject thiz, jobject jframeworkInfo, jobject
jsuppressedRoles)
+JNIEnv* env,
+jobject thiz,
+jobject jframeworkInfo,
+jobject jsuppressedRoles,
+jobject jofferConstraints)
{
+ using ::mesos::scheduler::OfferConstraints;
+
const FrameworkInfo& frameworkInfo =
construct(env, jframeworkInfo);
const vector suppressedRoles =
constructFromIterable(env, jsuppressedRoles);
+ ::mesos::scheduler::OfferConstraints offerConstraints =
+construct(env, jofferConstraints);
+
jclass clazz = env->GetObjectClass(thiz);
jfieldID __driver = env->GetFieldID(clazz, "__driver", "J");
@@ -1087,7 +1096,7 @@
Java_org_apache_mesos_MesosSchedulerDriver_updateFramework(
(MesosSchedulerDriver*) env->GetLongField(thiz, __driver);
Status status = driver->updateFramework(
- frameworkInfo, suppressedRoles, ::mesos::scheduler::OfferConstraints());
+ frameworkInfo, suppressedRoles, std::move(offerConstraints));
return convert(env, status);
}
diff --git a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
index 3e74be9..4fdae33 100644
--- a/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
+++ b/src/java/src/org/apache/mesos/MesosSchedulerDriver.java
@@ -19,6 +19,7 @@
package org.apache.mesos;
import org.apache.mesos.Protos.*;
+import org.apache.mesos.scheduler.Protos.OfferConstraints;
import java.util.ArrayList;
import java.util.Collection;
@@ -408,7 +409,14 @@ public class MesosSchedulerDriver implements
SchedulerDriver {
public native Status reconcileTasks(Collection statuses);
public native Status updateFramework(FrameworkInfo frameworkInfo,
-
[mesos] branch master updated: Exposed hierarchical allocator recovery parameters as master flags.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 098315f Exposed hierarchical allocator recovery parameters as master
flags.
098315f is described below
commit 098315f6b92ca076ef7cd197b039efcbcaf80e77
Author: Thomas Langé
AuthorDate: Wed Sep 9 15:59:34 2020 +0200
Exposed hierarchical allocator recovery parameters as master flags.
This closes #367
---
include/mesos/allocator/allocator.hpp | 8
src/master/allocator/mesos/hierarchical.cpp | 10 +++---
src/master/flags.cpp| 14 ++
src/master/flags.hpp| 2 ++
src/master/master.cpp | 2 ++
5 files changed, 29 insertions(+), 7 deletions(-)
diff --git a/include/mesos/allocator/allocator.hpp
b/include/mesos/allocator/allocator.hpp
index 6d67d5d..b0a5d6a 100644
--- a/include/mesos/allocator/allocator.hpp
+++ b/include/mesos/allocator/allocator.hpp
@@ -43,6 +43,10 @@
namespace mesos {
namespace allocator {
+constexpr Duration DEFAULT_ALLOCATOR_RECOVERY_TIMEOUT = Minutes(10);
+constexpr double DEFAULT_ALLOCATOR_AGENT_RECOVERY_FACTOR = 0.80;
+
+
/**
* Pass in configuration to the allocator.
*/
@@ -71,6 +75,10 @@ struct Options
// Mesos master's authorizer.
Option<::mesos::Authorizer*> authorizer;
+
+ // Recovery options
+ Duration recoveryTimeout = DEFAULT_ALLOCATOR_RECOVERY_TIMEOUT;
+ double agentRecoveryFactor = DEFAULT_ALLOCATOR_AGENT_RECOVERY_FACTOR;
};
diff --git a/src/master/allocator/mesos/hierarchical.cpp
b/src/master/allocator/mesos/hierarchical.cpp
index d4374c3..35264b9 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -702,13 +702,9 @@ void HierarchicalAllocatorProcess::recover(
updateQuota(role, quota);
}
- // TODO(alexr): Consider exposing these constants.
- const Duration ALLOCATION_HOLD_OFF_RECOVERY_TIMEOUT = Minutes(10);
- const double AGENT_RECOVERY_FACTOR = 0.8;
-
// Record the number of expected agents.
expectedAgentCount =
-static_cast(_expectedAgentCount * AGENT_RECOVERY_FACTOR);
+static_cast(_expectedAgentCount * options.agentRecoveryFactor);
// Skip recovery if there are no expected agents. This is not strictly
// necessary for the allocator to function correctly, but maps better
@@ -726,11 +722,11 @@ void HierarchicalAllocatorProcess::recover(
pause();
// Setup recovery timer.
- delay(ALLOCATION_HOLD_OFF_RECOVERY_TIMEOUT, self(), ::resume);
+ delay(options.recoveryTimeout, self(), ::resume);
LOG(INFO) << "Triggered allocator recovery: waiting for "
<< expectedAgentCount.get() << " agents to reconnect or "
-<< ALLOCATION_HOLD_OFF_RECOVERY_TIMEOUT << " to pass";
+<< options.recoveryTimeout << " to pass";
}
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index 31a8da1..be4f010 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -24,6 +24,8 @@
#include "master/constants.hpp"
#include "master/flags.hpp"
+#include
+
using std::string;
mesos::internal::master::Flags::Flags()
@@ -460,6 +462,18 @@ mesos::internal::master::Flags::Flags()
"load an alternate allocator module using `--modules`.",
DEFAULT_ALLOCATOR);
+ add(::allocator_agent_recovery_factor,
+ "allocator_agent_recovery_factor",
+ "Minimum fraction of known agents re-registered after leader election\n"
+ "for the allocator to start generating offers.",
+ mesos::allocator::DEFAULT_ALLOCATOR_AGENT_RECOVERY_FACTOR);
+
+ add(::allocator_recovery_timeout,
+ "allocator_recovery_timeout",
+ "Maximum time to wait before sending offers after a leader\n"
+ "re-election.",
+ mesos::allocator::DEFAULT_ALLOCATOR_RECOVERY_TIMEOUT);
+
add(::fair_sharing_excluded_resource_names,
"fair_sharing_excluded_resource_names",
"A comma-separated list of the resource names (e.g. 'gpus')\n"
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index 9500a0a..862ae59 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -81,6 +81,8 @@ public:
Option modulesDir;
std::string authenticators;
std::string allocator;
+ double allocator_agent_recovery_factor;
+ Duration allocator_recovery_timeout;
Option> fair_sharing_excluded_resource_names;
bool filter_gpu_resources;
std::string min_allocatable_resources;
diff --git a/src/master/master.cpp b/src/master/master.cpp
index fefa72d..576ae10 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -796,6 +796,8 @@ void Master::init
[mesos] branch master updated: Consolidated protobuf construction by the JNI scheduler driver.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new a2c5cb8 Consolidated protobuf construction by the JNI scheduler
driver.
a2c5cb8 is described below
commit a2c5cb8d571dc0b967b469263ce236f977e11f4c
Author: Andrei Sekretenko
AuthorDate: Tue Sep 15 18:20:44 2020 +0200
Consolidated protobuf construction by the JNI scheduler driver.
This patch converts the copy-pasted internals of numerous
`construct<>()` specializations for protobufs into a single
template.
Review: https://reviews.apache.org/r/72875
---
src/java/jni/construct.cpp | 276 ++---
1 file changed, 36 insertions(+), 240 deletions(-)
diff --git a/src/java/jni/construct.cpp b/src/java/jni/construct.cpp
index 8d8f76e..a48ca07 100644
--- a/src/java/jni/construct.cpp
+++ b/src/java/jni/construct.cpp
@@ -52,6 +52,26 @@ T parse(const void* data, int size)
}
+template
+TMessage constructViaProtobufSerialization(JNIEnv* env, jobject jobj)
+{
+ jclass clazz = env->GetObjectClass(jobj);
+
+ // byte[] data = obj.toByteArray();
+ jmethodID toByteArray = env->GetMethodID(clazz, "toByteArray", "()[B");
+
+ jbyteArray jdata = (jbyteArray) env->CallObjectMethod(jobj, toByteArray);
+
+ jbyte* data = env->GetByteArrayElements(jdata, nullptr);
+ jsize length = env->GetArrayLength(jdata);
+
+ TMessage result = parse(data, length);
+
+ env->ReleaseByteArrayElements(jdata, data, 0);
+ return result;
+}
+
+
bool construct(JNIEnv* env, jboolean jbool)
{
return jbool == JNI_TRUE;
@@ -124,168 +144,56 @@ map construct(JNIEnv *env, jobject jobj)
template <>
FrameworkInfo construct(JNIEnv* env, jobject jobj)
{
- jclass clazz = env->GetObjectClass(jobj);
-
- // byte[] data = obj.toByteArray();
- jmethodID toByteArray = env->GetMethodID(clazz, "toByteArray", "()[B");
-
- jbyteArray jdata = (jbyteArray) env->CallObjectMethod(jobj, toByteArray);
-
- jbyte* data = env->GetByteArrayElements(jdata, nullptr);
- jsize length = env->GetArrayLength(jdata);
-
- const FrameworkInfo& framework = parse(data, length);
-
- env->ReleaseByteArrayElements(jdata, data, 0);
-
- return framework;
+ return constructViaProtobufSerialization(env, jobj);
}
template <>
Credential construct(JNIEnv* env, jobject jobj)
{
- jclass clazz = env->GetObjectClass(jobj);
-
- // byte[] data = obj.toByteArray();
- jmethodID toByteArray = env->GetMethodID(clazz, "toByteArray", "()[B");
-
- jbyteArray jdata = (jbyteArray) env->CallObjectMethod(jobj, toByteArray);
-
- jbyte* data = env->GetByteArrayElements(jdata, nullptr);
- jsize length = env->GetArrayLength(jdata);
-
- const Credential& credential = parse(data, length);
-
- env->ReleaseByteArrayElements(jdata, data, 0);
-
- return credential;
+ return constructViaProtobufSerialization(env, jobj);
}
template <>
Filters construct(JNIEnv* env, jobject jobj)
{
- jclass clazz = env->GetObjectClass(jobj);
-
- // byte[] data = obj.toByteArray();
- jmethodID toByteArray = env->GetMethodID(clazz, "toByteArray", "()[B");
-
- jbyteArray jdata = (jbyteArray) env->CallObjectMethod(jobj, toByteArray);
-
- jbyte* data = env->GetByteArrayElements(jdata, nullptr);
- jsize length = env->GetArrayLength(jdata);
-
- const Filters& filters = parse(data, length);
-
- env->ReleaseByteArrayElements(jdata, data, 0);
-
- return filters;
+ return constructViaProtobufSerialization(env, jobj);
}
template <>
FrameworkID construct(JNIEnv* env, jobject jobj)
{
- jclass clazz = env->GetObjectClass(jobj);
-
- // byte[] data = obj.toByteArray();
- jmethodID toByteArray = env->GetMethodID(clazz, "toByteArray", "()[B");
-
- jbyteArray jdata = (jbyteArray) env->CallObjectMethod(jobj, toByteArray);
-
- jbyte* data = env->GetByteArrayElements(jdata, nullptr);
- jsize length = env->GetArrayLength(jdata);
-
- const FrameworkID& frameworkId = parse(data, length);
-
- env->ReleaseByteArrayElements(jdata, data, 0);
-
- return frameworkId;
+ return constructViaProtobufSerialization(env, jobj);
}
template <>
ExecutorID construct(JNIEnv* env, jobject jobj)
{
- jclass clazz = env->GetObjectClass(jobj);
-
- // byte[] data = obj.toByteArray();
- jmethodID toByteArray = env->GetMethodID(clazz, "toByteArray", "()[B");
-
- jbyteArray jdata = (jbyteArray) env->CallObjectMethod(jobj, toByteArray);
-
- jbyte* data = env->GetByteArrayElements(jdata, nullptr);
- jsize length = env->GetArrayLength(jdata);
-
- const ExecutorID& executorId = parse(data, length);
-
[mesos] branch master updated: Tied the 1st hierarchical allocator process to a process ID "allocator".
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 81f231c Tied the 1st hierarchical allocator process to a process ID
"allocator".
81f231c is described below
commit 81f231ccd10e960b1485482a859bc449914668ed
Author: Andrei Sekretenko
AuthorDate: Thu Sep 17 21:44:53 2020 +0200
Tied the 1st hierarchical allocator process to a process ID "allocator".
The instances of `HierarchicalAllocatorProcess` launched in the same
instance of libprocess after that will be assigned IDs like
"allocator(2)", "allocator(3)" and so on.
Previously, to query an HTTP endpoint exposed by the allocator,
the user had to figure out the allocator process id and use it to build
the URL (like "https://localhost:5050/hierarchical_allocator(1)/...");
now, the allocator debugging endpoints are accessible via fixed URLs
like "https://localhost:5050/allocator/offer_constraints_debug;.
Review: https://reviews.apache.org/r/72887
---
src/master/allocator/mesos/hierarchical.hpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/master/allocator/mesos/hierarchical.hpp
b/src/master/allocator/mesos/hierarchical.hpp
index 63444de..4ec15b6 100644
--- a/src/master/allocator/mesos/hierarchical.hpp
+++ b/src/master/allocator/mesos/hierarchical.hpp
@@ -35,6 +35,7 @@
#include
#include
#include
+#include
#include "common/protobuf_utils.hpp"
@@ -968,7 +969,8 @@ class HierarchicalAllocatorProcess
{
public:
HierarchicalAllocatorProcess()
-: ProcessBase(process::ID::generate("hierarchical-allocator")),
+: ProcessBase(strings::remove(
+ process::ID::generate("allocator"), "(1)", strings::Mode::SUFFIX)),
internal::HierarchicalAllocatorProcess(
[this]() -> Sorter* {
return new RoleSorter(this->self(), "allocator/mesos/roles/");
[mesos] branch master updated: Added an endpoint for debugging offer constraints.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 7aee5a8 Added an endpoint for debugging offer constraints.
7aee5a8 is described below
commit 7aee5a87d117601b0f551a68dac4ea25903e5f81
Author: Andrei Sekretenko
AuthorDate: Wed Sep 9 14:48:00 2020 +0200
Added an endpoint for debugging offer constraints.
Review: https://reviews.apache.org/r/72851
---
include/mesos/allocator/allocator.hpp | 9 +-
src/master/allocator/mesos/hierarchical.cpp | 125
src/master/allocator/mesos/hierarchical.hpp | 9 ++
src/master/master.cpp | 2 +
4 files changed, 143 insertions(+), 2 deletions(-)
diff --git a/include/mesos/allocator/allocator.hpp
b/include/mesos/allocator/allocator.hpp
index c6fca65..6d67d5d 100644
--- a/include/mesos/allocator/allocator.hpp
+++ b/include/mesos/allocator/allocator.hpp
@@ -24,10 +24,9 @@
#include
#include
+#include
#include
-
#include
-
#include
#include
@@ -66,6 +65,12 @@ struct Options
size_t maxCompletedFrameworks = 0;
bool publishPerFrameworkMetrics = true;
+
+ // Authentication realm for HTTP debug endpoints exposed by the allocator.
+ Option readonlyHttpAuthenticationRealm;
+
+ // Mesos master's authorizer.
+ Option<::mesos::Authorizer*> authorizer;
};
diff --git a/src/master/allocator/mesos/hierarchical.cpp
b/src/master/allocator/mesos/hierarchical.cpp
index 0b9588d..d4374c3 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -32,6 +32,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -42,6 +43,7 @@
#include
#include
+#include "common/authorization.hpp"
#include "common/http.hpp"
#include "common/protobuf_utils.hpp"
#include "common/resources_utils.hpp"
@@ -58,6 +60,7 @@ using mesos::allocator::InverseOfferStatus;
using mesos::allocator::Options;
using process::after;
+using process::http::authentication::Principal;
using process::Continue;
using process::ControlFlow;
using process::Failure;
@@ -612,6 +615,9 @@ Framework::Framework(
{}
+static string OFFER_CONSTRAINTS_DEBUG_HELP();
+
+
void HierarchicalAllocatorProcess::initialize(
const Options& _options,
const lambda::function<
@@ -633,6 +639,18 @@ void HierarchicalAllocatorProcess::initialize(
BoundedHashMap>(
options.maxCompletedFrameworks);
+ route("/offer_constraints_debug",
+options.readonlyHttpAuthenticationRealm.getOrElse(""),
+OFFER_CONSTRAINTS_DEBUG_HELP(),
+[this](const process::http::Request& request,
+ const Option& principal) {
+ logRequest(request);
+ return offerConstraintsDebug(request, principal)
+.onReady([request](const process::http::Response& response) {
+ logResponse(request, response);
+});
+});
+
roleSorter->initialize(options.fairnessExcludeResourceNames);
VLOG(1) << "Initialized hierarchical allocator process";
@@ -3212,6 +3230,113 @@ void
HierarchicalAllocatorProcess::untrackAllocatedResources(
}
+static string OFFER_CONSTRAINTS_DEBUG_HELP()
+{
+ return process::HELP(
+process::TLDR(
+"Evaluates current framework offer constraints and returns results."),
+process::DESCRIPTION(
+"This endpoint evaluates for each role of each framework a list",
+"of agents excluded from allocation by offer constraints.",
+"",
+"Example:",
+"```",
+"{\"frameworks\": {",
+" \"0f4c63a9-be1e-4a90-9e11-d7bf0aa6c8ad-0017\": {",
+" \"excluded_by_attribute_constraints\": {",
+" \"role1\": [",
+" \"0b1e7d60-dfbc-44c9-8222-48b57eca8637-S123\",",
+" \"654af69c-80f7-45ad-bcb3-c7c917f1811b-S045\"],",
+" \"role2\": [] }},",
+" \"b0377da6-090d-4338-9e2e-bf6cf0f309b7-0011\": {}",
+"}}",
+"```",
+"In this example, two agents are excluded from allocation",
+"to the first framework (-0017) under the role \"role1\", no agents",
+"are excluded from allocation to this framework under \"role2\",",
+"and the second framework (-0011) has no offer constraints set."
+),
+process::AUTHENTICATION(true),
+process::AUTHORIZATION(
+"This endpoint skips fram
[mesos] 01/02: Made the hierarchical allocator store `FrameworkInfo` internally.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 89576d8f4d322e2b2436ddd9ed882e71937ca61c Author: Andrei Sekretenko AuthorDate: Thu Sep 17 19:37:51 2020 +0200 Made the hierarchical allocator store `FrameworkInfo` internally. To make it possible to add into the allocator debugguing HTTP endpoints that will need to authorize principal's viewing access to frameworks, it becomes necessary to store `FrameworkInfo` in the allocator, so that it can compose an authorization object for the VIEW_FRAMEWORK action. Review: https://reviews.apache.org/r/72885 --- src/master/allocator/mesos/hierarchical.cpp | 2 ++ src/master/allocator/mesos/hierarchical.hpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp index 47f8a2b..d8ebc2c 100644 --- a/src/master/allocator/mesos/hierarchical.cpp +++ b/src/master/allocator/mesos/hierarchical.cpp @@ -600,6 +600,7 @@ Framework::Framework( bool _active, bool publishPerFrameworkMetrics) : frameworkId(frameworkInfo.id()), +info(frameworkInfo), roles(protobuf::framework::getRoles(frameworkInfo)), suppressedRoles(std::move(options.suppressedRoles)), capabilities(frameworkInfo.capabilities()), @@ -930,6 +931,7 @@ void HierarchicalAllocatorProcess::updateFramework( framework.suppressedRoles.erase(role); } + framework.info = frameworkInfo; framework.roles = newRoles; framework.capabilities = frameworkInfo.capabilities(); framework.minAllocatableResources = diff --git a/src/master/allocator/mesos/hierarchical.hpp b/src/master/allocator/mesos/hierarchical.hpp index 7e1980e..225de16 100644 --- a/src/master/allocator/mesos/hierarchical.hpp +++ b/src/master/allocator/mesos/hierarchical.hpp @@ -87,6 +87,8 @@ struct Framework const FrameworkID frameworkId; + FrameworkInfo info; + std::set roles; std::set suppressedRoles;
[mesos] 02/02: Enforced explicit construction of `protobuf::...::Capabilities`.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 095a409913506b0f12772c9a1ee380a67417b083
Author: Andrei Sekretenko
AuthorDate: Thu Sep 17 19:23:19 2020 +0200
Enforced explicit construction of `protobuf::...::Capabilities`.
This makes Mesos code more compliant with the Google C++ styleguide's
rule banning implicit conversions for non-interchangeable types,
for which the Mesos styleguide makes no exception.
In particular, not enabling a type conversion from an iterable into
`Capabilities` disallows puzzling code that looks like copying the same
protobuf twice. Now, an example like
```
framework.info = info;
framework.capabilities = info.capabilities();`
```
can no longer occur.
Review: https://reviews.apache.org/r/72886
---
src/common/protobuf_utils.hpp | 6 +++---
src/master/allocator/mesos/hierarchical.cpp | 4 +++-
src/master/master.cpp | 2 +-
src/slave/slave.cpp | 3 ++-
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/src/common/protobuf_utils.hpp b/src/common/protobuf_utils.hpp
index 0558249..e6ece8d 100644
--- a/src/common/protobuf_utils.hpp
+++ b/src/common/protobuf_utils.hpp
@@ -334,7 +334,7 @@ struct Capabilities
Capabilities() = default;
template
- Capabilities(const Iterable& capabilities)
+ explicit Capabilities(const Iterable& capabilities)
{
foreach (const SlaveInfo::Capability& capability, capabilities) {
switch (capability.type()) {
@@ -550,7 +550,7 @@ struct Capabilities
Capabilities() = default;
template
- Capabilities(const Iterable& capabilities)
+ explicit Capabilities(const Iterable& capabilities)
{
foreach (const MasterInfo::Capability& capability, capabilities) {
switch (capability.type()) {
@@ -633,7 +633,7 @@ struct Capabilities
Capabilities() = default;
template
- Capabilities(const Iterable& capabilities)
+ explicit Capabilities(const Iterable& capabilities)
{
foreach (const FrameworkInfo::Capability& capability, capabilities) {
switch (capability.type()) {
diff --git a/src/master/allocator/mesos/hierarchical.cpp
b/src/master/allocator/mesos/hierarchical.cpp
index d8ebc2c..0b9588d 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -933,7 +933,9 @@ void HierarchicalAllocatorProcess::updateFramework(
framework.info = frameworkInfo;
framework.roles = newRoles;
- framework.capabilities = frameworkInfo.capabilities();
+ framework.capabilities =
+protobuf::framework::Capabilities(frameworkInfo.capabilities());
+
framework.minAllocatableResources =
unpackFrameworkOfferFilters(frameworkInfo.offer_filters());
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 54b24b4..599d2c9 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -12989,7 +12989,7 @@ Try Slave::update(
}
version = _version;
- capabilities = _capabilities;
+ capabilities = protobuf::slave::Capabilities(_capabilities);
info = _info;
checkpointedResources = _checkpointedResources;
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index a69937b..c715951 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -4461,7 +4461,8 @@ void Slave::updateFramework(
// if from a master older than 1.3.
if (message.has_framework_info()) {
framework->info.CopyFrom(message.framework_info());
-framework->capabilities = message.framework_info().capabilities();
+framework->capabilities = protobuf::framework::Capabilities(
+message.framework_info().capabilities());
}
if (pid == UPID()) {
[mesos] branch master updated (b441034 -> 095a409)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from b441034 Made `UpdateFrameworkTest.OfferConstraints` build constraints from JSON. new 89576d8 Made the hierarchical allocator store `FrameworkInfo` internally. new 095a409 Enforced explicit construction of `protobuf::...::Capabilities`. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: src/common/protobuf_utils.hpp | 6 +++--- src/master/allocator/mesos/hierarchical.cpp | 6 +- src/master/allocator/mesos/hierarchical.hpp | 2 ++ src/master/master.cpp | 2 +- src/slave/slave.cpp | 3 ++- 5 files changed, 13 insertions(+), 6 deletions(-)
[mesos] branch master updated: Made `UpdateFrameworkTest.OfferConstraints` build constraints from JSON.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new b441034 Made `UpdateFrameworkTest.OfferConstraints` build constraints
from JSON.
b441034 is described below
commit b441034156bec33d096e0ec45ece3cc2dc8fa3e0
Author: Andrei Sekretenko
AuthorDate: Fri Sep 11 19:08:00 2020 +0200
Made `UpdateFrameworkTest.OfferConstraints` build constraints from JSON.
Review: https://reviews.apache.org/r/72864
---
src/tests/master/update_framework_tests.cpp | 80 +++--
1 file changed, 53 insertions(+), 27 deletions(-)
diff --git a/src/tests/master/update_framework_tests.cpp
b/src/tests/master/update_framework_tests.cpp
index 7c83bac..d6c45f6 100644
--- a/src/tests/master/update_framework_tests.cpp
+++ b/src/tests/master/update_framework_tests.cpp
@@ -810,6 +810,28 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
using ::mesos::v1::scheduler::AttributeConstraint;
using ::mesos::v1::scheduler::OfferConstraints;
+ const Try initialConstraintsJson =
+JSON::parse(R"~(
+{
+ "role_constraints": {
+")~" + DEFAULT_FRAMEWORK_INFO.roles(0) + R"~(": {
+ "groups": [{
+"attribute_constraints": [{
+ "selector": {"attribute_name": "foo"},
+ "predicate": {"exists": {}}
+}]
+ }]
+}
+ }
+})~");
+
+ ASSERT_SOME(initialConstraintsJson);
+
+ const Try initialConstraints =
+::protobuf::parse(*initialConstraintsJson);
+
+ ASSERT_SOME(initialConstraints);
+
mesos::internal::master::Flags masterFlags = CreateMasterFlags();
Try> master = StartMaster(masterFlags);
ASSERT_SOME(master);
@@ -823,22 +845,14 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
auto scheduler = std::make_shared();
EXPECT_CALL(*scheduler, connected(_))
-.WillOnce(Invoke([](Mesos* mesos) {
+.WillOnce(Invoke([initialConstraints](Mesos* mesos) {
Call call;
call.set_type(Call::SUBSCRIBE);
*call.mutable_subscribe()->mutable_framework_info() =
DEFAULT_FRAMEWORK_INFO;
- AttributeConstraint* constraint =
-(*call.mutable_subscribe()
-->mutable_offer_constraints()
-->mutable_role_constraints())[DEFAULT_FRAMEWORK_INFO.roles(0)]
- .add_groups()
- ->add_attribute_constraints();
-
- *constraint->mutable_selector()->mutable_attribute_name() = "foo";
- *constraint->mutable_predicate()->mutable_exists() =
-AttributeConstraint::Predicate::Exists();
+ *call.mutable_subscribe()->mutable_offer_constraints() =
+*initialConstraints;
mesos->send(call);
}));
@@ -870,22 +884,34 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
// Change constraint to `NotExists` so that the agent will now be offered to
// the framework.
- OfferConstraints updatedConstraints;
+ const Try updatedConstraintsJson =
+JSON::parse(R"~(
+{
+ "role_constraints": {
+")~" + DEFAULT_FRAMEWORK_INFO.roles(0) + R"~(": {
+ "groups": [{
+"attribute_constraints": [{
+ "selector": {"attribute_name": "foo"},
+ "predicate": {"not_exists": {}}
+}]
+ }]
+}
+ }
+})~");
+
+ ASSERT_SOME(updatedConstraintsJson);
+
+ const Try updatedConstraints =
+::protobuf::parse(*updatedConstraintsJson);
+
+ ASSERT_SOME(updatedConstraints);
{
FrameworkInfo framework = DEFAULT_FRAMEWORK_INFO;
*framework.mutable_id() = subscribed->framework_id();
-AttributeConstraint* constraint =
- (*updatedConstraints.mutable_role_constraints())[framework.roles(0)]
-.add_groups()
-->add_attribute_constraints();
-
-*constraint->mutable_selector()->mutable_attribute_name() = "foo";
-*constraint->mutable_predicate()->mutable_not_exists() =
- AttributeConstraint::Predicate::NotExists();
-
-AWAIT_READY(callUpdateFramework(, framework, {},
updatedConstraints));
+AWAIT_READY(
+callUpdateFramework(, framework, {}, *updatedConstraints));
}
Clock::pause();
@@ -911,10 +937,10 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
Try parse = JSON::parse(response->body);
ASSERT_SOME(parse);
-Result reportedConstraints = parse->find(
+Result reportedConstraintsJson = parse->find(
"frameworks[0].offer_constraints");
-EXPECT_SOME_EQ(JSON::protobuf(
[mesos] 02/02: Added MESOS-10134 to the 1.7.4 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 5a8842259f9ad2506d718a4667d756421ef1c55e Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:48:16 2020 +0200 Added MESOS-10134 to the 1.7.4 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 893e630..4096e9a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,7 @@ Release Notes - Mesos - Version 1.7.4 (WIP) ** Bug * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state.
[mesos] branch 1.7.x updated (8fc2878 -> 5a88422)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 8fc2878 Added a test for discarding image pull on discard of getting image. new 87a03ff Added MESOS-10169 to the 1.7.4 CHANGELOG. new 5a88422 Added MESOS-10134 to the 1.7.4 CHANGELOG. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 2 ++ 1 file changed, 2 insertions(+)
[mesos] 01/02: Added MESOS-10169 to the 1.7.4 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 87a03ffee7af6f0cb2d5755414d7d7e1bdec2378 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:32:06 2020 +0200 Added MESOS-10169 to the 1.7.4 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 395305d..893e630 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,7 @@ Release Notes - Mesos - Version 1.7.4 (WIP) ** Bug * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. Release Notes - Mesos - Version 1.7.3
[mesos] 02/02: Added MESOS-10134 to the 1.8.2 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.8.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 59797580e87572ae0b05ed3c2806aaae87d37270 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:47:23 2020 +0200 Added MESOS-10134 to the 1.8.2 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 4f74a48..771bb26 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,7 @@ Release Notes - Mesos - Version 1.8.2 (WIP) * [MESOS-10007] - Command executor can miss exit status for short-lived commands due to double-reaping. * [MESOS-10015] - updateAllocation() can stall the allocator with a huge number of reservations on an agent. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement
[mesos] 01/02: Added MESOS-10169 to the 1.8.2 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.8.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 12d47b0226b0a553f7ccfaf118264bcb839e363f Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:31:43 2020 +0200 Added MESOS-10169 to the 1.8.2 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 8896297..4f74a48 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,7 @@ Release Notes - Mesos - Version 1.8.2 (WIP) * [MESOS-10007] - Command executor can miss exit status for short-lived commands due to double-reaping. * [MESOS-10015] - updateAllocation() can stall the allocator with a huge number of reservations on an agent. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement * [MESOS-9889] - Master CPU high due to unexpected foreachkey behaviour in Master::__reregisterSlave.
[mesos] branch 1.8.x updated (644b7c3 -> 5979758)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.8.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 644b7c3 Fixed a compilation issue on GCC 5.5 and older. new 12d47b0 Added MESOS-10169 to the 1.8.2 CHANGELOG. new 5979758 Added MESOS-10134 to the 1.8.2 CHANGELOG. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 2 ++ 1 file changed, 2 insertions(+)
[mesos] 02/02: Added MESOS-10134 to the 1.9.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.9.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit abb00b65007aa17ed1b8af73650b9ecfd545cbf6 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:46:46 2020 +0200 Added MESOS-10134 to the 1.9.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index c9048d9..ea638db 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -17,6 +17,7 @@ Release Notes - Mesos - Version 1.9.1 (WIP) * [MESOS-10096] - Reactivating a draining agent leaves the agent in draining state. * [MESOS-10118] - Agent incorrectly handles draining when empty. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement
[mesos] 01/02: Added MESOS-10169 to the 1.9.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.9.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 262b3c205e0ce7d9b5fd70c1db8b9829d25d7d9f Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:30:30 2020 +0200 Added MESOS-10169 to the 1.9.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 0986f35..c9048d9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -17,6 +17,7 @@ Release Notes - Mesos - Version 1.9.1 (WIP) * [MESOS-10096] - Reactivating a draining agent leaves the agent in draining state. * [MESOS-10118] - Agent incorrectly handles draining when empty. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement * [MESOS-9889] - Master CPU high due to unexpected foreachkey behaviour in Master::__reregisterSlave.
[mesos] branch 1.9.x updated (911b197 -> abb00b6)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.9.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 911b197 Fixed a compilation issue on GCC 5.5 and older. new 262b3c2 Added MESOS-10169 to the 1.9.1 CHANGELOG. new abb00b6 Added MESOS-10134 to the 1.9.1 CHANGELOG. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 2 ++ 1 file changed, 2 insertions(+)
[mesos] branch 1.10.x updated (6db2521 -> e79a434)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.10.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 6db2521 Fixed a compilation issue on GCC 5.5 and older. new 1915ea4 Added MESOS-10169 to the 1.10.1 CHANGELOG. new e79a434 Added MESOS-10134 to the 1.10.1 CHANGELOG. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 2 ++ 1 file changed, 2 insertions(+)
[mesos] 02/02: Added MESOS-10134 to the 1.10.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.10.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit e79a434033f4d56944ef94490a2908f616bb66fd Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:46:17 2020 +0200 Added MESOS-10134 to the 1.10.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 844c112..0473881 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ Release Notes - Mesos - Version 1.10.1 (WIP) ** Bug * [MESOS-9609] - Master check failure when marking agent unreachable. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state.
[mesos] 01/02: Added MESOS-10169 to the 1.10.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch 1.10.x in repository https://gitbox.apache.org/repos/asf/mesos.git commit 1915ea44e2fa217ed4d2c52a91c3226b84b6d686 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:29:50 2020 +0200 Added MESOS-10169 to the 1.10.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index e39e79d..844c112 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ Release Notes - Mesos - Version 1.10.1 (WIP) ** Bug * [MESOS-9609] - Master check failure when marking agent unreachable. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. Release Notes - Mesos - Version 1.10.0
[mesos] branch master updated (049a0c7 -> e00e7a1)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 049a0c7 Fixed a compilation issue on GCC 5.5 and older. new aa14054 Fixed a stale comment about delayed permissions when batching requests. new c8f8fbd Added `OfferConstraints` to `struct Framework` in the master. new ae35c0c Exposed offer constraints via the `/state` and `/frameworks` endpoints. new 8369e77 Added offer constraints to the `GetFrameworks` protobuf. new e00e7a1 Exposed framework offer constraints via the GET_FRAMEWORKS call. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: include/mesos/master/master.proto | 6 +++ include/mesos/v1/master/master.proto| 6 +++ src/master/framework.cpp| 41 ++-- src/master/http.cpp | 19 ++-- src/master/master.cpp | 74 ++--- src/master/master.hpp | 18 ++- src/master/readonly_handler.cpp | 5 ++ src/tests/master/update_framework_tests.cpp | 65 +++-- 8 files changed, 201 insertions(+), 33 deletions(-)
[mesos] 05/05: Exposed framework offer constraints via the GET_FRAMEWORKS call.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit e00e7a1b578de1161fbb181124cad2a20df5bb95
Author: Andrei Sekretenko
AuthorDate: Mon Sep 7 21:43:27 2020 +0200
Exposed framework offer constraints via the GET_FRAMEWORKS call.
Review: https://reviews.apache.org/r/72842
---
src/master/http.cpp | 4
src/tests/master/update_framework_tests.cpp | 16 ++--
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/src/master/http.cpp b/src/master/http.cpp
index aede6ee..3d8e709 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -1301,6 +1301,10 @@ mesos::master::Response::GetFrameworks::Framework model(
_framework.mutable_offered_resources()->Add()->CopyFrom(resource);
}
+ if (framework.offerConstraints().isSome()) {
+*_framework.mutable_offer_constraints() = *framework.offerConstraints();
+ }
+
return _framework;
}
diff --git a/src/tests/master/update_framework_tests.cpp
b/src/tests/master/update_framework_tests.cpp
index aab94e3..7c83bac 100644
--- a/src/tests/master/update_framework_tests.cpp
+++ b/src/tests/master/update_framework_tests.cpp
@@ -938,8 +938,20 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
ASSERT_SOME_EQ(JSON::protobuf(updatedConstraints), reportedConstraints);
}
- // TODO(asekretenko): After master starts exposing offer constraints via the
- // V1 API (MESOS-10179), check the constraints in the GET_FRAMEWORKS
response.
+ Future frameworks =
+getFrameworks(master->get()->pid);
+ AWAIT_READY(frameworks);
+
+ ASSERT_EQ(frameworks->frameworks_size(), 1);
+ ASSERT_TRUE(frameworks->frameworks(0).has_offer_constraints());
+
+ // TODO(asekretenko): As the semantics of the offer constraints does not
+ // depend on the order of constraints groups and the order of individual
+ // constraints inside groups, we should consider using a `MessageDifferencer`
+ // configured to take this into account.
+ ASSERT_EQ(
+ updatedConstraints.SerializeAsString(),
+ frameworks->frameworks(0).offer_constraints().SerializeAsString());
}
[mesos] 02/05: Added `OfferConstraints` to `struct Framework` in the master.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c8f8fbd328235fcd7aa1b32e079fb6f9a424b881
Author: Andrei Sekretenko
AuthorDate: Fri Sep 4 20:23:25 2020 +0200
Added `OfferConstraints` to `struct Framework` in the master.
This is a prerequisite for exposing framework's offer constraints
via master API endpoints.
Review: https://reviews.apache.org/r/72839
---
src/master/framework.cpp | 41 +++
src/master/master.cpp| 74 +---
src/master/master.hpp| 18 +++-
3 files changed, 110 insertions(+), 23 deletions(-)
diff --git a/src/master/framework.cpp b/src/master/framework.cpp
index e9e2d20..980828e 100644
--- a/src/master/framework.cpp
+++ b/src/master/framework.cpp
@@ -25,6 +25,7 @@ using process::Owned;
using process::http::authentication::Principal;
using mesos::authorization::ActionObject;
+using mesos::scheduler::OfferConstraints;
namespace mesos {
namespace internal {
@@ -34,10 +35,19 @@ Framework::Framework(
Master* const master,
const Flags& masterFlags,
const FrameworkInfo& info,
+Option&& offerConstraints,
const process::UPID& pid,
const Owned& approvers,
const process::Time& time)
- : Framework(master, masterFlags, info, CONNECTED, true, approvers, time)
+ : Framework(
+master,
+masterFlags,
+info,
+std::move(offerConstraints),
+CONNECTED,
+true,
+approvers,
+time)
{
pid_ = pid;
}
@@ -47,10 +57,19 @@ Framework::Framework(
Master* const master,
const Flags& masterFlags,
const FrameworkInfo& info,
+Option&& offerConstraints,
const StreamingHttpConnection& http,
const Owned& approvers,
const process::Time& time)
- : Framework(master, masterFlags, info, CONNECTED, true, approvers, time)
+ : Framework(
+master,
+masterFlags,
+info,
+std::move(offerConstraints),
+CONNECTED,
+true,
+approvers,
+time)
{
http_ = http;
}
@@ -61,7 +80,14 @@ Framework::Framework(
const Flags& masterFlags,
const FrameworkInfo& info)
: Framework(
- master, masterFlags, info, RECOVERED, false, nullptr, process::Time())
+master,
+masterFlags,
+info,
+None(),
+RECOVERED,
+false,
+nullptr,
+process::Time())
{}
@@ -69,6 +95,7 @@ Framework::Framework(
Master* const _master,
const Flags& masterFlags,
const FrameworkInfo& _info,
+Option&& offerConstraints,
State state,
bool active_,
const Owned& approvers,
@@ -84,7 +111,8 @@ Framework::Framework(
metrics(_info, masterFlags.publish_per_framework_metrics),
active_(active_),
state(state),
-objectApprovers(approvers)
+objectApprovers(approvers),
+offerConstraints_(std::move(offerConstraints))
{
CHECK(_info.has_id());
@@ -508,7 +536,9 @@ void Framework::removeOperation(Operation* operation)
}
-void Framework::update(const FrameworkInfo& newInfo)
+void Framework::update(
+const FrameworkInfo& newInfo,
+Option&& offerConstraints)
{
// We only merge 'info' from the same framework 'id'.
CHECK_EQ(info.id(), newInfo.id());
@@ -523,6 +553,7 @@ void Framework::update(const FrameworkInfo& newInfo)
CHECK_EQ(info.checkpoint(), newInfo.checkpoint());
info.CopyFrom(newInfo);
+ offerConstraints_ = std::move(offerConstraints);
// Save the old list of roles for later.
std::set oldRoles = roles;
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 438ef98..54b24b4 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -155,6 +155,8 @@ using mesos::master::contender::MasterContender;
using mesos::master::detector::MasterDetector;
+using mesos::scheduler::OfferConstraints;
+
static bool isValidFailoverTimeout(const FrameworkInfo& frameworkInfo);
@@ -2669,13 +2671,17 @@ void Master::subscribe(
Option validationError =
validateFramework(frameworkInfo, subscribe.suppressed_roles());
+ Option offerConstraints;
+ if (subscribe.has_offer_constraints()) {
+offerConstraints = std::move(*subscribe.mutable_offer_constraints());
+ }
+
allocator::FrameworkOptions allocatorOptions;
// TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
- if (validationError.isNone() && subscribe.has_offer_constraints()) {
+ if (validationError.isNone() && offerConstraints.isSome()) {
Try filter = OfferConstraintsFilter::create(
-offerConstraintsFilterOptions,
-std::move(*subscribe.mutable_offer_constraints()));
+offerConstraintsFilterOptions, OfferConstraints(*offe
[mesos] 04/05: Added offer constraints to the `GetFrameworks` protobuf.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8369e77b23de9453156626344a3ea0e36b4dd529
Author: Andrei Sekretenko
AuthorDate: Mon Sep 7 21:42:32 2020 +0200
Added offer constraints to the `GetFrameworks` protobuf.
Review: https://reviews.apache.org/r/72841
---
include/mesos/master/master.proto| 6 ++
include/mesos/v1/master/master.proto | 6 ++
2 files changed, 12 insertions(+)
diff --git a/include/mesos/master/master.proto
b/include/mesos/master/master.proto
index 021dadc..d51350b 100644
--- a/include/mesos/master/master.proto
+++ b/include/mesos/master/master.proto
@@ -22,6 +22,8 @@ import "mesos/maintenance/maintenance.proto";
import "mesos/quota/quota.proto";
+import "mesos/scheduler/scheduler.proto";
+
package mesos.master;
option cc_enable_arenas = true;
@@ -488,11 +490,15 @@ message Response {
optional TimeInfo reregistered_time = 5;
optional TimeInfo unregistered_time = 6;
+ // NOTE: Offers, inverse offers, allocated/offered resources
+ // and offer constraints are never reported via master API events.
repeated Offer offers = 7;
repeated InverseOffer inverse_offers = 8;
repeated Resource allocated_resources = 9;
repeated Resource offered_resources = 10;
+
+ optional scheduler.OfferConstraints offer_constraints = 12;
}
// Frameworks that have subscribed with the master. Note that this includes
diff --git a/include/mesos/v1/master/master.proto
b/include/mesos/v1/master/master.proto
index 488fe29..6735228 100644
--- a/include/mesos/v1/master/master.proto
+++ b/include/mesos/v1/master/master.proto
@@ -23,6 +23,8 @@ import "mesos/v1/mesos.proto";
import "mesos/v1/maintenance/maintenance.proto";
+import "mesos/v1/scheduler/scheduler.proto";
+
import "mesos/v1/quota/quota.proto";
package mesos.v1.master;
@@ -489,11 +491,15 @@ message Response {
optional TimeInfo reregistered_time = 5;
optional TimeInfo unregistered_time = 6;
+ // NOTE: Offers, inverse offers, allocated/offered resources
+ // and offer constraints are never reported via master API events.
repeated Offer offers = 7;
repeated InverseOffer inverse_offers = 8;
repeated Resource allocated_resources = 9;
repeated Resource offered_resources = 10;
+
+ optional scheduler.OfferConstraints offer_constraints = 12;
}
// Frameworks that have subscribed with the master. Note that this includes
[mesos] 03/05: Exposed offer constraints via the `/state` and `/frameworks` endpoints.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit ae35c0ca003adb41298d256678788e41cdea0523
Author: Andrei Sekretenko
AuthorDate: Fri Sep 4 21:31:41 2020 +0200
Exposed offer constraints via the `/state` and `/frameworks` endpoints.
Review: https://reviews.apache.org/r/72840
---
src/master/readonly_handler.cpp | 5 +++
src/tests/master/update_framework_tests.cpp | 53 ++---
2 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/src/master/readonly_handler.cpp b/src/master/readonly_handler.cpp
index b1336f9..df499d1 100644
--- a/src/master/readonly_handler.cpp
+++ b/src/master/readonly_handler.cpp
@@ -257,6 +257,11 @@ void FullFrameworkWriter::operator()(JSON::ObjectWriter*
writer) const
if (framework_->info.has_labels()) {
writer->field("labels", framework_->info.labels());
}
+
+ if (framework_->offerConstraints().isSome()) {
+writer->field(
+"offer_constraints", JSON::Protobuf(*framework_->offerConstraints()));
+ };
}
diff --git a/src/tests/master/update_framework_tests.cpp
b/src/tests/master/update_framework_tests.cpp
index 87bc3c7..aab94e3 100644
--- a/src/tests/master/update_framework_tests.cpp
+++ b/src/tests/master/update_framework_tests.cpp
@@ -870,13 +870,14 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
// Change constraint to `NotExists` so that the agent will now be offered to
// the framework.
+ OfferConstraints updatedConstraints;
+
{
FrameworkInfo framework = DEFAULT_FRAMEWORK_INFO;
*framework.mutable_id() = subscribed->framework_id();
-OfferConstraints constraints;
AttributeConstraint* constraint =
- (*constraints.mutable_role_constraints())[framework.roles(0)]
+ (*updatedConstraints.mutable_role_constraints())[framework.roles(0)]
.add_groups()
->add_attribute_constraints();
@@ -884,7 +885,7 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
*constraint->mutable_predicate()->mutable_not_exists() =
AttributeConstraint::Predicate::NotExists();
-AWAIT_READY(callUpdateFramework(, framework, {}, constraints));
+AWAIT_READY(callUpdateFramework(, framework, {},
updatedConstraints));
}
Clock::pause();
@@ -895,8 +896,50 @@ TEST_F(UpdateFrameworkTest, OfferConstraints)
AWAIT_READY(offers);
EXPECT_EQ(offers->offers().size(), 1);
- // TODO(asekretenko): After master starts exposing offer constraints via
- // its endpoints (MESOS-10179), check the constraints in the endpoints.
+ // Ensure that the updated offer constraints are reflected in the master's
+ // '/state' response.
+ {
+Future response = process::http::get(
+master->get()->pid,
+"state",
+None(),
+createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+AWAIT_ASSERT_RESPONSE_STATUS_EQ(process::http::OK().status, response);
+AWAIT_ASSERT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type",
response);
+
+Try parse = JSON::parse(response->body);
+ASSERT_SOME(parse);
+
+Result reportedConstraints = parse->find(
+"frameworks[0].offer_constraints");
+
+EXPECT_SOME_EQ(JSON::protobuf(updatedConstraints), reportedConstraints);
+ }
+
+ // Ensure that the updated offer constraints are reflected in the master's
+ // '/frameworks' response.
+ {
+Future response = process::http::get(
+master->get()->pid,
+"frameworks",
+None(),
+createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+AWAIT_ASSERT_RESPONSE_STATUS_EQ(process::http::OK().status, response);
+AWAIT_ASSERT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type",
response);
+
+Try parse = JSON::parse(response->body);
+ASSERT_SOME(parse);
+
+Result reportedConstraints = parse->find(
+"frameworks[0].offer_constraints");
+
+ASSERT_SOME_EQ(JSON::protobuf(updatedConstraints), reportedConstraints);
+ }
+
+ // TODO(asekretenko): After master starts exposing offer constraints via the
+ // V1 API (MESOS-10179), check the constraints in the GET_FRAMEWORKS
response.
}
[mesos] 01/05: Fixed a stale comment about delayed permissions when batching requests.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit aa14054722c38ca9e7e24e77133ad82d70c4ddc2
Author: Andrei Sekretenko
AuthorDate: Tue Sep 8 16:50:22 2020 +0200
Fixed a stale comment about delayed permissions when batching requests.
Now that synchronous authorization requires the authorizer to keep
object approvers up-to-date by the authorizer, batched processing
of readonly requests in the Master no longer intorduces an additional
delay into propagation of permission changes.
Review: https://reviews.apache.org/r/72843
---
src/master/http.cpp | 15 ++-
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/src/master/http.cpp b/src/master/http.cpp
index f34ea54..aede6ee 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -2335,6 +2335,16 @@ Future Master::Http::deferBatchedRequest(
// specific to the batched requests which are always members of
// `ReadOnlyHandler`, since we rely on the response only depending
// on query parameters and the current master state.
+//
+// TODO(asekretenko): We should consider replacing `approvers` with
+// taking the set of authorization actions and creating
+// `ObjectApprovers` inside of this method, because the code below
+// implicilty relies on the fact that `approvers` for a given handler
+// always contain `ObjectApprover`s for the same set of actions. Note
+// that together with the fact that permissions in all
`ObjectApprover`s
+// for a given principal are equally up-to-date (regardless of when
they
+// were created), this allows us to compare `handler` and `principal`
+// instead of comparing `approvers`.
return handler == batchedRequest.handler &&
principal == batchedRequest.principal &&
outputContentType == batchedRequest.outputContentType &&
@@ -2360,11 +2370,6 @@ Future Master::Http::deferBatchedRequest(
std::move(promise)});
} else {
// Return the existing future if we have a matching request.
-// NOTE: This is effectively adding a layer of authorization permissions
-// caching since we only checked the equality of principals, not the
-// equality of the approvers themselves.
-// On heavily-loaded masters, this could lead to a delay of several seconds
-// before permission changes for a principal take effect.
future = it->promise.future();
++master->metrics->http_cache_hits;
[mesos] branch master updated (21a0399 -> 5d68474)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 21a0399 Added MESOS-10169 to the 1.7.4 CHANGELOG. new 2aae475 Added MESOS-10134 to the 1.10.1 CHANGELOG. new 3f1a189 Added MESOS-10134 to the 1.9.1 CHANGELOG. new 8964fba Added MESOS-10134 to the 1.8.2 CHANGELOG. new 5d68474 Added MESOS-10134 to the 1.7.4 CHANGELOG. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 4 1 file changed, 4 insertions(+)
[mesos] 04/04: Added MESOS-10134 to the 1.7.4 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 5d684743682ee7bb28dd66dddb1128b8e2b387ac Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:48:16 2020 +0200 Added MESOS-10134 to the 1.7.4 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 5aeccb5..951d1af 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -951,6 +951,7 @@ Release Notes - Mesos - Version 1.7.4 (WIP) ** Bug * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state.
[mesos] 01/04: Added MESOS-10134 to the 1.10.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 2aae47501571608a40df8ffd9131341a69f24a16 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:46:17 2020 +0200 Added MESOS-10134 to the 1.10.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 54ffc69..0c0f456 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,7 @@ Release Notes - Mesos - Version 1.10.1 (WIP) ** Bug * [MESOS-9609] - Master check failure when marking agent unreachable. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state.
[mesos] 02/04: Added MESOS-10134 to the 1.9.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 3f1a18980b0d937d089c797af3dd3d461ac3be46 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:46:46 2020 +0200 Added MESOS-10134 to the 1.9.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 0c0f456..2e371c8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -236,6 +236,7 @@ Release Notes - Mesos - Version 1.9.1 (WIP) * [MESOS-10096] - Reactivating a draining agent leaves the agent in draining state. * [MESOS-10118] - Agent incorrectly handles draining when empty. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement
[mesos] 03/04: Added MESOS-10134 to the 1.8.2 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 8964fba6d5521867887a48184fcc4a0d0adbbebd Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:47:23 2020 +0200 Added MESOS-10134 to the 1.8.2 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 2e371c8..5aeccb5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -517,6 +517,7 @@ Release Notes - Mesos - Version 1.8.2 (WIP) * [MESOS-10007] - Command executor can miss exit status for short-lived commands due to double-reaping. * [MESOS-10015] - updateAllocation() can stall the allocator with a huge number of reservations on an agent. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10134] - Race between concurrent `javah` runs trying to create `java/jni` output directory. * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement
[mesos] 02/04: Added MESOS-10169 to the 1.9.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit e52684f2592c2889afab689fac439c230572f6b1 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:30:30 2020 +0200 Added MESOS-10169 to the 1.9.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index d87bb04..9f5f64f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -235,6 +235,7 @@ Release Notes - Mesos - Version 1.9.1 (WIP) * [MESOS-10096] - Reactivating a draining agent leaves the agent in draining state. * [MESOS-10118] - Agent incorrectly handles draining when empty. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement * [MESOS-9889] - Master CPU high due to unexpected foreachkey behaviour in Master::__reregisterSlave.
[mesos] 04/04: Added MESOS-10169 to the 1.7.4 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 21a0399f70f3af52bc1bab8e2bcd227c7defe456 Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:32:06 2020 +0200 Added MESOS-10169 to the 1.7.4 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index faba819..54ffc69 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -948,6 +948,7 @@ Release Notes - Mesos - Version 1.7.4 (WIP) ** Bug * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. Release Notes - Mesos - Version 1.7.3
[mesos] 01/04: Added MESOS-10169 to the 1.10.1 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit b22e95c2ad34b37cabb0365af257779d7a451aaa Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:29:50 2020 +0200 Added MESOS-10169 to the 1.10.1 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index dd51488..d87bb04 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,7 @@ Release Notes - Mesos - Version 1.10.1 (WIP) ** Bug * [MESOS-9609] - Master check failure when marking agent unreachable. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. Release Notes - Mesos - Version 1.10.0
[mesos] branch master updated (508afd5 -> 21a0399)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git. from 508afd5 Added MESOS-9609 to the 1.8.2 CHANGELOG. new b22e95c Added MESOS-10169 to the 1.10.1 CHANGELOG. new e52684f Added MESOS-10169 to the 1.9.1 CHANGELOG. new 230744d Added MESOS-10169 to the 1.8.2 CHANGELOG. new 21a0399 Added MESOS-10169 to the 1.7.4 CHANGELOG. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG | 4 1 file changed, 4 insertions(+)
[mesos] 03/04: Added MESOS-10169 to the 1.8.2 CHANGELOG.
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git commit 230744db39b9870f1bc76473c9ee7eff96835aae Author: Andrei Sekretenko AuthorDate: Thu Sep 10 21:31:43 2020 +0200 Added MESOS-10169 to the 1.8.2 CHANGELOG. --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 9f5f64f..faba819 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -515,6 +515,7 @@ Release Notes - Mesos - Version 1.8.2 (WIP) * [MESOS-10007] - Command executor can miss exit status for short-lived commands due to double-reaping. * [MESOS-10015] - updateAllocation() can stall the allocator with a huge number of reservations on an agent. * [MESOS-10126] - Docker volume isolator needs to clean up the `info` struct regardless the result of unmount operation + * [MESOS-10169] - Reintroduce image fetch deduplication while keeping it possible to destroy UCR containers in PROVISIONING state. ** Improvement * [MESOS-9889] - Master CPU high due to unexpected foreachkey behaviour in Master::__reregisterSlave.
[mesos] branch master updated: Moved framework update out of `connectAndActivateRecoveredFramework`.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new abf0c7a Moved framework update out of
`connectAndActivateRecoveredFramework`.
abf0c7a is described below
commit abf0c7ab8ae94b06349fdbb74c4e76e78d5c371d
Author: Andrei Sekretenko
AuthorDate: Fri Sep 4 17:04:25 2020 +0200
Moved framework update out of `connectAndActivateRecoveredFramework`.
This patch consolidates updating framework in the Subscribe call
by moving `updateFramework()` invocation from
`connectAndActivateRecoveredFramework()` into `Master::_subscribe()`.
Review: https://reviews.apache.org/r/72838
---
src/master/master.cpp | 71 ---
src/master/master.hpp | 4 +--
2 files changed, 29 insertions(+), 46 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 5b5d5c3..e99cc6b 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -2823,12 +2823,11 @@ void Master::_subscribe(
framework->metrics.incrementCall(scheduler::Call::SUBSCRIBE);
+ updateFramework(framework, frameworkInfo, std::move(options));
+
if (!framework->recovered()) {
// The framework has previously been registered with this master;
// it may or may not currently be connected.
-
-updateFramework(framework, frameworkInfo, std::move(options));
-
framework->reregisteredTime = Clock::now();
// Always failover the old framework connection. See MESOS-4712 for
details.
@@ -2836,12 +2835,7 @@ void Master::_subscribe(
} else {
// The framework has not yet reregistered after master failover.
connectAndActivateRecoveredFramework(
-framework,
-frameworkInfo,
-None(),
-http,
-objectApprovers.get(),
-std::move(options));
+framework, None(), http, objectApprovers.get());
}
// TODO(asekretenko): Consider avoiding to broadcast `FrameworkInfo` to
agents
@@ -3123,32 +3117,32 @@ void Master::_subscribe(
return;
}
+ // Using the "force" field of the scheduler allows us to reject a
+ // scheduler that got partitioned but didn't die (in ZooKeeper
+ // speak this means didn't lose their session) and then
+ // eventually tried to connect to this master even though
+ // another instance of their scheduler has reconnected.
+
+ // Test that the scheduler trying to subscribe with a new PID sets `force`.
+ if (!framework->recovered() && framework->pid() != from && !force) {
+LOG(ERROR) << "Disallowing subscription attempt of"
+ << " framework " << *framework
+ << " because it is not expected from " << from;
+
+FrameworkErrorMessage message;
+message.set_message("Framework failed over");
+send(from, message);
+return;
+ }
+
+ // It is now safe to update the framework fields since the request is now
+ // guaranteed to be successful. We use the fields passed in during
+ // re-registration.
+ updateFramework(framework, frameworkInfo, std::move(options));
+
if (!framework->recovered()) {
// The framework has previously been registered with this master;
// it may or may not currently be connected.
-//
-// Using the "force" field of the scheduler allows us to keep a
-// scheduler that got partitioned but didn't die (in ZooKeeper
-// speak this means didn't lose their session) and then
-// eventually tried to connect to this master even though
-// another instance of their scheduler has reconnected.
-
-// Test for the error case first.
-if ((framework->pid() != from) && !force) {
- LOG(ERROR) << "Disallowing subscription attempt of"
- << " framework " << *framework
- << " because it is not expected from " << from;
-
- FrameworkErrorMessage message;
- message.set_message("Framework failed over");
- send(from, message);
- return;
-}
-
-// It is now safe to update the framework fields since the request is now
-// guaranteed to be successful. We use the fields passed in during
-// re-registration.
-updateFramework(framework, frameworkInfo, std::move(options));
framework->reregisteredTime = Clock::now();
@@ -3207,12 +3201,7 @@ void Master::_subscribe(
} else {
// The framework has not yet reregistered after master failover.
connectAndActivateRecoveredFramework(
- framework,
-frameworkInfo,
-from,
-None(),
-objectApprovers.get(),
-std::move(options));
+framework, from, None(), objectApprovers.get());
}
// TODO(asekrete
[mesos] 05/05: Added a test for discarding image pull on discard of getting image.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8fc28789a7441a24ce0165281f9cadedf5cce417
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:36:40 2020 +0200
Added a test for discarding image pull on discard of getting image.
This test checks that when all futures returned by docker store's
`get()` that are pending image pull are discarded by the callers,
the pull future (returned by the puller to the store) is discarded
by the store as well.
Review: https://reviews.apache.org/r/72794
---
.../containerizer/provisioner_docker_tests.cpp | 45 ++
1 file changed, 45 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index dc0e7e9..23c7cd4 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -23,6 +23,7 @@
#include
#include
+#include
#include
#include
#include
@@ -59,6 +60,7 @@ namespace spec = ::docker::spec;
using std::string;
using std::vector;
+using process::Clock;
using process::Future;
using process::Owned;
using process::PID;
@@ -410,6 +412,49 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
}
+// This tests that pulling the image will be cancelled if all the pending
+// futures returned by `Store::get()` for this pull are discarded.
+TEST_F(ProvisionerDockerLocalStoreTest, PullDiscarded)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pullCalled;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pullCalled);
+
+ imageInfo1.discard();
+ imageInfo2.discard();
+
+ Clock::pause();
+ Clock::settle();
+
+ ASSERT_TRUE(promise.future().hasDiscard());
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
[mesos] 01/05: Deduplicated concurrent image pulls by docker store.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b4a49fd915cacd111cedc3c583d69bc2b9d6468e
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:02:51 2020 +0200
Deduplicated concurrent image pulls by docker store.
This patch makes the docker store reuse a pending pull if asked for
an image that is already being pulled.
The pull caching follows the same approach as the earlier attempt in
https://reviews.apache.org/r/39331 . However, handing out futures to
the store callers and handling their discards are performed differently,
using a form of reference counting, so that the pull itself is discarded
only if all the futures returned by `Store::get()` have been discarded.
Review: https://reviews.apache.org/r/72790
---
.../mesos/provisioner/docker/store.cpp | 110 -
1 file changed, 85 insertions(+), 25 deletions(-)
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index bf2be90..bbba8cf 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -143,6 +143,26 @@ private:
Owned metadataManager;
Owned puller;
+ // The get() method deduplicates image pulls by caching a single Future
+ // per each initiated image pull, handing out `undiscardable(pull)` to the
+ // caller (to prevent discards by the callers from propagating and discarding
+ // the pull) and tracking the number of the handed out `undiscardable`s for
+ // each pending pull that themselves have not been discarded yet
(`useCount`).
+ //
+ // The pull future itself is discarded as soon as its `useCount` drops to
+ // zero to notify the puller that the pull has been cancelled.
+
+ // A pull future and its use count.
+ struct Pull
+ {
+Future future;
+size_t useCount;
+ };
+
+ // The pending pull futures are keyed by the stringified `ImageReference`
+ // for the image being pulled.
+ hashmap pulling;
+
// For executing path removals in a separated actor.
process::Executor executor;
@@ -338,34 +358,69 @@ Future StoreProcess::_get(
}
}
- Try staging =
-os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+ const string pullKey = stringify(reference);
- if (staging.isError()) {
-return Failure(
-"Failed to create a staging directory: " + staging.error());
- }
+ Future pull = [&]() -> Future {
+if (pulling.contains(pullKey)) {
+ pulling.at(pullKey).useCount += 1;
- LOG(INFO) << "Pulling image '" << reference << "'";
-
- return metrics.image_pull.time(puller->pull(
- reference,
- staging.get(),
- backend,
- config)
-.then(defer(self(), ::moveLayers, staging.get(), lambda::_1, backend))
-.then(defer(self(), [=](const Image& image) {
- LOG(INFO) << "Caching image '" << reference << "'";
- return metadataManager->put(image);
-}))
-.onAny(defer(self(), [=](const Future& image) {
- LOG(INFO) << "Removing staging directory '" << staging.get() << "'";
- Try rmdir = os::rmdir(staging.get());
- if (rmdir.isError()) {
-LOG(WARNING) << "Failed to remove staging directory '" << staging.get()
- << "': " << rmdir.error();
+ // NOTE: In the rare case when the future has already failed but the
+ // onAny() callback has not been executed yet, we will hand out an
already
+ // FAILED future. This is basically equivalent to handing out a PENDING
+ // future that will fail immediately, and should not be an issue.
+ CHECK(!pulling.at(pullKey).future.hasDiscard());
+ return pulling.at(pullKey).future;
+}
+
+Try staging =
+ os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+
+if (staging.isError()) {
+ return Failure(
+ "Failed to create a staging directory: " + staging.error());
+}
+
+LOG(INFO) << "Pulling image '" << reference << "'";
+
+Future pull = metrics.image_pull.time(
+puller->pull(reference, staging.get(), backend, config)
+ .then(defer(
+ self(), ::moveLayers, staging.get(), lambda::_1, backend))
+ .then(defer(
+ self(),
+ [=](const Image& image) {
+LOG(INFO) << "Caching image '" << reference << "'";
+return metadataManager->put(image);
+ }))
+ .onAny(defer(self(), [=](const Future& image) {
+pulling.erase(pullK
[mesos] 04/05: Added discarding a future returned by get() into the docker store test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit f1008ce5bc0105d00e9061efc61a9fc5f61a289d
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:16:39 2020 +0200
Added discarding a future returned by get() into the docker store test.
This patch extends the `PullingSameImageSimultaneously` test
with discarding one of the futures returned by `store->get()`
and making sure that the pull still completes.
Review: https://reviews.apache.org/r/72793
---
src/tests/containerizer/provisioner_docker_tests.cpp | 11 +++
1 file changed, 11 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index bd165ba..dc0e7e9 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -330,6 +330,9 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
+// In addition, this test verifies that if some of the futures
+// returned by `Store::get()` that are pending image pull are discarded,
+// the pull still completes.
TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
@@ -380,6 +383,9 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
Future imageInfo2 =
store.get()->get(mesosImage, COPY_BACKEND);
+ Future imageInfo3 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
Try reference =
spec::parseImageReference(mesosImage.docker().name());
@@ -390,6 +396,11 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
result.add_layer_ids("456");
ASSERT_TRUE(imageInfo2.isPending());
+ ASSERT_TRUE(imageInfo3.isPending());
+
+ // Pull should still complete, despite of the discard of a single future.
+ imageInfo3.discard();
+
promise.set(result);
AWAIT_READY(imageInfo1);
[mesos] branch 1.7.x updated (47f4850 -> 8fc2878)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 47f4850 Fixed build broken by a comment in the patch that fixes `javah` race. new b4a49fd Deduplicated concurrent image pulls by docker store. new df5fcf7 Reverted removal of the PullingSameImageSimutanuously test. new 66f8ed1 Fixed typos in the name of `PullingSameImageSimutanuously` test. new f1008ce Added discarding a future returned by get() into the docker store test. new 8fc2878 Added a test for discarding image pull on discard of getting image. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mesos/provisioner/docker/store.cpp | 110 ++ .../containerizer/provisioner_docker_tests.cpp | 163 + 2 files changed, 248 insertions(+), 25 deletions(-)
[mesos] 02/05: Reverted removal of the PullingSameImageSimutanuously test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit df5fcf75b1dd04daa44631b43441d1bd85dd0241
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:49:13 2020 +0200
Reverted removal of the PullingSameImageSimutanuously test.
Now that the docker store triggers pull at most once per multiple
simultaneous requests to the store, removal of the
`PullingSameImageSimutanuously` test in
33c61a1907129126f3b2e37b1f53827a04e89a34 can be reverted.
Review: https://reviews.apache.org/r/72791
---
.../containerizer/provisioner_docker_tests.cpp | 107 +
1 file changed, 107 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index caac470..76ba5f1 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -39,6 +39,8 @@
#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
#include "tests/environment.hpp"
@@ -74,6 +76,8 @@ using mesos::master::detector::MasterDetector;
using slave::ImageInfo;
using slave::Slave;
+using slave::docker::Puller;
+using slave::docker::RegistryPuller;
using slave::docker::Store;
using testing::WithParamInterface;
@@ -292,6 +296,109 @@ TEST_F(ProvisionerDockerLocalStoreTest, MissingLayer)
}
+class MockPuller : public Puller
+{
+public:
+ MockPuller()
+ {
+EXPECT_CALL(*this, pull(_, _, _, _))
+ .WillRepeatedly(Invoke(this, ::unmocked_pull));
+ }
+
+ ~MockPuller() override {}
+
+ MOCK_METHOD4(
+ pull,
+ Future(
+ const spec::ImageReference&,
+ const string&,
+ const string&,
+ const Option&));
+
+ Future unmocked_pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend,
+ const Option& config)
+ {
+// TODO(gilbert): Allow return Image to be overridden.
+return slave::docker::Image();
+ }
+};
+
+
+// This tests the store to pull the same image simultaneously.
+// This test verifies that the store only calls the puller once
+// when multiple requests for the same image is in flight.
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pull;
+ Future directory;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ FutureArg<1>(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pull);
+ AWAIT_READY(directory);
+
+ // TODO(gilbert): Need a helper method to create test layers
+ // which will allow us to set manifest so that we can add
+ // checks here.
+ const string layerPath = path::join(directory.get(), "456");
+
+ Try mkdir = os::mkdir(layerPath);
+ ASSERT_SOME(mkdir);
+
+ JSON::Value manifest = JSON::parse(
+"{"
+" \"parent\": \"\""
+"}").get();
+
+ ASSERT_SOME(
+ os::write(path::join(layerPath, "json"), stringify(manifest)));
+
+ ASSERT_TRUE(imageInfo1.isPending());
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Try reference =
+spec::parseImageReference(mesosImage.docker().name());
+
+ ASSERT_SOME(reference);
+
+ slave::docker::Image result;
+ result.mutable_reference()->CopyFrom(reference.get());
+ result.add_layer_ids("456");
+
+ ASSERT_TRUE(imageInfo2.isPending());
+ promise.set(result);
+
+ AWAIT_READY(imageInfo1);
+ AWAIT_READY(imageInfo2);
+
+ EXPECT_EQ(imageInfo1->layers, imageInfo2->layers);
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
[mesos] 03/05: Fixed typos in the name of `PullingSameImageSimutanuously` test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 66f8ed162a2d6022b49af426144413830be6501c
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:45:44 2020 +0200
Fixed typos in the name of `PullingSameImageSimutanuously` test.
Review: https://reviews.apache.org/r/72792
---
src/tests/containerizer/provisioner_docker_tests.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 76ba5f1..bd165ba 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -330,7 +330,7 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
-TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
flags.docker_registry = path::join(os::getcwd(), "images");
[mesos] 03/05: Fixed typos in the name of `PullingSameImageSimutanuously` test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 9aed797be27d6b831a7adf4b5e373c1c9e6640b6
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:45:44 2020 +0200
Fixed typos in the name of `PullingSameImageSimutanuously` test.
Review: https://reviews.apache.org/r/72792
---
src/tests/containerizer/provisioner_docker_tests.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 1c5d720..6bddb40 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -335,7 +335,7 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
-TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
flags.docker_registry = path::join(os::getcwd(), "images");
[mesos] branch 1.8.x updated (3728a1b -> d160015)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.8.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 3728a1b Fixed build broken by a comment in the patch that fixes `javah` race. new cd49d82 Deduplicated concurrent image pulls by docker store. new c42364a Reverted removal of the PullingSameImageSimutanuously test. new 9aed797 Fixed typos in the name of `PullingSameImageSimutanuously` test. new 6afaf09 Added discarding a future returned by get() into the docker store test. new d160015 Added a test for discarding image pull on discard of getting image. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mesos/provisioner/docker/store.cpp | 110 ++ .../containerizer/provisioner_docker_tests.cpp | 163 + 2 files changed, 248 insertions(+), 25 deletions(-)
[mesos] 05/05: Added a test for discarding image pull on discard of getting image.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit d160015e544082ff310a7e4d2e94e3c4eae9499b
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:36:40 2020 +0200
Added a test for discarding image pull on discard of getting image.
This test checks that when all futures returned by docker store's
`get()` that are pending image pull are discarded by the callers,
the pull future (returned by the puller to the store) is discarded
by the store as well.
Review: https://reviews.apache.org/r/72794
---
.../containerizer/provisioner_docker_tests.cpp | 45 ++
1 file changed, 45 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 130c904..4271a1c 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -23,6 +23,7 @@
#include
#include
+#include
#include
#include
#include
@@ -59,6 +60,7 @@ namespace spec = ::docker::spec;
using std::string;
using std::vector;
+using process::Clock;
using process::Future;
using process::Owned;
using process::PID;
@@ -415,6 +417,49 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
}
+// This tests that pulling the image will be cancelled if all the pending
+// futures returned by `Store::get()` for this pull are discarded.
+TEST_F(ProvisionerDockerLocalStoreTest, PullDiscarded)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pullCalled;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pullCalled);
+
+ imageInfo1.discard();
+ imageInfo2.discard();
+
+ Clock::pause();
+ Clock::settle();
+
+ ASSERT_TRUE(promise.future().hasDiscard());
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
[mesos] 02/05: Reverted removal of the PullingSameImageSimutanuously test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c42364a833df202752a15f27542e6e5ab6a78179
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:49:13 2020 +0200
Reverted removal of the PullingSameImageSimutanuously test.
Now that the docker store triggers pull at most once per multiple
simultaneous requests to the store, removal of the
`PullingSameImageSimutanuously` test in
33c61a1907129126f3b2e37b1f53827a04e89a34 can be reverted.
Review: https://reviews.apache.org/r/72791
---
.../containerizer/provisioner_docker_tests.cpp | 107 +
1 file changed, 107 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index b136ce5..1c5d720 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -39,6 +39,8 @@
#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
#include "tests/environment.hpp"
@@ -79,6 +81,8 @@ using mesos::slave::ContainerTermination;
using slave::ImageInfo;
using slave::Slave;
+using slave::docker::Puller;
+using slave::docker::RegistryPuller;
using slave::docker::Store;
using testing::WithParamInterface;
@@ -297,6 +301,109 @@ TEST_F(ProvisionerDockerLocalStoreTest, MissingLayer)
}
+class MockPuller : public Puller
+{
+public:
+ MockPuller()
+ {
+EXPECT_CALL(*this, pull(_, _, _, _))
+ .WillRepeatedly(Invoke(this, ::unmocked_pull));
+ }
+
+ ~MockPuller() override {}
+
+ MOCK_METHOD4(
+ pull,
+ Future(
+ const spec::ImageReference&,
+ const string&,
+ const string&,
+ const Option&));
+
+ Future unmocked_pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend,
+ const Option& config)
+ {
+// TODO(gilbert): Allow return Image to be overridden.
+return slave::docker::Image();
+ }
+};
+
+
+// This tests the store to pull the same image simultaneously.
+// This test verifies that the store only calls the puller once
+// when multiple requests for the same image is in flight.
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pull;
+ Future directory;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ FutureArg<1>(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pull);
+ AWAIT_READY(directory);
+
+ // TODO(gilbert): Need a helper method to create test layers
+ // which will allow us to set manifest so that we can add
+ // checks here.
+ const string layerPath = path::join(directory.get(), "456");
+
+ Try mkdir = os::mkdir(layerPath);
+ ASSERT_SOME(mkdir);
+
+ JSON::Value manifest = JSON::parse(
+"{"
+" \"parent\": \"\""
+"}").get();
+
+ ASSERT_SOME(
+ os::write(path::join(layerPath, "json"), stringify(manifest)));
+
+ ASSERT_TRUE(imageInfo1.isPending());
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Try reference =
+spec::parseImageReference(mesosImage.docker().name());
+
+ ASSERT_SOME(reference);
+
+ slave::docker::Image result;
+ result.mutable_reference()->CopyFrom(reference.get());
+ result.add_layer_ids("456");
+
+ ASSERT_TRUE(imageInfo2.isPending());
+ promise.set(result);
+
+ AWAIT_READY(imageInfo1);
+ AWAIT_READY(imageInfo2);
+
+ EXPECT_EQ(imageInfo1->layers, imageInfo2->layers);
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
[mesos] 01/05: Deduplicated concurrent image pulls by docker store.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit cd49d821de12a3d2a237ae57844878b649b50295
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:02:51 2020 +0200
Deduplicated concurrent image pulls by docker store.
This patch makes the docker store reuse a pending pull if asked for
an image that is already being pulled.
The pull caching follows the same approach as the earlier attempt in
https://reviews.apache.org/r/39331 . However, handing out futures to
the store callers and handling their discards are performed differently,
using a form of reference counting, so that the pull itself is discarded
only if all the futures returned by `Store::get()` have been discarded.
Review: https://reviews.apache.org/r/72790
---
.../mesos/provisioner/docker/store.cpp | 110 -
1 file changed, 85 insertions(+), 25 deletions(-)
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index bf2be90..bbba8cf 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -143,6 +143,26 @@ private:
Owned metadataManager;
Owned puller;
+ // The get() method deduplicates image pulls by caching a single Future
+ // per each initiated image pull, handing out `undiscardable(pull)` to the
+ // caller (to prevent discards by the callers from propagating and discarding
+ // the pull) and tracking the number of the handed out `undiscardable`s for
+ // each pending pull that themselves have not been discarded yet
(`useCount`).
+ //
+ // The pull future itself is discarded as soon as its `useCount` drops to
+ // zero to notify the puller that the pull has been cancelled.
+
+ // A pull future and its use count.
+ struct Pull
+ {
+Future future;
+size_t useCount;
+ };
+
+ // The pending pull futures are keyed by the stringified `ImageReference`
+ // for the image being pulled.
+ hashmap pulling;
+
// For executing path removals in a separated actor.
process::Executor executor;
@@ -338,34 +358,69 @@ Future StoreProcess::_get(
}
}
- Try staging =
-os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+ const string pullKey = stringify(reference);
- if (staging.isError()) {
-return Failure(
-"Failed to create a staging directory: " + staging.error());
- }
+ Future pull = [&]() -> Future {
+if (pulling.contains(pullKey)) {
+ pulling.at(pullKey).useCount += 1;
- LOG(INFO) << "Pulling image '" << reference << "'";
-
- return metrics.image_pull.time(puller->pull(
- reference,
- staging.get(),
- backend,
- config)
-.then(defer(self(), ::moveLayers, staging.get(), lambda::_1, backend))
-.then(defer(self(), [=](const Image& image) {
- LOG(INFO) << "Caching image '" << reference << "'";
- return metadataManager->put(image);
-}))
-.onAny(defer(self(), [=](const Future& image) {
- LOG(INFO) << "Removing staging directory '" << staging.get() << "'";
- Try rmdir = os::rmdir(staging.get());
- if (rmdir.isError()) {
-LOG(WARNING) << "Failed to remove staging directory '" << staging.get()
- << "': " << rmdir.error();
+ // NOTE: In the rare case when the future has already failed but the
+ // onAny() callback has not been executed yet, we will hand out an
already
+ // FAILED future. This is basically equivalent to handing out a PENDING
+ // future that will fail immediately, and should not be an issue.
+ CHECK(!pulling.at(pullKey).future.hasDiscard());
+ return pulling.at(pullKey).future;
+}
+
+Try staging =
+ os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+
+if (staging.isError()) {
+ return Failure(
+ "Failed to create a staging directory: " + staging.error());
+}
+
+LOG(INFO) << "Pulling image '" << reference << "'";
+
+Future pull = metrics.image_pull.time(
+puller->pull(reference, staging.get(), backend, config)
+ .then(defer(
+ self(), ::moveLayers, staging.get(), lambda::_1, backend))
+ .then(defer(
+ self(),
+ [=](const Image& image) {
+LOG(INFO) << "Caching image '" << reference << "'";
+return metadataManager->put(image);
+ }))
+ .onAny(defer(self(), [=](const Future& image) {
+pulling.erase(pullK
[mesos] 04/05: Added discarding a future returned by get() into the docker store test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 6afaf0957048aeec30ff2383213cca7dc0893d32
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:16:39 2020 +0200
Added discarding a future returned by get() into the docker store test.
This patch extends the `PullingSameImageSimultaneously` test
with discarding one of the futures returned by `store->get()`
and making sure that the pull still completes.
Review: https://reviews.apache.org/r/72793
---
src/tests/containerizer/provisioner_docker_tests.cpp | 11 +++
1 file changed, 11 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 6bddb40..130c904 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -335,6 +335,9 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
+// In addition, this test verifies that if some of the futures
+// returned by `Store::get()` that are pending image pull are discarded,
+// the pull still completes.
TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
@@ -385,6 +388,9 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
Future imageInfo2 =
store.get()->get(mesosImage, COPY_BACKEND);
+ Future imageInfo3 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
Try reference =
spec::parseImageReference(mesosImage.docker().name());
@@ -395,6 +401,11 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
result.add_layer_ids("456");
ASSERT_TRUE(imageInfo2.isPending());
+ ASSERT_TRUE(imageInfo3.isPending());
+
+ // Pull should still complete, despite of the discard of a single future.
+ imageInfo3.discard();
+
promise.set(result);
AWAIT_READY(imageInfo1);
[mesos] 05/05: Added a test for discarding image pull on discard of getting image.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.9.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit e98ced1e93751b78c3248221115fcf4db8a39b29
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:36:40 2020 +0200
Added a test for discarding image pull on discard of getting image.
This test checks that when all futures returned by docker store's
`get()` that are pending image pull are discarded by the callers,
the pull future (returned by the puller to the store) is discarded
by the store as well.
Review: https://reviews.apache.org/r/72794
---
.../containerizer/provisioner_docker_tests.cpp | 45 ++
1 file changed, 45 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 5de97e2..b1f38ae 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -23,6 +23,7 @@
#include
#include
+#include
#include
#include
#include
@@ -59,6 +60,7 @@ namespace spec = ::docker::spec;
using std::string;
using std::vector;
+using process::Clock;
using process::Future;
using process::Owned;
using process::PID;
@@ -415,6 +417,49 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
}
+// This tests that pulling the image will be cancelled if all the pending
+// futures returned by `Store::get()` for this pull are discarded.
+TEST_F(ProvisionerDockerLocalStoreTest, PullDiscarded)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pullCalled;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pullCalled);
+
+ imageInfo1.discard();
+ imageInfo2.discard();
+
+ Clock::pause();
+ Clock::settle();
+
+ ASSERT_TRUE(promise.future().hasDiscard());
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
[mesos] 04/05: Added discarding a future returned by get() into the docker store test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.9.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 404f4650a6c0f68c535b4e4832103ecd2d288fec
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:16:39 2020 +0200
Added discarding a future returned by get() into the docker store test.
This patch extends the `PullingSameImageSimultaneously` test
with discarding one of the futures returned by `store->get()`
and making sure that the pull still completes.
Review: https://reviews.apache.org/r/72793
---
src/tests/containerizer/provisioner_docker_tests.cpp | 11 +++
1 file changed, 11 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 477a875..5de97e2 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -335,6 +335,9 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
+// In addition, this test verifies that if some of the futures
+// returned by `Store::get()` that are pending image pull are discarded,
+// the pull still completes.
TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
@@ -385,6 +388,9 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
Future imageInfo2 =
store.get()->get(mesosImage, COPY_BACKEND);
+ Future imageInfo3 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
Try reference =
spec::parseImageReference(mesosImage.docker().name());
@@ -395,6 +401,11 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
result.add_layer_ids("456");
ASSERT_TRUE(imageInfo2.isPending());
+ ASSERT_TRUE(imageInfo3.isPending());
+
+ // Pull should still complete, despite of the discard of a single future.
+ imageInfo3.discard();
+
promise.set(result);
AWAIT_READY(imageInfo1);
[mesos] 02/05: Reverted removal of the PullingSameImageSimutanuously test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.9.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 0e54e8f51afc5d139496a0d0b5378a765f2cf9dd
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:49:13 2020 +0200
Reverted removal of the PullingSameImageSimutanuously test.
Now that the docker store triggers pull at most once per multiple
simultaneous requests to the store, removal of the
`PullingSameImageSimutanuously` test in
33c61a1907129126f3b2e37b1f53827a04e89a34 can be reverted.
Review: https://reviews.apache.org/r/72791
---
.../containerizer/provisioner_docker_tests.cpp | 107 +
1 file changed, 107 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index c3ef4a0..5d5a355 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -39,6 +39,8 @@
#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
#include "tests/environment.hpp"
@@ -79,6 +81,8 @@ using mesos::slave::ContainerTermination;
using slave::ImageInfo;
using slave::Slave;
+using slave::docker::Puller;
+using slave::docker::RegistryPuller;
using slave::docker::Store;
using testing::WithParamInterface;
@@ -297,6 +301,109 @@ TEST_F(ProvisionerDockerLocalStoreTest, MissingLayer)
}
+class MockPuller : public Puller
+{
+public:
+ MockPuller()
+ {
+EXPECT_CALL(*this, pull(_, _, _, _))
+ .WillRepeatedly(Invoke(this, ::unmocked_pull));
+ }
+
+ ~MockPuller() override {}
+
+ MOCK_METHOD4(
+ pull,
+ Future(
+ const spec::ImageReference&,
+ const string&,
+ const string&,
+ const Option&));
+
+ Future unmocked_pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend,
+ const Option& config)
+ {
+// TODO(gilbert): Allow return Image to be overridden.
+return slave::docker::Image();
+ }
+};
+
+
+// This tests the store to pull the same image simultaneously.
+// This test verifies that the store only calls the puller once
+// when multiple requests for the same image is in flight.
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pull;
+ Future directory;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ FutureArg<1>(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pull);
+ AWAIT_READY(directory);
+
+ // TODO(gilbert): Need a helper method to create test layers
+ // which will allow us to set manifest so that we can add
+ // checks here.
+ const string layerPath = path::join(directory.get(), "456");
+
+ Try mkdir = os::mkdir(layerPath);
+ ASSERT_SOME(mkdir);
+
+ JSON::Value manifest = JSON::parse(
+"{"
+" \"parent\": \"\""
+"}").get();
+
+ ASSERT_SOME(
+ os::write(path::join(layerPath, "json"), stringify(manifest)));
+
+ ASSERT_TRUE(imageInfo1.isPending());
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Try reference =
+spec::parseImageReference(mesosImage.docker().name());
+
+ ASSERT_SOME(reference);
+
+ slave::docker::Image result;
+ result.mutable_reference()->CopyFrom(reference.get());
+ result.add_layer_ids("456");
+
+ ASSERT_TRUE(imageInfo2.isPending());
+ promise.set(result);
+
+ AWAIT_READY(imageInfo1);
+ AWAIT_READY(imageInfo2);
+
+ EXPECT_EQ(imageInfo1->layers, imageInfo2->layers);
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
[mesos] 01/05: Deduplicated concurrent image pulls by docker store.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.9.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 30d63f89e5fbac10192d7daa33fcea4e38701241
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:02:51 2020 +0200
Deduplicated concurrent image pulls by docker store.
This patch makes the docker store reuse a pending pull if asked for
an image that is already being pulled.
The pull caching follows the same approach as the earlier attempt in
https://reviews.apache.org/r/39331 . However, handing out futures to
the store callers and handling their discards are performed differently,
using a form of reference counting, so that the pull itself is discarded
only if all the futures returned by `Store::get()` have been discarded.
Review: https://reviews.apache.org/r/72790
---
.../mesos/provisioner/docker/store.cpp | 110 -
1 file changed, 85 insertions(+), 25 deletions(-)
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index bf2be90..bbba8cf 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -143,6 +143,26 @@ private:
Owned metadataManager;
Owned puller;
+ // The get() method deduplicates image pulls by caching a single Future
+ // per each initiated image pull, handing out `undiscardable(pull)` to the
+ // caller (to prevent discards by the callers from propagating and discarding
+ // the pull) and tracking the number of the handed out `undiscardable`s for
+ // each pending pull that themselves have not been discarded yet
(`useCount`).
+ //
+ // The pull future itself is discarded as soon as its `useCount` drops to
+ // zero to notify the puller that the pull has been cancelled.
+
+ // A pull future and its use count.
+ struct Pull
+ {
+Future future;
+size_t useCount;
+ };
+
+ // The pending pull futures are keyed by the stringified `ImageReference`
+ // for the image being pulled.
+ hashmap pulling;
+
// For executing path removals in a separated actor.
process::Executor executor;
@@ -338,34 +358,69 @@ Future StoreProcess::_get(
}
}
- Try staging =
-os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+ const string pullKey = stringify(reference);
- if (staging.isError()) {
-return Failure(
-"Failed to create a staging directory: " + staging.error());
- }
+ Future pull = [&]() -> Future {
+if (pulling.contains(pullKey)) {
+ pulling.at(pullKey).useCount += 1;
- LOG(INFO) << "Pulling image '" << reference << "'";
-
- return metrics.image_pull.time(puller->pull(
- reference,
- staging.get(),
- backend,
- config)
-.then(defer(self(), ::moveLayers, staging.get(), lambda::_1, backend))
-.then(defer(self(), [=](const Image& image) {
- LOG(INFO) << "Caching image '" << reference << "'";
- return metadataManager->put(image);
-}))
-.onAny(defer(self(), [=](const Future& image) {
- LOG(INFO) << "Removing staging directory '" << staging.get() << "'";
- Try rmdir = os::rmdir(staging.get());
- if (rmdir.isError()) {
-LOG(WARNING) << "Failed to remove staging directory '" << staging.get()
- << "': " << rmdir.error();
+ // NOTE: In the rare case when the future has already failed but the
+ // onAny() callback has not been executed yet, we will hand out an
already
+ // FAILED future. This is basically equivalent to handing out a PENDING
+ // future that will fail immediately, and should not be an issue.
+ CHECK(!pulling.at(pullKey).future.hasDiscard());
+ return pulling.at(pullKey).future;
+}
+
+Try staging =
+ os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+
+if (staging.isError()) {
+ return Failure(
+ "Failed to create a staging directory: " + staging.error());
+}
+
+LOG(INFO) << "Pulling image '" << reference << "'";
+
+Future pull = metrics.image_pull.time(
+puller->pull(reference, staging.get(), backend, config)
+ .then(defer(
+ self(), ::moveLayers, staging.get(), lambda::_1, backend))
+ .then(defer(
+ self(),
+ [=](const Image& image) {
+LOG(INFO) << "Caching image '" << reference << "'";
+return metadataManager->put(image);
+ }))
+ .onAny(defer(self(), [=](const Future& image) {
+pulling.erase(pullK
[mesos] 03/05: Fixed typos in the name of `PullingSameImageSimutanuously` test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.9.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit edd7930356250fb8cd42f57ac79d1749a815fb8a
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:45:44 2020 +0200
Fixed typos in the name of `PullingSameImageSimutanuously` test.
Review: https://reviews.apache.org/r/72792
---
src/tests/containerizer/provisioner_docker_tests.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 5d5a355..477a875 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -335,7 +335,7 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
-TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
flags.docker_registry = path::join(os::getcwd(), "images");
[mesos] branch 1.9.x updated (7f5d1c6 -> e98ced1)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.9.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 7f5d1c6 Fixed build broken by a comment in the patch that fixes `javah` race. new 30d63f8 Deduplicated concurrent image pulls by docker store. new 0e54e8f Reverted removal of the PullingSameImageSimutanuously test. new edd7930 Fixed typos in the name of `PullingSameImageSimutanuously` test. new 404f465 Added discarding a future returned by get() into the docker store test. new e98ced1 Added a test for discarding image pull on discard of getting image. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mesos/provisioner/docker/store.cpp | 110 ++ .../containerizer/provisioner_docker_tests.cpp | 163 + 2 files changed, 248 insertions(+), 25 deletions(-)
[mesos] 01/05: Deduplicated concurrent image pulls by docker store.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 51cf5a37570caa0405a1edd5b461f484384cb526
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:02:51 2020 +0200
Deduplicated concurrent image pulls by docker store.
This patch makes the docker store reuse a pending pull if asked for
an image that is already being pulled.
The pull caching follows the same approach as the earlier attempt in
https://reviews.apache.org/r/39331 . However, handing out futures to
the store callers and handling their discards are performed differently,
using a form of reference counting, so that the pull itself is discarded
only if all the futures returned by `Store::get()` have been discarded.
Review: https://reviews.apache.org/r/72790
---
.../mesos/provisioner/docker/store.cpp | 110 -
1 file changed, 85 insertions(+), 25 deletions(-)
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index bf2be90..bbba8cf 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -143,6 +143,26 @@ private:
Owned metadataManager;
Owned puller;
+ // The get() method deduplicates image pulls by caching a single Future
+ // per each initiated image pull, handing out `undiscardable(pull)` to the
+ // caller (to prevent discards by the callers from propagating and discarding
+ // the pull) and tracking the number of the handed out `undiscardable`s for
+ // each pending pull that themselves have not been discarded yet
(`useCount`).
+ //
+ // The pull future itself is discarded as soon as its `useCount` drops to
+ // zero to notify the puller that the pull has been cancelled.
+
+ // A pull future and its use count.
+ struct Pull
+ {
+Future future;
+size_t useCount;
+ };
+
+ // The pending pull futures are keyed by the stringified `ImageReference`
+ // for the image being pulled.
+ hashmap pulling;
+
// For executing path removals in a separated actor.
process::Executor executor;
@@ -338,34 +358,69 @@ Future StoreProcess::_get(
}
}
- Try staging =
-os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+ const string pullKey = stringify(reference);
- if (staging.isError()) {
-return Failure(
-"Failed to create a staging directory: " + staging.error());
- }
+ Future pull = [&]() -> Future {
+if (pulling.contains(pullKey)) {
+ pulling.at(pullKey).useCount += 1;
- LOG(INFO) << "Pulling image '" << reference << "'";
-
- return metrics.image_pull.time(puller->pull(
- reference,
- staging.get(),
- backend,
- config)
-.then(defer(self(), ::moveLayers, staging.get(), lambda::_1, backend))
-.then(defer(self(), [=](const Image& image) {
- LOG(INFO) << "Caching image '" << reference << "'";
- return metadataManager->put(image);
-}))
-.onAny(defer(self(), [=](const Future& image) {
- LOG(INFO) << "Removing staging directory '" << staging.get() << "'";
- Try rmdir = os::rmdir(staging.get());
- if (rmdir.isError()) {
-LOG(WARNING) << "Failed to remove staging directory '" << staging.get()
- << "': " << rmdir.error();
+ // NOTE: In the rare case when the future has already failed but the
+ // onAny() callback has not been executed yet, we will hand out an
already
+ // FAILED future. This is basically equivalent to handing out a PENDING
+ // future that will fail immediately, and should not be an issue.
+ CHECK(!pulling.at(pullKey).future.hasDiscard());
+ return pulling.at(pullKey).future;
+}
+
+Try staging =
+ os::mkdtemp(paths::getStagingTempDir(flags.docker_store_dir));
+
+if (staging.isError()) {
+ return Failure(
+ "Failed to create a staging directory: " + staging.error());
+}
+
+LOG(INFO) << "Pulling image '" << reference << "'";
+
+Future pull = metrics.image_pull.time(
+puller->pull(reference, staging.get(), backend, config)
+ .then(defer(
+ self(), ::moveLayers, staging.get(), lambda::_1, backend))
+ .then(defer(
+ self(),
+ [=](const Image& image) {
+LOG(INFO) << "Caching image '" << reference << "'";
+return metadataManager->put(image);
+ }))
+ .onAny(defer(self(), [=](const Future& image) {
+pulling.erase(pullK
[mesos] branch 1.10.x updated (8683463 -> 3ca3879)
This is an automated email from the ASF dual-hosted git repository. asekretenko pushed a change to branch 1.10.x in repository https://gitbox.apache.org/repos/asf/mesos.git. from 8683463 Fixed build broken by a comment in the patch that fixes `javah` race. new 51cf5a3 Deduplicated concurrent image pulls by docker store. new d4e4654 Reverted removal of the PullingSameImageSimutanuously test. new 1fbbbea Fixed typos in the name of `PullingSameImageSimutanuously` test. new 0f8a37a Added discarding a future returned by get() into the docker store test. new 3ca3879 Added a test for discarding image pull on discard of getting image. The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mesos/provisioner/docker/store.cpp | 110 ++ .../containerizer/provisioner_docker_tests.cpp | 163 + 2 files changed, 248 insertions(+), 25 deletions(-)
[mesos] 04/05: Added discarding a future returned by get() into the docker store test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 0f8a37a3a27e5458083866c716dd44c9bc11b5c0
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 18:16:39 2020 +0200
Added discarding a future returned by get() into the docker store test.
This patch extends the `PullingSameImageSimultaneously` test
with discarding one of the futures returned by `store->get()`
and making sure that the pull still completes.
Review: https://reviews.apache.org/r/72793
---
src/tests/containerizer/provisioner_docker_tests.cpp | 11 +++
1 file changed, 11 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index 477a875..5de97e2 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -335,6 +335,9 @@ public:
// This tests the store to pull the same image simultaneously.
// This test verifies that the store only calls the puller once
// when multiple requests for the same image is in flight.
+// In addition, this test verifies that if some of the futures
+// returned by `Store::get()` that are pending image pull are discarded,
+// the pull still completes.
TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimultaneously)
{
slave::Flags flags;
@@ -385,6 +388,9 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
Future imageInfo2 =
store.get()->get(mesosImage, COPY_BACKEND);
+ Future imageInfo3 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
Try reference =
spec::parseImageReference(mesosImage.docker().name());
@@ -395,6 +401,11 @@ TEST_F(ProvisionerDockerLocalStoreTest,
PullingSameImageSimultaneously)
result.add_layer_ids("456");
ASSERT_TRUE(imageInfo2.isPending());
+ ASSERT_TRUE(imageInfo3.isPending());
+
+ // Pull should still complete, despite of the discard of a single future.
+ imageInfo3.discard();
+
promise.set(result);
AWAIT_READY(imageInfo1);
[mesos] 02/05: Reverted removal of the PullingSameImageSimutanuously test.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch 1.10.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit d4e4654f82cd5dd99fc9a8318df02674c0ed6320
Author: Andrei Sekretenko
AuthorDate: Thu Aug 20 17:49:13 2020 +0200
Reverted removal of the PullingSameImageSimutanuously test.
Now that the docker store triggers pull at most once per multiple
simultaneous requests to the store, removal of the
`PullingSameImageSimutanuously` test in
33c61a1907129126f3b2e37b1f53827a04e89a34 can be reverted.
Review: https://reviews.apache.org/r/72791
---
.../containerizer/provisioner_docker_tests.cpp | 107 +
1 file changed, 107 insertions(+)
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp
b/src/tests/containerizer/provisioner_docker_tests.cpp
index c3ef4a0..5d5a355 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -39,6 +39,8 @@
#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
#include "tests/environment.hpp"
@@ -79,6 +81,8 @@ using mesos::slave::ContainerTermination;
using slave::ImageInfo;
using slave::Slave;
+using slave::docker::Puller;
+using slave::docker::RegistryPuller;
using slave::docker::Store;
using testing::WithParamInterface;
@@ -297,6 +301,109 @@ TEST_F(ProvisionerDockerLocalStoreTest, MissingLayer)
}
+class MockPuller : public Puller
+{
+public:
+ MockPuller()
+ {
+EXPECT_CALL(*this, pull(_, _, _, _))
+ .WillRepeatedly(Invoke(this, ::unmocked_pull));
+ }
+
+ ~MockPuller() override {}
+
+ MOCK_METHOD4(
+ pull,
+ Future(
+ const spec::ImageReference&,
+ const string&,
+ const string&,
+ const Option&));
+
+ Future unmocked_pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend,
+ const Option& config)
+ {
+// TODO(gilbert): Allow return Image to be overridden.
+return slave::docker::Image();
+ }
+};
+
+
+// This tests the store to pull the same image simultaneously.
+// This test verifies that the store only calls the puller once
+// when multiple requests for the same image is in flight.
+TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
+{
+ slave::Flags flags;
+ flags.docker_registry = path::join(os::getcwd(), "images");
+ flags.docker_store_dir = path::join(os::getcwd(), "store");
+
+ MockPuller* puller = new MockPuller();
+ Future pull;
+ Future directory;
+ Promise promise;
+
+ EXPECT_CALL(*puller, pull(_, _, _, _))
+.WillOnce(testing::DoAll(FutureSatisfy(),
+ FutureArg<1>(),
+ Return(promise.future(;
+
+ Try> store =
+ slave::docker::Store::create(flags, Owned(puller));
+ ASSERT_SOME(store);
+
+ Image mesosImage;
+ mesosImage.set_type(Image::DOCKER);
+ mesosImage.mutable_docker()->set_name("abc");
+
+ Future imageInfo1 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ AWAIT_READY(pull);
+ AWAIT_READY(directory);
+
+ // TODO(gilbert): Need a helper method to create test layers
+ // which will allow us to set manifest so that we can add
+ // checks here.
+ const string layerPath = path::join(directory.get(), "456");
+
+ Try mkdir = os::mkdir(layerPath);
+ ASSERT_SOME(mkdir);
+
+ JSON::Value manifest = JSON::parse(
+"{"
+" \"parent\": \"\""
+"}").get();
+
+ ASSERT_SOME(
+ os::write(path::join(layerPath, "json"), stringify(manifest)));
+
+ ASSERT_TRUE(imageInfo1.isPending());
+ Future imageInfo2 =
+store.get()->get(mesosImage, COPY_BACKEND);
+
+ Try reference =
+spec::parseImageReference(mesosImage.docker().name());
+
+ ASSERT_SOME(reference);
+
+ slave::docker::Image result;
+ result.mutable_reference()->CopyFrom(reference.get());
+ result.add_layer_ids("456");
+
+ ASSERT_TRUE(imageInfo2.isPending());
+ promise.set(result);
+
+ AWAIT_READY(imageInfo1);
+ AWAIT_READY(imageInfo2);
+
+ EXPECT_EQ(imageInfo1->layers, imageInfo2->layers);
+}
+
+
#ifdef __linux__
class ProvisionerDockerTest
: public MesosTest,
