Repository: mesos Updated Branches: refs/heads/master 04b214de8 -> 1794d4e0f
Added a performance benchmark for hierarchical allocator addSlave. Review: https://reviews.apache.org/r/35631 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1794d4e0 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1794d4e0 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1794d4e0 Branch: refs/heads/master Commit: 1794d4e0f644a0f6177caacdb5bd516f0348f07a Parents: 04b214d Author: Jie Yu <[email protected]> Authored: Thu Jun 18 15:48:20 2015 -0700 Committer: Jie Yu <[email protected]> Committed: Thu Jun 18 19:19:12 2015 -0700 ---------------------------------------------------------------------- src/tests/hierarchical_allocator_tests.cpp | 103 ++++++++++++++++++++++-- src/tests/registrar_tests.cpp | 5 +- 2 files changed, 99 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/1794d4e0/src/tests/hierarchical_allocator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp index 85bb29e..d2f65d7 100644 --- a/src/tests/hierarchical_allocator_tests.cpp +++ b/src/tests/hierarchical_allocator_tests.cpp @@ -18,6 +18,8 @@ #include <gmock/gmock.h> +#include <atomic> +#include <iostream> #include <string> #include <queue> #include <vector> @@ -30,9 +32,12 @@ #include <process/shared.hpp> #include <process/queue.hpp> +#include <stout/duration.hpp> #include <stout/gtest.hpp> #include <stout/hashmap.hpp> #include <stout/hashset.hpp> +#include <stout/os.hpp> +#include <stout/stopwatch.hpp> #include <stout/utils.hpp> #include "master/constants.hpp" @@ -53,10 +58,15 @@ using process::Clock; using process::Future; using process::Shared; +using std::atomic; +using std::cout; +using std::endl; using std::queue; using std::string; using std::vector; +using testing::WithParamInterface; + namespace mesos { namespace internal { namespace tests { @@ -69,22 +79,25 @@ struct Allocation }; -class HierarchicalAllocatorTest : public ::testing::Test +class HierarchicalAllocatorTestBase : public ::testing::Test { protected: - HierarchicalAllocatorTest() + HierarchicalAllocatorTestBase() : allocator(createAllocator<HierarchicalDRFAllocator>()), nextSlaveId(1), nextFrameworkId(1) {} - ~HierarchicalAllocatorTest() + ~HierarchicalAllocatorTestBase() { delete allocator; } void initialize( const vector<string>& _roles, - const master::Flags& _flags = master::Flags()) + const master::Flags& _flags = master::Flags(), + const Option<lambda::function< + void(const FrameworkID&, + const hashmap<SlaveID, Resources>&)>>& offerCallback = None()) { flags = _flags; @@ -98,10 +111,17 @@ protected: roles[role] = info; } - allocator->initialize( - flags.allocation_interval, - lambda::bind(&put, &queue, lambda::_1, lambda::_2), - roles); + if (offerCallback.isSome()) { + allocator->initialize( + flags.allocation_interval, + offerCallback.get(), + roles); + } else { + allocator->initialize( + flags.allocation_interval, + lambda::bind(&put, &queue, lambda::_1, lambda::_2), + roles); + } } SlaveInfo createSlaveInfo(const string& resources) @@ -166,6 +186,9 @@ private: }; +class HierarchicalAllocatorTest : public HierarchicalAllocatorTestBase {}; + + // TODO(bmahler): These tests were transformed directly from // integration tests into unit tests. However, these tests // should be simplified even further to each test a single @@ -948,6 +971,70 @@ TEST_F(HierarchicalAllocatorTest, Whitelist) EXPECT_EQ(slave.resources(), Resources::sum(allocation.get().resources)); } + +class HierarchicalAllocator_BENCHMARK_Test + : public HierarchicalAllocatorTestBase, + public WithParamInterface<size_t> +{}; + + +// The Hierarchical Allocator benchmark tests are parameterized +// by the number of slaves. +INSTANTIATE_TEST_CASE_P( + SlaveCount, + HierarchicalAllocator_BENCHMARK_Test, + ::testing::Values(1000U, 5000U, 10000U, 20000U, 30000U, 50000U)); + + +TEST_P(HierarchicalAllocator_BENCHMARK_Test, AddSlave) +{ + Clock::pause(); + + // How many 'addSlave' that have been processed. This is used to + // determine the termination condition. + atomic<size_t> finished(0); + + auto offerCallback = [&finished]( + const FrameworkID& frameworkId, + const hashmap<SlaveID, Resources>& resources) { + finished++; + }; + + initialize({}, master::Flags(), offerCallback); + + FrameworkInfo framework = createFrameworkInfo("*"); + allocator->addFramework(framework.id(), framework, {}); + + size_t slaveCount = GetParam(); + + // Create slaves. + vector<SlaveInfo> slaves; + for (size_t i = 0; i < slaveCount; i++) { + slaves.push_back(createSlaveInfo( + "cpus:2;mem:1024;disk:4096;ports:[31000-32000]")); + } + + // Used resources on each slave. + hashmap<FrameworkID, Resources> used; + used[framework.id()] = Resources::parse( + "cpus:1;mem:128;disk:1024;" + "ports:[31126-31510,31512-31623,31810-31852,31854-31964]").get(); + + Stopwatch watch; + watch.start(); + + foreach (const SlaveInfo& slave, slaves) { + allocator->addSlave(slave.id(), slave, slave.resources(), used); + } + + // Wait for all the 'addSlave' to be processed. + while (finished.load() != slaveCount) { + os::sleep(Milliseconds(10)); + } + + cout << "Added " << slaveCount << " slaves in " << watch.elapsed() << endl; +} + } // namespace tests { } // namespace internal { } // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/1794d4e0/src/tests/registrar_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/registrar_tests.cpp b/src/tests/registrar_tests.cpp index ebcce73..97d0b68 100644 --- a/src/tests/registrar_tests.cpp +++ b/src/tests/registrar_tests.cpp @@ -17,6 +17,7 @@ */ #include <algorithm> +#include <iostream> #include <map> #include <set> #include <string> @@ -60,6 +61,8 @@ using namespace process; using mesos::internal::log::Log; using mesos::internal::log::Replica; +using std::cout; +using std::endl; using std::map; using std::set; using std::string; @@ -525,7 +528,7 @@ TEST_P(Registrar_BENCHMARK_Test, performance) result = registrar2.apply(Owned<Operation>(new RemoveSlave(info))); } AWAIT_READY_FOR(result, Minutes(5)); - LOG(INFO) << "Removed " << slaveCount << " slaves in " << watch.elapsed(); + cout << "Removed " << slaveCount << " slaves in " << watch.elapsed() << endl; } } // namespace tests {
