Repository: mesos Updated Branches: refs/heads/master 559cd7f19 -> 09ab3fce8
Added a benchmark to simulate frameworks declining offers. This benchmark starts a number of slaves and frameworks, then cycles the allocator. On each allocation pass, the frameworks decline all the offers. This leads to increasing numbers of refusal filters in the allocator, and the allocation slows down. After around 200 cycles the slowdown increases out of proportion. Review: https://reviews.apache.org/r/38649 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/09ab3fce Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/09ab3fce Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/09ab3fce Branch: refs/heads/master Commit: 09ab3fce894d13d62f05b9f5458e88724cd88d8c Parents: 559cd7f Author: James Peach <[email protected]> Authored: Thu Dec 10 10:43:18 2015 -0800 Committer: Joris Van Remoortere <[email protected]> Committed: Thu Dec 10 11:36:27 2015 -0800 ---------------------------------------------------------------------- src/tests/hierarchical_allocator_tests.cpp | 155 ++++++++++++++++++++++++ 1 file changed, 155 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/09ab3fce/src/tests/hierarchical_allocator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp index e664866..e239b47 100644 --- a/src/tests/hierarchical_allocator_tests.cpp +++ b/src/tests/hierarchical_allocator_tests.cpp @@ -77,6 +77,44 @@ struct Allocation hashmap<SlaveID, Resources> resources; }; +static Resource +makePortRanges(const ::mesos::Value::Range& bounds, unsigned numRanges) +{ + unsigned numPorts = bounds.end() - bounds.begin(); + unsigned step = numPorts / numRanges; + ::mesos::Value::Ranges ranges; + + ranges.mutable_range()->Reserve(numRanges); + + for (unsigned i = 0; i < numRanges; ++i) { + Value::Range *range = ranges.add_range(); + unsigned start = bounds.begin() + (i * step); + unsigned end = start + 1; + + range->set_begin(start); + range->set_end(end); + } + + Value values; + Resource resource; + + values.set_type(Value::RANGES); + values.mutable_ranges()->CopyFrom(ranges); + resource.set_type(Value::RANGES); + resource.set_role("*"); + resource.set_name("ports"); + resource.mutable_ranges()->CopyFrom(values.ranges()); + + return resource; +} + +static ::mesos::Value::Range makeRange(unsigned begin, unsigned end) +{ + ::mesos::Value::Range range; + range.set_begin(begin); + range.set_end(end); + return range; +} struct Deallocation { @@ -1923,6 +1961,123 @@ TEST_P(HierarchicalAllocator_BENCHMARK_Test, AddAndUpdateSlave) cout << "Updated " << slaveCount << " slaves in " << watch.elapsed() << endl; } + +// This benchmark simulates a number of frameworks that have a fixed amount of +// work to do. Once they have reached their targets, they start declining all +// subsequent offers. +TEST_F(HierarchicalAllocator_BENCHMARK_Test, DeclineOffers) +{ + unsigned frameworkCount = 200; + unsigned slaveCount = 2000; + master::Flags flags; + + FLAGS_v = 5; + __sync_synchronize(); // Ensure 'FLAGS_v' visible in other threads. + + // Choose an interval longer than the time we expect a single cycle to take so + // that we don't back up the process queue. + flags.allocation_interval = Hours(1); + + // Pause the clock because we want to manually drive the allocations. + Clock::pause(); + + // Number of allocations. This is used to determine the termination + // condition. + atomic<size_t> offerCount(0); + + struct OfferedResources { + FrameworkID frameworkId; + SlaveID slaveId; + Resources resources; + }; + + std::vector<OfferedResources> offers; + + auto offerCallback = [&offerCount, &offers]( + const FrameworkID& frameworkId, + const hashmap<SlaveID, Resources>& resources_) + { + for (auto resources : resources_) { + offers.push_back( + OfferedResources{frameworkId, resources.first, resources.second}); + } + + offerCount++; + }; + + vector<SlaveInfo> slaves; + vector<FrameworkInfo> frameworks; + + cout << "Using " << slaveCount << " slaves and " + << frameworkCount << " frameworks" << endl; + + slaves.reserve(slaveCount); + frameworks.reserve(frameworkCount); + + initialize({}, flags, offerCallback); + + for (unsigned i = 0; i < frameworkCount; ++i) { + frameworks.push_back(createFrameworkInfo("*")); + allocator->addFramework(frameworks[i].id(), frameworks[i], {}); + } + + Resources resources = Resources::parse( + "cpus:16;mem:2014;disk:1024;").get(); + + Resources ports = makePortRanges(makeRange(31000, 32000), 16); + + resources += ports; + + for (unsigned i = 0; i < slaveCount; ++i) { + slaves.push_back(createSlaveInfo( + "cpus:24;mem:4096;disk:4096;ports:[31000-32000]")); + + // Add some used resources on each slave. Let's say there are 16 tasks, each + // is allocated 1 cpu and a random port from the port range. + hashmap<FrameworkID, Resources> used; + used[frameworks[i % frameworkCount].id()] = resources; + allocator->addSlave( + slaves[i].id(), slaves[i], None(), slaves[i].resources(), used); + } + + // Wait for all the 'addSlave' operations to be processed. + Clock::settle(); + + // Loop enough times for all the frameworks to get offerred all the resources. + for (unsigned count = 0; count < frameworkCount * 2; ++count) { + // Permanently decline any offered resources. + for (auto offer : offers) { + Filters filters; + + filters.set_refuse_seconds(INT_MAX); + allocator->recoverResources( + offer.frameworkId, offer.slaveId, offer.resources, filters); + } + + // Wait for the declined offers. + Clock::settle(); + offers.clear(); + offerCount = 0; + + { + Stopwatch watch; + + watch.start(); + + // Advance the clock and trigger a background allocation cycle. + Clock::advance(flags.allocation_interval); + Clock::settle(); + + cout << "round " << count + << " allocate took " << watch.elapsed() + << " to make " << offerCount.load() << " offers" + << endl; + } + } + + Clock::resume(); +} + } // namespace tests { } // namespace internal { } // namespace mesos {
