Updated the balloon tests to test the memory threshold. Review: https://reviews.apache.org/r/14044
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ebba5c10 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ebba5c10 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ebba5c10 Branch: refs/heads/master Commit: ebba5c109d17fac2a18c16467097846f398e7217 Parents: 8d74239 Author: Benjamin Mahler <bmah...@twitter.com> Authored: Thu Aug 29 11:14:04 2013 -0700 Committer: Benjamin Mahler <bmah...@twitter.com> Committed: Wed Oct 2 12:17:52 2013 -0700 ---------------------------------------------------------------------- src/examples/balloon_executor.cpp | 54 ++++++++++++++++--------------- src/examples/balloon_framework.cpp | 52 +++++++++++++++++------------ src/tests/balloon_framework_test.sh | 11 +++++-- src/tests/cgroups_isolator_tests.cpp | 37 ++++++++++++++++++--- src/tests/examples_tests.cpp | 12 ++++--- src/tests/script.cpp | 17 ++++++++-- src/tests/script.hpp | 39 +++++++++++++++++++--- 7 files changed, 159 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/examples/balloon_executor.cpp ---------------------------------------------------------------------- diff --git a/src/examples/balloon_executor.cpp b/src/examples/balloon_executor.cpp index ad1d861..358afaf 100644 --- a/src/examples/balloon_executor.cpp +++ b/src/examples/balloon_executor.cpp @@ -26,51 +26,50 @@ #include <iostream> #include <string> +#include <vector> #include <mesos/executor.hpp> +#include <stout/bytes.hpp> #include <stout/duration.hpp> #include <stout/numify.hpp> #include <stout/os.hpp> using namespace mesos; - -// The amount of memory in MB each balloon step consumes. -const static size_t BALLOON_STEP_MB = 64; +using std::string; +using std::vector; // This function will increase the memory footprint gradually. The parameter -// limit specifies the upper limit (in MB) of the memory footprint. The -// parameter step specifies the step size (in MB). -static void balloon(size_t limit) +// limit specifies the upper limit of the memory footprint. The +// parameter step specifies the step size. +static void balloon(const Bytes& limit, const Bytes& step) { - size_t chunk = BALLOON_STEP_MB * 1024 * 1024; - for (size_t i = 0; i < limit / BALLOON_STEP_MB; i++) { - std::cout << "Increasing memory footprint by " - << BALLOON_STEP_MB << " MB" << std::endl; + for (size_t i = 0; i < limit.bytes() / step.bytes(); i++) { + std::cout << "Increasing memory footprint by " << step << std::endl; // Allocate page-aligned virtual memory. void* buffer = NULL; - if (posix_memalign(&buffer, getpagesize(), chunk) != 0) { + if (posix_memalign(&buffer, getpagesize(), step.bytes()) != 0) { perror("Failed to allocate page-aligned memory, posix_memalign"); abort(); } // We use mlock and memset here to make sure that the memory // actually gets paged in and thus accounted for. - if (mlock(buffer, chunk) != 0) { + if (mlock(buffer, step.bytes()) != 0) { perror("Failed to lock memory, mlock"); abort(); } - if (memset(buffer, 1, chunk) != buffer) { + if (memset(buffer, 1, step.bytes()) != buffer) { perror("Failed to fill memory, memset"); abort(); } // Try not to increase the memory footprint too fast. - os::sleep(Seconds(1)); + os::sleep(Milliseconds(50)); } } @@ -109,18 +108,21 @@ public: driver->sendStatusUpdate(status); - // Get the balloon limit (in MB). - Try<size_t> limit = numify<size_t>(task.data()); + // Get the balloon step and limit. + vector<string> split = strings::split(task.data(), ","); + + Try<Bytes> step = Bytes::parse(split[0]); + assert(step.isSome()); + + Try<Bytes> limit = Bytes::parse(split[1]); assert(limit.isSome()); - size_t balloonLimit = limit.get(); - // Artificially increase the memory usage gradually. The - // balloonLimit specifies the upper limit. The balloonLimit can be - // larger than the amount of memory allocated to this executor. In - // that case, the isolator (e.g. cgroups) should be able to detect - // that and the task should not be able to reach TASK_FINISHED - // state. - balloon(balloonLimit); + // Artificially increase the memory usage gradually. The limit + // can be larger than the amount of memory allocated to this + // executor. In that case, the isolator (e.g. cgroups) should be + // able to detect that and the task should not be able to reach + // TASK_FINISHED state. + balloon(limit.get(), step.get()); std::cout << "Finishing task " << task.task_id().value() << std::endl; @@ -135,7 +137,7 @@ public: std::cout << "Kill task " << taskId.value() << std::endl; } - virtual void frameworkMessage(ExecutorDriver* driver, const std::string& data) + virtual void frameworkMessage(ExecutorDriver* driver, const string& data) { std::cout << "Framework message: " << data << std::endl; } @@ -145,7 +147,7 @@ public: std::cout << "Shutdown" << std::endl; } - virtual void error(ExecutorDriver* driver, const std::string& message) + virtual void error(ExecutorDriver* driver, const string& message) { std::cout << "Error message: " << message << std::endl; } http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/examples/balloon_framework.cpp ---------------------------------------------------------------------- diff --git a/src/examples/balloon_framework.cpp b/src/examples/balloon_framework.cpp index 53859bc..1df5e22 100644 --- a/src/examples/balloon_framework.cpp +++ b/src/examples/balloon_framework.cpp @@ -28,6 +28,8 @@ #include <mesos/scheduler.hpp> +#include <stout/bytes.hpp> +#include <stout/exit.hpp> #include <stout/numify.hpp> #include <stout/os.hpp> #include <stout/stringify.hpp> @@ -39,16 +41,21 @@ using namespace mesos; using namespace mesos::internal; +using std::string; + // The amount of memory in MB the executor itself takes. -const static size_t EXECUTOR_MEMORY_MB = 64; +const static Bytes EXECUTOR_MEMORY = Megabytes(64); class BalloonScheduler : public Scheduler { public: - BalloonScheduler(const ExecutorInfo& _executor, - size_t _balloonLimit) + BalloonScheduler( + const ExecutorInfo& _executor, + const Bytes& _balloonStep, + const Bytes& _balloonLimit) : executor(_executor), + balloonStep(_balloonStep), balloonLimit(_balloonLimit), taskLaunched(false) {} @@ -82,7 +89,7 @@ public: // We just launch one task. if (!taskLaunched) { double mem = getScalarResource(offer, "mem"); - assert(mem > EXECUTOR_MEMORY_MB); + assert(mem > EXECUTOR_MEMORY.megabytes()); std::vector<TaskInfo> tasks; std::cout << "Starting the task" << std::endl; @@ -92,14 +99,15 @@ public: task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offer.slave_id()); task.mutable_executor()->MergeFrom(executor); - task.set_data(stringify<size_t>(balloonLimit)); + task.set_data(stringify(balloonStep) + "," + stringify(balloonLimit)); // Use up all the memory from the offer. Resource* resource; resource = task.add_resources(); resource->set_name("mem"); resource->set_type(Value::SCALAR); - resource->mutable_scalar()->set_value(mem - EXECUTOR_MEMORY_MB); + resource->mutable_scalar()->set_value( + mem - EXECUTOR_MEMORY.megabytes()); tasks.push_back(task); driver->launchTasks(offer.id(), tasks); @@ -161,29 +169,33 @@ public: private: const ExecutorInfo executor; - const size_t balloonLimit; + const Bytes balloonStep; + const Bytes balloonLimit; bool taskLaunched; }; int main(int argc, char** argv) { - if (argc != 3) { - std::cerr << "Usage: " << argv[0] - << " <master> <balloon limit in MB>" << std::endl; - return -1; + if (argc != 4) { + EXIT(1) << "Usage: " << argv[0] + << " <master> <balloon step> <balloon limit>"; + } + + // Parse the balloon step. + Try<Bytes> step = Bytes::parse(argv[2]); + if (step.isError()) { + EXIT(1) << "Balloon memory step is invalid: " << step.error(); } - // Verify the balloon limit. - Try<size_t> limit = numify<size_t>(argv[2]); + // Parse the balloon limit. + Try<Bytes> limit = Bytes::parse(argv[3]); if (limit.isError()) { - std::cerr << "Balloon limit is not a valid number" << std::endl; - return -1; + EXIT(1) << "Balloon memory limit is invalid: " << limit.error(); } - if (limit.get() < EXECUTOR_MEMORY_MB) { - std::cerr << "Please use a balloon limit bigger than " - << EXECUTOR_MEMORY_MB << " MB" << std::endl; + if (limit.get() < EXECUTOR_MEMORY) { + EXIT(1) << "Please use an executor limit smaller than " << EXECUTOR_MEMORY; } // Find this executable's directory to locate executor. @@ -202,9 +214,9 @@ int main(int argc, char** argv) Resource* mem = executor.add_resources(); mem->set_name("mem"); mem->set_type(Value::SCALAR); - mem->mutable_scalar()->set_value(EXECUTOR_MEMORY_MB); + mem->mutable_scalar()->set_value(EXECUTOR_MEMORY.megabytes()); - BalloonScheduler scheduler(executor, limit.get()); + BalloonScheduler scheduler(executor, step.get(), limit.get()); FrameworkInfo framework; framework.set_user(""); // Have Mesos fill in the current user. http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/tests/balloon_framework_test.sh ---------------------------------------------------------------------- diff --git a/src/tests/balloon_framework_test.sh b/src/tests/balloon_framework_test.sh index 8e99456..7229064 100755 --- a/src/tests/balloon_framework_test.sh +++ b/src/tests/balloon_framework_test.sh @@ -3,6 +3,12 @@ # This script runs the balloon framework on a cluster using the cgroups # isolator and checks that the framework returns a status of 1. +if [ $# -ne 3 ]; then + echo "usage: `basename $0` cgroups_oom_buffer balloon_step balloon_limit" + echo "example: `basename $0` 127MB 32MB 1GB" + exit 1 +fi + source ${MESOS_SOURCE_DIR}/support/colors.sh source ${MESOS_SOURCE_DIR}/support/atexit.sh @@ -90,7 +96,8 @@ ${SLAVE} \ --isolation=cgroups \ --cgroups_hierarchy=${TEST_CGROUP_HIERARCHY} \ --cgroups_root=${TEST_CGROUP_ROOT} \ - --resources="cpus:1;mem:96" & + --resources="cpus:1;mem:96" \ + --cgroups_oom_buffer=${1} & SLAVE_PID=${!} echo "${GREEN}Launched slave at ${SLAVE_PID}${NORMAL}" sleep 2 @@ -104,7 +111,7 @@ if [[ ${STATUS} -ne 0 ]]; then fi # The main event! -${BALLOON_FRAMEWORK} localhost:5432 1024 +${BALLOON_FRAMEWORK} localhost:5432 ${2} ${3} STATUS=${?} # Make sure the balloon framework "failed". http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/tests/cgroups_isolator_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/cgroups_isolator_tests.cpp b/src/tests/cgroups_isolator_tests.cpp index 1f5ce76..cdac4a9 100644 --- a/src/tests/cgroups_isolator_tests.cpp +++ b/src/tests/cgroups_isolator_tests.cpp @@ -35,10 +35,39 @@ using namespace mesos::internal::slave; using std::map; -// Run the balloon framework under the cgroups isolator. -TEST_SCRIPT(CgroupsIsolatorTest, - ROOT_CGROUPS_BalloonFramework, - "balloon_framework_test.sh") +// Run the balloon framework under the cgroups isolator with a few +// configurations. + +// No memory buffer on the slave. +// Step with 64MB, limit 1GB. +// This should behave as if there's only a hard threshold. +TEST_SCRIPT_3(CgroupsIsolatorTest, + ROOT_CGROUPS_BalloonFramework_NoBuffer, + "balloon_framework_test.sh", + stringify(Megabytes(0)), + stringify(Megabytes(64)), + stringify(Gigabytes(1))) + + +// 128MB memory buffer on the slave, step with 32 MB, limit 1GB. +// This ensures we'll have time to act on the threshold notification. +TEST_SCRIPT_3(CgroupsIsolatorTest, + ROOT_CGROUPS_BalloonFramework_Threshold, + "balloon_framework_test.sh", + stringify(Megabytes(128)), + stringify(Megabytes(32)), + stringify(Gigabytes(1))) + + +// 1MB memory buffer on the slave, step with 64 MB, limit 1GB. +// This ensures the OOM occurs before we can act on the threshold +// notification. +TEST_SCRIPT_3(CgroupsIsolatorTest, + ROOT_CGROUPS_BalloonFramework_OOM, + "balloon_framework_test.sh", + stringify(Megabytes(1)), + stringify(Megabytes(128)), + stringify(Gigabytes(1))) #define GROW_USAGE(delta, cpuset, usage) \ http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/tests/examples_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/examples_tests.cpp b/src/tests/examples_tests.cpp index 28ff0f3..d7429bb 100644 --- a/src/tests/examples_tests.cpp +++ b/src/tests/examples_tests.cpp @@ -22,14 +22,16 @@ // Run each of the sample frameworks in local mode. -TEST_SCRIPT(ExamplesTest, TestFramework, "test_framework_test.sh") -TEST_SCRIPT(ExamplesTest, NoExecutorFramework, "no_executor_framework_test.sh") +TEST_SCRIPT_0(ExamplesTest, TestFramework, "test_framework_test.sh") +TEST_SCRIPT_0(ExamplesTest, + NoExecutorFramework, + "no_executor_framework_test.sh") #ifdef MESOS_HAS_JAVA -TEST_SCRIPT(ExamplesTest, JavaFramework, "java_framework_test.sh") -TEST_SCRIPT(ExamplesTest, JavaException, "java_exception_test.sh") +TEST_SCRIPT_0(ExamplesTest, JavaFramework, "java_framework_test.sh") +TEST_SCRIPT_0(ExamplesTest, JavaException, "java_exception_test.sh") #endif #ifdef MESOS_HAS_PYTHON -TEST_SCRIPT(ExamplesTest, PythonFramework, "python_framework_test.sh") +TEST_SCRIPT_0(ExamplesTest, PythonFramework, "python_framework_test.sh") #endif http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/tests/script.cpp ---------------------------------------------------------------------- diff --git a/src/tests/script.cpp b/src/tests/script.cpp index e72eea6..04150ba 100644 --- a/src/tests/script.cpp +++ b/src/tests/script.cpp @@ -23,6 +23,7 @@ #include <sys/wait.h> // For wait (and associated macros). #include <string> +#include <vector> #include <stout/check.hpp> #include <stout/os.hpp> @@ -34,13 +35,16 @@ #include "tests/script.hpp" using std::string; +using std::vector; namespace mesos { namespace internal { namespace tests { -void execute(const string& script) +void execute(const string& script, const vector<string>& arguments) { + CHECK_LT(arguments.size(), static_cast<size_t>(ARG_MAX)); + // Create a temporary directory for the test. Try<string> directory = environment->mkdtemp(); @@ -104,8 +108,17 @@ void execute(const string& script) os::setenv("MESOS_WEBUI_DIR", path::join(flags.source_dir, "src", "webui")); os::setenv("MESOS_LAUNCHER_DIR", path::join(flags.build_dir, "src")); + // Construct the argument array. + const char** args = (const char**) new char*[arguments.size() + 1]; + args[0] = path.get().c_str(); + size_t index = 1; + foreach (const string& argument, arguments) { + args[index++] = argument.c_str(); + } + args[arguments.size() + 1] = NULL; + // Now execute the script. - execl(path.get().c_str(), path.get().c_str(), (char*) NULL); + execv(path.get().c_str(), (char* const*) args); std::cerr << "Failed to execute '" << script << "': " << strerror(errno) << std::endl; http://git-wip-us.apache.org/repos/asf/mesos/blob/ebba5c10/src/tests/script.hpp ---------------------------------------------------------------------- diff --git a/src/tests/script.hpp b/src/tests/script.hpp index f0c71e1..980df20 100644 --- a/src/tests/script.hpp +++ b/src/tests/script.hpp @@ -21,12 +21,19 @@ #include <gtest/gtest.h> +#include <string> +#include <vector> + +#include <stout/preprocessor.hpp> + namespace mesos { namespace internal { namespace tests { // Helper used by TEST_SCRIPT to execute the script. -void execute(const std::string& script); +void execute( + const std::string& script, + const std::vector<std::string>& arguments = std::vector<std::string>()); } // namespace tests { } // namespace internal { @@ -37,9 +44,33 @@ void execute(const std::string& script); // script in temporary directory and pipe its output to '/dev/null' // unless the verbose option is specified. The "test" passes if the // script returns 0. -#define TEST_SCRIPT(test_case_name, test_name, script) \ - TEST(test_case_name, test_name) { \ - mesos::internal::tests::execute(script); \ +#define TEST_SCRIPT_0(test_case_name, test_name, script) \ + TEST(test_case_name, test_name) { \ + mesos::internal::tests::execute(script); \ + } + +#define TEST_SCRIPT_1(test_case_name, test_name, script, arg1) \ + TEST(test_case_name, test_name) { \ + std::vector<std::string> arguments; \ + arguments.push_back(arg1); \ + mesos::internal::tests::execute(script, arguments); \ + } + +#define TEST_SCRIPT_2(test_case_name, test_name, script, arg1, arg2) \ + TEST(test_case_name, test_name) { \ + std::vector<std::string> arguments; \ + arguments.push_back(arg1); \ + arguments.push_back(arg2); \ + mesos::internal::tests::execute(script, arguments); \ + } + +#define TEST_SCRIPT_3(test_case_name, test_name, script, arg1, arg2, arg3) \ + TEST(test_case_name, test_name) { \ + std::vector<std::string> arguments; \ + arguments.push_back(arg1); \ + arguments.push_back(arg2); \ + arguments.push_back(arg3); \ + mesos::internal::tests::execute(script, arguments); \ } #endif // __TESTS_SCRIPT_HPP__
