Added check tests for command executor. Review: https://reviews.apache.org/r/56213/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/96683388 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/96683388 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/96683388 Branch: refs/heads/master Commit: 966833886efd9ba55b30c5ff225ce5691294e7bf Parents: 9ce5b02 Author: Alexander Rukletsov <[email protected]> Authored: Thu Mar 23 17:11:40 2017 +0100 Committer: Alexander Rukletsov <[email protected]> Committed: Fri Mar 24 00:17:27 2017 +0100 ---------------------------------------------------------------------- src/tests/check_tests.cpp | 824 +++++++++++++++++++++++++++++++++- src/tests/health_check_tests.cpp | 7 +- src/tests/mesos.hpp | 4 + 3 files changed, 832 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/96683388/src/tests/check_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/check_tests.cpp b/src/tests/check_tests.cpp index f035c16..31b7317 100644 --- a/src/tests/check_tests.cpp +++ b/src/tests/check_tests.cpp @@ -14,18 +14,838 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <mesos/mesos.hpp> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include <mesos/v1/mesos.hpp> + +#include <process/clock.hpp> +#include <process/owned.hpp> + +#include <stout/foreach.hpp> +#include <stout/nothing.hpp> +#include <stout/path.hpp> +#include <stout/try.hpp> + +#include <stout/os/getcwd.hpp> #include "checks/checker.hpp" +#include "tests/flags.hpp" +#include "tests/health_check_test_helper.hpp" #include "tests/mesos.hpp" +#include "tests/utils.hpp" + +using mesos::master::detector::MasterDetector; + +using mesos::v1::scheduler::Call; +using mesos::v1::scheduler::Event; +using mesos::v1::scheduler::Mesos; + +using process::Future; +using process::Owned; + +using std::pair; +using std::string; +using std::vector; namespace mesos { namespace internal { namespace tests { -class CheckTest : public MesosTest {}; +// This command fails every other invocation. Assuming `path` does not +// initially exist, for all runs i in Nat0, the following case i % 2 applies: +// +// Case 0: +// - Attempt to remove the nonexistent temporary file. +// - Create the temporary file. +// - Exit with a non-zero status. +// +// Case 1: +// - Remove the temporary file. +// - Exit with a zero status. +#ifndef __WINDOWS__ +#define FLAPPING_CHECK_COMMAND(path) \ + string("rm ") + path + " || (touch " + path + "; exit 1)" +#else +#define FLAPPING_CHECK_COMMAND(path) \ + string("powershell -command ") + \ + "$ri_err = Remove-Item -ErrorAction SilentlyContinue" \ + " \"" + path + "\";" \ + "if (-not $?) {" \ + " Set-Content -Path (\"" + path + "\") -Value ($null);" \ + " exit 1" \ + "}" +#endif // !__WINDOWS__ + + +// This command stalls each invocation except the first one. Assuming `path` +// does not initially exist, for all runs i in Nat0, the following applies: +// +// Case 0: +// - Test whether the nonexistent temporary file exists. +// - Create the temporary file. +// - Exit with a non-zero status. +// +// Cases 1..n: +// - Hang for 1000 seconds. +// - Exit with a zero status. +#ifndef __WINDOWS__ +#define STALLING_CHECK_COMMAND(path) \ + string("(ls ") + path + " && " + SLEEP_COMMAND(1000) + \ + ") || (touch " + path + "; exit 1)" +#else +#define STALLING_CHECK_COMMAND(path) \ + string("powershell -command ") + \ + "if (Test-Path \"" + path + "\") {" + \ + SLEEP_COMMAND(1000) + \ + "} else {" \ + " Set-Content -Path (\"" + path + "\") -Value ($null);" \ + " exit 1" \ + "}" +#endif // !__WINDOWS__ + + +// Tests for checks support in built in executors. Logically the tests +// are elements of the cartesian product `executor-type` x `check-type` +// and are split into groups by `executor-type`: +// * command executor tests, + +class CheckTest : public MesosTest +{ +public: + virtual void acknowledge( + Mesos* mesos, + const v1::FrameworkID& frameworkId, + const v1::TaskStatus& status) + { + Call call; + call.mutable_framework_id()->CopyFrom(frameworkId); + call.set_type(Call::ACKNOWLEDGE); + + Call::Acknowledge* acknowledge = call.mutable_acknowledge(); + acknowledge->mutable_task_id()->CopyFrom(status.task_id()); + acknowledge->mutable_agent_id()->CopyFrom(status.agent_id()); + acknowledge->set_uuid(status.uuid()); + + mesos->send(call); + } + + virtual void launchTask( + Mesos* mesos, + const v1::Offer& offer, + const v1::TaskInfo& task) + { + Call call; + call.mutable_framework_id()->CopyFrom(offer.framework_id()); + call.set_type(Call::ACCEPT); + + Call::Accept* accept = call.mutable_accept(); + accept->add_offer_ids()->CopyFrom(offer.id()); + + v1::Offer::Operation* operation = accept->add_operations(); + operation->set_type(v1::Offer::Operation::LAUNCH); + operation->mutable_launch()->add_task_infos()->CopyFrom(task); + + mesos->send(call); + } + + virtual void launchTaskGroup( + Mesos* mesos, + const v1::Offer& offer, + const v1::ExecutorInfo& executor, + const v1::TaskGroupInfo& taskGroup) + { + Call call; + call.mutable_framework_id()->CopyFrom(offer.framework_id()); + call.set_type(Call::ACCEPT); + + Call::Accept* accept = call.mutable_accept(); + accept->add_offer_ids()->CopyFrom(offer.id()); + + v1::Offer::Operation* operation = accept->add_operations(); + operation->set_type(v1::Offer::Operation::LAUNCH_GROUP); + + v1::Offer::Operation::LaunchGroup* launchGroup = + operation->mutable_launch_group(); + + launchGroup->mutable_executor()->CopyFrom(executor); + launchGroup->mutable_task_group()->CopyFrom(taskGroup); + + mesos->send(call); + } + + virtual void reconcile( + Mesos* mesos, + const v1::FrameworkID& frameworkId, + const vector<pair<const v1::TaskID, const v1::AgentID>>& tasks) + { + Call call; + call.mutable_framework_id()->CopyFrom(frameworkId); + call.set_type(Call::RECONCILE); + + call.mutable_reconcile(); + + foreach (const auto& task, tasks) { + Call::Reconcile::Task* reconcile = + call.mutable_reconcile()->add_tasks(); + reconcile->mutable_task_id()->CopyFrom(task.first); + reconcile->mutable_agent_id()->CopyFrom(task.second); + } + + mesos->send(call); + } + + virtual void subscribe( + Mesos* mesos, + const v1::FrameworkInfo& framework) + { + Call call; + call.set_type(Call::SUBSCRIBE); + Call::Subscribe* subscribe = call.mutable_subscribe(); + subscribe->mutable_framework_info()->CopyFrom(framework); + + mesos->send(call); + } +}; + + +// These are check tests with the command executor. +class CommandExecutorCheckTest : public CheckTest {}; + +// Verifies that a command check is supported by the command executor, +// its status is delivered in a task status update, and the last known +// status can be obtained during explicit and implicit reconciliation. +// Additionally ensures that the specified environment of the command +// check is honored. +TEST_F_TEMP_DISABLED_ON_WINDOWS( + CommandExecutorCheckTest, + CommandCheckDeliveredAndReconciled) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + Owned<MasterDetector> detector = master.get()->createDetector(); + Try<Owned<cluster::Slave>> agent = StartSlave(detector.get()); + ASSERT_SOME(agent); + + v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO; + + auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); + + Future<Nothing> connected; + EXPECT_CALL(*scheduler, connected(_)) + .WillOnce(FutureSatisfy(&connected)); + + v1::scheduler::TestMesos mesos( + master.get()->pid, + ContentType::PROTOBUF, + scheduler); + + AWAIT_READY(connected); + + Future<v1::scheduler::Event::Subscribed> subscribed; + EXPECT_CALL(*scheduler, subscribed(_, _)) + .WillOnce(FutureArg<1>(&subscribed)); + + Future<v1::scheduler::Event::Offers> offers; + EXPECT_CALL(*scheduler, offers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + EXPECT_CALL(*scheduler, heartbeat(_)) + .WillRepeatedly(Return()); // Ignore heartbeats. + + subscribe(&mesos, frameworkInfo); + + AWAIT_READY(subscribed); + + v1::FrameworkID frameworkId(subscribed->framework_id()); + + AWAIT_READY(offers); + EXPECT_NE(0, offers->offers().size()); + const v1::Offer& offer = offers->offers(0); + const v1::AgentID agentId = offer.agent_id(); + + Future<Event::Update> updateTaskRunning; + Future<Event::Update> updateCheckResult; + Future<Event::Update> updateExplicitReconciliation; + Future<Event::Update> updateImplicitReconciliation; + + EXPECT_CALL(*scheduler, update(_, _)) + .WillOnce(FutureArg<1>(&updateTaskRunning)) + .WillOnce(FutureArg<1>(&updateCheckResult)) + .WillOnce(FutureArg<1>(&updateExplicitReconciliation)) + .WillOnce(FutureArg<1>(&updateImplicitReconciliation)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + v1::Resources resources = + v1::Resources::parse("cpus:0.1;mem:32;disk:32").get(); + + v1::TaskInfo taskInfo = + v1::createTask(agentId, resources, SLEEP_COMMAND(10000)); + + v1::CheckInfo* checkInfo = taskInfo.mutable_check(); + checkInfo->set_type(v1::CheckInfo::COMMAND); + checkInfo->set_delay_seconds(0); + checkInfo->set_interval_seconds(0); + + v1::CommandInfo* checkCommand = + checkInfo->mutable_command()->mutable_command(); + checkCommand->set_value("exit $STATUS"); + + v1::Environment::Variable* variable = + checkCommand->mutable_environment()->add_variables(); + variable->set_name("STATUS"); + variable->set_value("1"); + + launchTask(&mesos, offer, taskInfo); + + AWAIT_READY(updateTaskRunning); + const v1::TaskStatus& taskRunning = updateTaskRunning->status(); + + ASSERT_EQ(TASK_RUNNING, taskRunning.state()); + EXPECT_EQ(taskInfo.task_id(), taskRunning.task_id()); + EXPECT_TRUE(taskRunning.has_check_status()); + EXPECT_TRUE(taskRunning.check_status().has_command()); + EXPECT_FALSE(taskRunning.check_status().command().has_exit_code()); + + acknowledge(&mesos, frameworkId, taskRunning); + + AWAIT_READY(updateCheckResult); + const v1::TaskStatus& checkResult = updateCheckResult->status(); + + ASSERT_EQ(TASK_RUNNING, checkResult.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResult.reason()); + EXPECT_EQ(taskInfo.task_id(), checkResult.task_id()); + EXPECT_TRUE(checkResult.has_check_status()); + EXPECT_TRUE(checkResult.check_status().command().has_exit_code()); + EXPECT_EQ(1, checkResult.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, checkResult); + + // Trigger explicit reconciliation. + reconcile( + &mesos, + frameworkId, + {std::make_pair(checkResult.task_id(), checkResult.agent_id())}); + + AWAIT_READY(updateExplicitReconciliation); + const v1::TaskStatus& explicitReconciliation = + updateExplicitReconciliation->status(); + + ASSERT_EQ(TASK_RUNNING, explicitReconciliation.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_RECONCILIATION, + explicitReconciliation.reason()); + EXPECT_EQ(taskInfo.task_id(), explicitReconciliation.task_id()); + EXPECT_TRUE(explicitReconciliation.has_check_status()); + EXPECT_TRUE(explicitReconciliation.check_status().command().has_exit_code()); + EXPECT_EQ(1, explicitReconciliation.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, explicitReconciliation); + + // Trigger implicit reconciliation. + reconcile(&mesos, frameworkId, {}); + + AWAIT_READY(updateImplicitReconciliation); + const v1::TaskStatus& implicitReconciliation = + updateImplicitReconciliation->status(); + + ASSERT_EQ(TASK_RUNNING, implicitReconciliation.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_RECONCILIATION, + implicitReconciliation.reason()); + EXPECT_EQ(taskInfo.task_id(), implicitReconciliation.task_id()); + EXPECT_TRUE(implicitReconciliation.has_check_status()); + EXPECT_TRUE(implicitReconciliation.check_status().command().has_exit_code()); + EXPECT_EQ(1, implicitReconciliation.check_status().command().exit_code()); +} + + +// Verifies that a command check's status changes are delivered. +// +// TODO(alexr): When check mocking is available, ensure that *only* +// status changes are delivered. +TEST_F(CommandExecutorCheckTest, CommandCheckStatusChange) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + Owned<MasterDetector> detector = master.get()->createDetector(); + Try<Owned<cluster::Slave>> agent = StartSlave(detector.get()); + ASSERT_SOME(agent); + + v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO; + + auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); + + Future<Nothing> connected; + EXPECT_CALL(*scheduler, connected(_)) + .WillOnce(FutureSatisfy(&connected)); + + v1::scheduler::TestMesos mesos( + master.get()->pid, + ContentType::PROTOBUF, + scheduler); + + AWAIT_READY(connected); + + Future<v1::scheduler::Event::Subscribed> subscribed; + EXPECT_CALL(*scheduler, subscribed(_, _)) + .WillOnce(FutureArg<1>(&subscribed)); + + Future<v1::scheduler::Event::Offers> offers; + EXPECT_CALL(*scheduler, offers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + EXPECT_CALL(*scheduler, heartbeat(_)) + .WillRepeatedly(Return()); // Ignore heartbeats. + + subscribe(&mesos, frameworkInfo); + + AWAIT_READY(subscribed); + + v1::FrameworkID frameworkId(subscribed->framework_id()); + + AWAIT_READY(offers); + EXPECT_NE(0, offers->offers().size()); + const v1::Offer& offer = offers->offers(0); + const v1::AgentID agentId = offer.agent_id(); + + Future<Event::Update> updateTaskRunning; + Future<Event::Update> updateCheckResult; + Future<Event::Update> updateCheckResultChanged; + Future<Event::Update> updateCheckResultBack; + + EXPECT_CALL(*scheduler, update(_, _)) + .WillOnce(FutureArg<1>(&updateTaskRunning)) + .WillOnce(FutureArg<1>(&updateCheckResult)) + .WillOnce(FutureArg<1>(&updateCheckResultChanged)) + .WillOnce(FutureArg<1>(&updateCheckResultBack)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + v1::Resources resources = + v1::Resources::parse("cpus:0.1;mem:32;disk:32").get(); + + v1::TaskInfo taskInfo = + v1::createTask(agentId, resources, SLEEP_COMMAND(10000)); + + v1::CheckInfo* checkInfo = taskInfo.mutable_check(); + checkInfo->set_type(v1::CheckInfo::COMMAND); + checkInfo->set_delay_seconds(0); + checkInfo->set_interval_seconds(0); + checkInfo->mutable_command()->mutable_command()->set_value( + FLAPPING_CHECK_COMMAND(path::join(os::getcwd(), "XXXXXX"))); + + launchTask(&mesos, offer, taskInfo); + + AWAIT_READY(updateTaskRunning); + ASSERT_EQ(TASK_RUNNING, updateTaskRunning->status().state()); + EXPECT_EQ(taskInfo.task_id(), updateTaskRunning->status().task_id()); + + acknowledge(&mesos, frameworkId, updateTaskRunning->status()); + AWAIT_READY(updateCheckResult); + const v1::TaskStatus& checkResult = updateCheckResult->status(); + + ASSERT_EQ(TASK_RUNNING, checkResult.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResult.reason()); + EXPECT_TRUE(checkResult.check_status().command().has_exit_code()); + EXPECT_EQ(1, checkResult.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, checkResult); + + AWAIT_READY(updateCheckResultChanged); + const v1::TaskStatus& checkResultChanged = updateCheckResultChanged->status(); + + ASSERT_EQ(TASK_RUNNING, checkResultChanged.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResultChanged.reason()); + EXPECT_TRUE(checkResultChanged.check_status().command().has_exit_code()); + EXPECT_EQ(0, checkResultChanged.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, checkResultChanged); + + AWAIT_READY(updateCheckResultBack); + const v1::TaskStatus& checkResultBack = updateCheckResultBack->status(); + + ASSERT_EQ(TASK_RUNNING, checkResultBack.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResultBack.reason()); + EXPECT_TRUE(checkResultBack.check_status().command().has_exit_code()); + EXPECT_EQ(1, checkResultBack.check_status().command().exit_code()); +} + + +// Verifies that when a command check times out after a successful check, +// an empty check status update is delivered. +TEST_F(CommandExecutorCheckTest, CommandCheckTimeout) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + Owned<MasterDetector> detector = master.get()->createDetector(); + Try<Owned<cluster::Slave>> agent = StartSlave(detector.get()); + ASSERT_SOME(agent); + + v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO; + + auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); + + Future<Nothing> connected; + EXPECT_CALL(*scheduler, connected(_)) + .WillOnce(FutureSatisfy(&connected)); + + v1::scheduler::TestMesos mesos( + master.get()->pid, + ContentType::PROTOBUF, + scheduler); + + AWAIT_READY(connected); + + Future<v1::scheduler::Event::Subscribed> subscribed; + EXPECT_CALL(*scheduler, subscribed(_, _)) + .WillOnce(FutureArg<1>(&subscribed)); + + Future<v1::scheduler::Event::Offers> offers; + EXPECT_CALL(*scheduler, offers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + EXPECT_CALL(*scheduler, heartbeat(_)) + .WillRepeatedly(Return()); // Ignore heartbeats. + + subscribe(&mesos, frameworkInfo); + + AWAIT_READY(subscribed); + + v1::FrameworkID frameworkId(subscribed->framework_id()); + + AWAIT_READY(offers); + EXPECT_NE(0, offers->offers().size()); + const v1::Offer& offer = offers->offers(0); + const v1::AgentID agentId = offer.agent_id(); + + Future<Event::Update> updateTaskRunning; + Future<Event::Update> updateCheckResult; + Future<Event::Update> updateCheckResultTimeout; + + EXPECT_CALL(*scheduler, update(_, _)) + .WillOnce(FutureArg<1>(&updateTaskRunning)) + .WillOnce(FutureArg<1>(&updateCheckResult)) + .WillOnce(FutureArg<1>(&updateCheckResultTimeout)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + v1::Resources resources = + v1::Resources::parse("cpus:0.1;mem:32;disk:32").get(); + + v1::TaskInfo taskInfo = + v1::createTask(agentId, resources, SLEEP_COMMAND(10000)); + + v1::CheckInfo* checkInfo = taskInfo.mutable_check(); + checkInfo->set_type(v1::CheckInfo::COMMAND); + checkInfo->set_delay_seconds(0); + checkInfo->set_interval_seconds(0); + checkInfo->set_timeout_seconds(1); + checkInfo->mutable_command()->mutable_command()->set_value( + STALLING_CHECK_COMMAND(path::join(os::getcwd(), "XXXXXX"))); + + launchTask(&mesos, offer, taskInfo); + + AWAIT_READY(updateTaskRunning); + ASSERT_EQ(TASK_RUNNING, updateTaskRunning->status().state()); + EXPECT_EQ(taskInfo.task_id(), updateTaskRunning->status().task_id()); + + acknowledge(&mesos, frameworkId, updateTaskRunning->status()); + + AWAIT_READY(updateCheckResult); + const v1::TaskStatus& checkResult = updateCheckResult->status(); + + ASSERT_EQ(TASK_RUNNING, checkResult.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResult.reason()); + EXPECT_TRUE(checkResult.check_status().command().has_exit_code()); + EXPECT_EQ(1, checkResult.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, checkResult); + + AWAIT_READY(updateCheckResultTimeout); + const v1::TaskStatus& checkResultTimeout = updateCheckResultTimeout->status(); + + ASSERT_EQ(TASK_RUNNING, checkResultTimeout.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResultTimeout.reason()); + EXPECT_FALSE(checkResultTimeout.check_status().command().has_exit_code()); +} + + +// Verifies that when both command check and health check are specified, +// health and check updates include both statuses. Also verifies that +// both statuses are included upon reconciliation. +TEST_F(CommandExecutorCheckTest, CommandCheckAndHealthCheckNoShadowing) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + Owned<MasterDetector> detector = master.get()->createDetector(); + Try<Owned<cluster::Slave>> agent = StartSlave(detector.get()); + ASSERT_SOME(agent); + + v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO; + + auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); + + Future<Nothing> connected; + EXPECT_CALL(*scheduler, connected(_)) + .WillOnce(FutureSatisfy(&connected)); + + v1::scheduler::TestMesos mesos( + master.get()->pid, + ContentType::PROTOBUF, + scheduler); + + AWAIT_READY(connected); + + Future<v1::scheduler::Event::Subscribed> subscribed; + EXPECT_CALL(*scheduler, subscribed(_, _)) + .WillOnce(FutureArg<1>(&subscribed)); + + Future<v1::scheduler::Event::Offers> offers; + EXPECT_CALL(*scheduler, offers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + EXPECT_CALL(*scheduler, heartbeat(_)) + .WillRepeatedly(Return()); // Ignore heartbeats. + + subscribe(&mesos, frameworkInfo); + + AWAIT_READY(subscribed); + + v1::FrameworkID frameworkId(subscribed->framework_id()); + + AWAIT_READY(offers); + EXPECT_NE(0, offers->offers().size()); + const v1::Offer& offer = offers->offers(0); + const v1::AgentID agentId = offer.agent_id(); + + Future<Event::Update> updateTaskRunning; + Future<Event::Update> updateCheckResult; + Future<Event::Update> updateHealthResult; + Future<Event::Update> updateImplicitReconciliation; + + EXPECT_CALL(*scheduler, update(_, _)) + .WillOnce(FutureArg<1>(&updateTaskRunning)) + .WillOnce(FutureArg<1>(&updateCheckResult)) + .WillOnce(FutureArg<1>(&updateHealthResult)) + .WillOnce(FutureArg<1>(&updateImplicitReconciliation)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + v1::Resources resources = + v1::Resources::parse("cpus:0.1;mem:32;disk:32").get(); + + v1::TaskInfo taskInfo = + v1::createTask(agentId, resources, SLEEP_COMMAND(10000)); + + v1::CheckInfo* checkInfo = taskInfo.mutable_check(); + checkInfo->set_type(v1::CheckInfo::COMMAND); + checkInfo->set_delay_seconds(0); + checkInfo->set_interval_seconds(0); + checkInfo->mutable_command()->mutable_command()->set_value("exit 1"); + + // Delay health check for 1s to ensure health update comes after check update. + // + // TODO(alexr): This can lead to flakiness on busy agents. A more robust + // approach could be setting the grace period to MAX_INT, and make the + // health check pass iff a file created by the check exists. Alternatively, + // we can relax the expectation that the check update is delivered first. + v1::HealthCheck* healthCheckInfo = taskInfo.mutable_health_check(); + healthCheckInfo->set_type(v1::HealthCheck::COMMAND); + healthCheckInfo->set_delay_seconds(1); + healthCheckInfo->set_interval_seconds(0); + healthCheckInfo->mutable_command()->set_value("exit 0"); + + launchTask(&mesos, offer, taskInfo); + + AWAIT_READY(updateTaskRunning); + ASSERT_EQ(TASK_RUNNING, updateTaskRunning->status().state()); + EXPECT_EQ(taskInfo.task_id(), updateTaskRunning->status().task_id()); + + acknowledge(&mesos, frameworkId, updateTaskRunning->status()); + + AWAIT_READY(updateCheckResult); + const v1::TaskStatus& checkResult = updateCheckResult->status(); + + ASSERT_EQ(TASK_RUNNING, checkResult.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResult.reason()); + EXPECT_EQ(taskInfo.task_id(), checkResult.task_id()); + EXPECT_FALSE(checkResult.has_healthy()); + EXPECT_TRUE(checkResult.has_check_status()); + EXPECT_TRUE(checkResult.check_status().command().has_exit_code()); + EXPECT_EQ(1, checkResult.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, checkResult); + + AWAIT_READY(updateHealthResult); + const v1::TaskStatus& healthResult = updateHealthResult->status(); + + ASSERT_EQ(TASK_RUNNING, healthResult.state()); + EXPECT_EQ(taskInfo.task_id(), healthResult.task_id()); + EXPECT_TRUE(healthResult.has_healthy()); + EXPECT_TRUE(healthResult.healthy()); + EXPECT_TRUE(healthResult.has_check_status()); + EXPECT_TRUE(healthResult.check_status().command().has_exit_code()); + EXPECT_EQ(1, healthResult.check_status().command().exit_code()); + + acknowledge(&mesos, frameworkId, healthResult); + + // Trigger implicit reconciliation. + reconcile(&mesos, frameworkId, {}); + + AWAIT_READY(updateImplicitReconciliation); + const v1::TaskStatus& implicitReconciliation = + updateImplicitReconciliation->status(); + + ASSERT_EQ(TASK_RUNNING, implicitReconciliation.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_RECONCILIATION, + implicitReconciliation.reason()); + EXPECT_EQ(taskInfo.task_id(), implicitReconciliation.task_id()); + EXPECT_TRUE(implicitReconciliation.has_healthy()); + EXPECT_TRUE(implicitReconciliation.healthy()); + EXPECT_TRUE(implicitReconciliation.has_check_status()); + EXPECT_TRUE(implicitReconciliation.check_status().command().has_exit_code()); + EXPECT_EQ(1, implicitReconciliation.check_status().command().exit_code()); +} + + +// Verifies that an HTTP check is supported by the command executor and +// its status is delivered in a task status update. +// +// TODO(josephw): Enable this. Mesos builds its own `curl.exe`, since it +// can't rely on a package manager to get it. We need to make this test use +// that executable. +TEST_F_TEMP_DISABLED_ON_WINDOWS(CommandExecutorCheckTest, HTTPCheckDelivered) +{ + Try<Owned<cluster::Master>> master = StartMaster(); + ASSERT_SOME(master); + + Owned<MasterDetector> detector = master.get()->createDetector(); + Try<Owned<cluster::Slave>> agent = StartSlave(detector.get()); + ASSERT_SOME(agent); + + v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO; + + auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); + + Future<Nothing> connected; + EXPECT_CALL(*scheduler, connected(_)) + .WillOnce(FutureSatisfy(&connected)); + + v1::scheduler::TestMesos mesos( + master.get()->pid, + ContentType::PROTOBUF, + scheduler); + + AWAIT_READY(connected); + + Future<v1::scheduler::Event::Subscribed> subscribed; + EXPECT_CALL(*scheduler, subscribed(_, _)) + .WillOnce(FutureArg<1>(&subscribed)); + + Future<v1::scheduler::Event::Offers> offers; + EXPECT_CALL(*scheduler, offers(_, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + EXPECT_CALL(*scheduler, heartbeat(_)) + .WillRepeatedly(Return()); // Ignore heartbeats. + + subscribe(&mesos, frameworkInfo); + + AWAIT_READY(subscribed); + + v1::FrameworkID frameworkId(subscribed->framework_id()); + + AWAIT_READY(offers); + EXPECT_NE(0, offers->offers().size()); + const v1::Offer& offer = offers->offers(0); + const v1::AgentID agentId = offer.agent_id(); + + Future<v1::scheduler::Event::Update> updateTaskRunning; + Future<v1::scheduler::Event::Update> updateCheckResult; + EXPECT_CALL(*scheduler, update(_, _)) + .WillOnce(FutureArg<1>(&updateTaskRunning)) + .WillOnce(FutureArg<1>(&updateCheckResult)) + .WillRepeatedly(Return()); // Ignore subsequent updates. + + const uint16_t testPort = getFreePort().get(); + + // Use `test-helper` to launch a simple HTTP + // server to respond to HTTP checks. + const string command = strings::format( + "%s %s --ip=127.0.0.1 --port=%u", + getTestHelperPath("test-helper"), + HealthCheckTestHelper::NAME, + testPort).get(); + + v1::Resources resources = + v1::Resources::parse("cpus:0.1;mem:32;disk:32").get(); + + v1::TaskInfo taskInfo = v1::createTask(agentId, resources, command); + + v1::CheckInfo* checkInfo = taskInfo.mutable_check(); + checkInfo->set_type(v1::CheckInfo::HTTP); + checkInfo->mutable_http()->set_port(testPort); + checkInfo->mutable_http()->set_path("/help"); + checkInfo->set_delay_seconds(0); + checkInfo->set_interval_seconds(0); + + launchTask(&mesos, offer, taskInfo); + + AWAIT_READY(updateTaskRunning); + const v1::TaskStatus& taskRunning = updateTaskRunning->status(); + + ASSERT_EQ(TASK_RUNNING, taskRunning.state()); + EXPECT_EQ(taskInfo.task_id(), taskRunning.task_id()); + EXPECT_TRUE(taskRunning.has_check_status()); + EXPECT_TRUE(taskRunning.check_status().has_http()); + EXPECT_FALSE(taskRunning.check_status().http().has_status_code()); + + acknowledge(&mesos, frameworkId, taskRunning); + + AWAIT_READY(updateCheckResult); + const v1::TaskStatus& checkResult = updateCheckResult->status(); + + ASSERT_EQ(TASK_RUNNING, checkResult.state()); + ASSERT_EQ( + v1::TaskStatus::REASON_TASK_CHECK_STATUS_UPDATED, + checkResult.reason()); + EXPECT_EQ(taskInfo.task_id(), checkResult.task_id()); + EXPECT_TRUE(checkResult.has_check_status()); + EXPECT_TRUE(checkResult.check_status().http().has_status_code()); + EXPECT_EQ(200, checkResult.check_status().http().status_code()); +} + + +// These are protobuf validation tests. +// +// TODO(alexr): Move these tests once validation code is moved closer to +// protobuf definitions. // This tests ensures `CheckInfo` protobuf is validated correctly. TEST_F(CheckTest, CheckInfoValidation) http://git-wip-us.apache.org/repos/asf/mesos/blob/96683388/src/tests/health_check_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/health_check_tests.cpp b/src/tests/health_check_tests.cpp index fc1c828..a5b35cf 100644 --- a/src/tests/health_check_tests.cpp +++ b/src/tests/health_check_tests.cpp @@ -1245,6 +1245,10 @@ TEST_F(HealthCheckTest, HealthyToUnhealthyTransitionWithinGracePeriod) // Tests a healthy non-contained task via HTTP. +// +// TODO(josephw): Enable this. Mesos builds its own `curl.exe`, since it +// can't rely on a package manager to get it. We need to make this test use +// that executable. TEST_F_TEMP_DISABLED_ON_WINDOWS(HealthCheckTest, HealthyTaskViaHTTP) { master::Flags masterFlags = CreateMasterFlags(); @@ -1327,7 +1331,8 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(HealthCheckTest, HealthyTaskViaHTTP) // with the difference being the health check type is not set. // // TODO(haosdent): Remove this after the deprecation cycle which starts in 2.0. -// TODO(hausdorff): Enable this. Mesos builds its own `curl.exe`, since it +// +// TODO(josephw): Enable this. Mesos builds its own `curl.exe`, since it // can't rely on a package manager to get it. We need to make this test use // that executable. TEST_F_TEMP_DISABLED_ON_WINDOWS(HealthCheckTest, HealthyTaskViaHTTPWithoutType) http://git-wip-us.apache.org/repos/asf/mesos/blob/96683388/src/tests/mesos.hpp ---------------------------------------------------------------------- diff --git a/src/tests/mesos.hpp b/src/tests/mesos.hpp index f39e243..e4a8a42 100644 --- a/src/tests/mesos.hpp +++ b/src/tests/mesos.hpp @@ -361,13 +361,17 @@ using mesos::v1::TASK_GONE_BY_OPERATOR; using mesos::v1::TASK_UNKNOWN; using mesos::v1::AgentID; +using mesos::v1::CheckInfo; +using mesos::v1::CommandInfo; using mesos::v1::ContainerID; using mesos::v1::ContainerStatus; +using mesos::v1::Environment; using mesos::v1::ExecutorID; using mesos::v1::ExecutorInfo; using mesos::v1::Filters; using mesos::v1::FrameworkID; using mesos::v1::FrameworkInfo; +using mesos::v1::HealthCheck; using mesos::v1::InverseOffer; using mesos::v1::MachineID; using mesos::v1::Metric;
