Added test fixture for a hung Docker daemon. The new 'HungDockerTest' class allows test authors to force certain Docker daemon calls to be delayed for a specified duration.
Review: https://reviews.apache.org/r/65751/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8c793a75 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8c793a75 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8c793a75 Branch: refs/heads/master Commit: 8c793a75bb59e59d4a2b8a63afea38b64b100a97 Parents: a20a317 Author: Greg Mann <g...@mesosphere.io> Authored: Fri Feb 23 16:41:48 2018 -0800 Committer: Greg Mann <gregorywm...@gmail.com> Committed: Fri Feb 23 17:44:58 2018 -0800 ---------------------------------------------------------------------- .../docker_containerizer_tests.cpp | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/8c793a75/src/tests/containerizer/docker_containerizer_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp b/src/tests/containerizer/docker_containerizer_tests.cpp index 97335e1..b84a750 100644 --- a/src/tests/containerizer/docker_containerizer_tests.cpp +++ b/src/tests/containerizer/docker_containerizer_tests.cpp @@ -5116,6 +5116,83 @@ TEST_F( ASSERT_FALSE( exists(docker, containerId.get(), ContainerState::RUNNING)); } + + +class HungDockerTest : public DockerContainerizerTest +{ +public: + const string testDockerEnvFile = "test-docker.env"; + const string testDockerBinary = "docker"; + const string testDockerScript = "test-docker.sh"; + string commandsEnv; + string delayEnv; + + virtual slave::Flags CreateSlaveFlags() + { + slave::Flags flags = MesosTest::CreateSlaveFlags(); + + flags.docker = path::join(os::getcwd(), testDockerScript); + + return flags; + } + + void writeEnv() + { + // TODO(greggomann): This write operation is not atomic, which means an + // ill-timed write may cause the shell script to be invoked when this + // file is in an unintended state. We should make this atomic. + Try<Nothing> write = + os::write(testDockerEnvFile, commandsEnv + "\n" + delayEnv); + ASSERT_SOME(write); + } + + void setDelayedCommands(const std::vector<string>& commands) + { + commandsEnv = "DELAYED_COMMANDS=( "; + foreach (const string& command, commands) { + commandsEnv += (command + " "); + } + commandsEnv += ")"; + + writeEnv(); + } + + void setDelay(const int seconds) + { + delayEnv = "DELAY_SECONDS=" + stringify(seconds); + + writeEnv(); + } + + virtual void SetUp() + { + DockerContainerizerTest::SetUp(); + + // Write a wrapper script which allows us to delay Docker commands. + const string dockerScriptText = + "#!/usr/bin/env bash\n" + "source " + stringify(path::join(os::getcwd(), testDockerEnvFile)) + "\n" + "ACTIVE_COMMAND=$3\n" + "for DELAYED_COMMAND in \"${DELAYED_COMMANDS[@]}\"; do\n" + " if [ \"$ACTIVE_COMMAND\" == \"$DELAYED_COMMAND\" ]; then\n" + " sleep $DELAY_SECONDS\n" + " fi\n" + "done\n" + + testDockerBinary + " $@\n"; + + Try<Nothing> write = os::write(testDockerScript, dockerScriptText); + ASSERT_SOME(write); + + Try<Nothing> chmod = os::chmod( + testDockerScript, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + ASSERT_SOME(chmod); + + // Set a very long delay by default to simulate an indefinitely + // hung Docker daemon. + setDelay(999999); + } +}; + } // namespace tests { } // namespace internal { } // namespace mesos {