Terminated driver-based executors if kill arrives before launch task. `ExecutorRegisteredMessage` or `RunTaskMessage` may not be delivered to a driver-based executor. Since these messages are not retried, without this patch an executor never starts a task and remains idle, ignoring kill task request. This patch ensures all built-in driver- based executors eventually shut down if kill task arrives before the task has been started.
Review: https://reviews.apache.org/r/64033/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9d8502cc Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9d8502cc Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9d8502cc Branch: refs/heads/1.4.x Commit: 9d8502cc4b916eaaa9c7aaa458d5fef46931a37d Parents: 09aaf33 Author: Alexander Rukletsov <ruklet...@gmail.com> Authored: Fri Dec 22 12:10:35 2017 +0100 Committer: Alexander Rukletsov <al...@apache.org> Committed: Fri Dec 22 12:29:50 2017 +0100 ---------------------------------------------------------------------- src/docker/executor.cpp | 6 ++++++ src/exec/exec.cpp | 11 +++++++++++ src/launcher/executor.cpp | 6 ++++++ 3 files changed, 23 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/9d8502cc/src/docker/executor.cpp ---------------------------------------------------------------------- diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp index 5c430dc..7a50e66 100644 --- a/src/docker/executor.cpp +++ b/src/docker/executor.cpp @@ -359,6 +359,12 @@ private: return; } + // Terminate if a kill task request is received before the task is launched. + // This can happen, for example, if `RunTaskMessage` has not been delivered. + // See MESOS-8297. + CHECK_SOME(run) << "Terminating because kill task message has been" + << " received before the task has been launched"; + // TODO(alexr): If a kill is in progress, consider adjusting // the grace period if a new one is provided. http://git-wip-us.apache.org/repos/asf/mesos/blob/9d8502cc/src/exec/exec.cpp ---------------------------------------------------------------------- diff --git a/src/exec/exec.cpp b/src/exec/exec.cpp index ea0b118..33a460d 100644 --- a/src/exec/exec.cpp +++ b/src/exec/exec.cpp @@ -347,6 +347,17 @@ protected: return; } + // A kill task request is received when the driver is not connected. This + // can happen, for example, if `ExecutorRegisteredMessage` has not been + // delivered. We do not shutdown the driver because there might be other + // still running tasks and the executor might eventually reconnect, e.g., + // after the agent failover. We do not drop ignore the message because the + // actual executor may still want to react, e.g., commit suicide. + if (!connected) { + LOG(WARNING) << "Executor received kill task message for task " << taskId + << " while disconnected from the agent!"; + } + VLOG(1) << "Executor asked to kill task '" << taskId << "'"; Stopwatch stopwatch; http://git-wip-us.apache.org/repos/asf/mesos/blob/9d8502cc/src/launcher/executor.cpp ---------------------------------------------------------------------- diff --git a/src/launcher/executor.cpp b/src/launcher/executor.cpp index e5d5595..b518030 100644 --- a/src/launcher/executor.cpp +++ b/src/launcher/executor.cpp @@ -772,6 +772,12 @@ private: return; } + // Terminate if a kill task request is received before the task is launched. + // This can happen, for example, if `RunTaskMessage` has not been delivered. + // See MESOS-8297. + CHECK(launched) << "Terminating because kill task message has been" + << " received before the task has been launched"; + // If the task is being killed but has not terminated yet and // we receive another kill request. Check if we need to adjust // the remaining grace period.
