With jobs launched using srun directly which end abnormally, there will
be a step-killed-message(slurmd[cn123]: *** 1234.0 KILLED AT ... WITH
SIGNAL 9 ***) from each node. And/or there will be a
task-exit-message(srun: error: task[0-1]: Terminated) for each node. For
large scale jobs, these messages become tedious and the other error
messages will be buried. The attached two patches(for slurm-2.5.1)
introduce two environment variables to control the output of such
messages:
SLURM_STEP_KILLED_MSG_NODE_ID: if set, only the specified node will
print the step-killed-message;
SLURM_SRUN_REDUCE_TASK_EXIT_MSG: if set and non-zero, successive task
exit messages with the same exit code will be printed only once.
Hope this will be helpful to someone.
--- src/slurmd/slurmstepd/req.c 2013-01-08 08:23:06.000000000 +0800
+++ src/slurmd/slurmstepd/req.c.new 2013-01-09 09:00:22.000000000 +0800
@@ -48,6 +48,7 @@
#include <unistd.h>
#include <signal.h>
#include <time.h>
+#include <stdlib.h>
#include "src/common/cpu_frequency.h"
#include "src/common/fd.h"
@@ -591,6 +592,7 @@
{
int rc = SLURM_SUCCESS;
int signal;
+ char *ptr = NULL;
debug3("_handle_signal_process_group for job %u.%u",
job->jobid, job->stepid);
@@ -628,8 +630,11 @@
/*
* Print a message in the step output before killing when
* SIGTERM or SIGKILL are sent
+ * hjcao: print JOB/STEP KILLED msg on specific node id only
*/
- if ((signal == SIGTERM) || (signal == SIGKILL)) {
+ ptr = getenvp(job->env, "SLURM_STEP_KILLED_MSG_NODE_ID");
+ if ((!ptr || atoi(ptr) == job->nodeid) &&
+ ((signal == SIGTERM) || (signal == SIGKILL))) {
time_t now = time(NULL);
char entity[24], time_str[24];
if (job->stepid == SLURM_BATCH_SCRIPT) {
--- src/plugins/launch/slurm/launch_slurm.c 2013-01-08 08:38:12.000000000 +0800
+++ src/plugins/launch/slurm/launch_slurm.c.new 2013-01-08 10:19:34.000000000 +0800
@@ -288,9 +288,19 @@
char *hosts;
uint32_t rc = 0;
int normal_exit = 0;
+ static int reduce_task_exit_msg = -1;
+ static int msg_printed = 0, last_task_exit_rc;
const char *task_str = _taskstr(msg->num_tasks);
+ if (reduce_task_exit_msg == -1) {
+ char *ptr = getenv("SLURM_SRUN_REDUCE_TASK_EXIT_MSG");
+ if (ptr && atoi(ptr) != 0)
+ reduce_task_exit_msg = 1;
+ else
+ reduce_task_exit_msg = 0;
+ }
+
verbose("Received task exit notification for %d %s (status=0x%04x).",
msg->num_tasks, task_str, msg->return_code);
@@ -306,8 +316,13 @@
_handle_openmpi_port_error(tasks, hosts,
local_srun_job->step_ctx);
} else {
- error("%s: %s %s: Exited with exit code %d",
- hosts, task_str, tasks, rc);
+ if (reduce_task_exit_msg == 0 ||
+ msg_printed == 0 ||
+ msg->return_code != last_task_exit_rc) {
+ error("%s: %s %s: Exited with exit code %d",
+ hosts, task_str, tasks, rc);
+ msg_printed = 1;
+ }
}
if (!WIFEXITED(*local_global_rc)
|| (rc > WEXITSTATUS(*local_global_rc)))
@@ -325,8 +340,13 @@
hosts, task_str, tasks, signal_str, core_str);
} else {
rc = msg->return_code;
- error("%s: %s %s: %s%s",
- hosts, task_str, tasks, signal_str, core_str);
+ if (reduce_task_exit_msg == 0 ||
+ msg_printed == 0 ||
+ msg->return_code != last_task_exit_rc) {
+ error("%s: %s %s: %s%s",
+ hosts, task_str, tasks, signal_str, core_str);
+ msg_printed = 1;
+ }
}
if (*local_global_rc == 0)
*local_global_rc = msg->return_code;
@@ -344,6 +364,8 @@
if (task_state_first_exit(task_state) && (opt.max_wait > 0))
_setup_max_wait_timer();
+
+ last_task_exit_rc = msg->return_code;
}
/* Load the multi_prog config file into argv, pass the entire file contents
