Hello there!
Currently there is no stackable plugins in slurmd but sometimes it's
crucial to do something in system environment before slurmstepd is even
forked. Apparently task plugin is the best one to make it stackable. And
attached patch make it so. It's still fully compatible with old behavior
but there is new one - TaskPlugin variable in slurm.conf can be a list of
plugins with comma as separator, each element can be either 'task/plugin'
or just 'plugin' name as it is for JobSubmitPlugins variable. Neither any
changes in documentation was made nor any changes for TaskPluginParams
variable, it's still sticky for task/affinity.
Hope to see it in next SLURM.
Andriy.diff -udpr slurm-2.2.6.orig/src/slurmd/common/task_plugin.c slurm-2.2.6/src/slurmd/common/task_plugin.c
--- slurm-2.2.6.orig/src/slurmd/common/task_plugin.c 2011-05-27 21:25:06.000000000 +0300
+++ slurm-2.2.6/src/slurmd/common/task_plugin.c 2011-06-08 14:52:13.000000000 +0300
@@ -73,7 +73,8 @@ typedef struct slurmd_task_context {
slurmd_task_ops_t ops;
} slurmd_task_context_t;
-static slurmd_task_context_t *g_task_context = NULL;
+static slurmd_task_context_t **g_task_context = NULL;
+static int g_task_context_num = -1;
static pthread_mutex_t g_task_context_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -195,34 +196,57 @@ _slurmd_task_context_destroy(slurmd_task
*/
extern int slurmd_task_init(void)
{
- int retval = SLURM_SUCCESS;
+ int retval = SLURM_SUCCESS, i;
char *task_plugin_type = NULL;
+ char *last = NULL, *task_plugin_list, *task_plugin = NULL;
slurm_mutex_lock( &g_task_context_lock );
- if ( g_task_context )
+ if ( g_task_context_num >= 0 )
goto done;
task_plugin_type = slurm_get_task_plugin();
- g_task_context = _slurmd_task_context_create( task_plugin_type );
- if ( g_task_context == NULL ) {
- error( "cannot create task context for %s",
- task_plugin_type );
- retval = SLURM_ERROR;
+ g_task_context_num = 0; /* mark it before anything else */
+ if (task_plugin_type == NULL || task_plugin_type[0] == '\0')
goto done;
- }
- if ( _slurmd_task_get_ops( g_task_context ) == NULL ) {
- error( "cannot resolve task plugin operations" );
- _slurmd_task_context_destroy( g_task_context );
- g_task_context = NULL;
- retval = SLURM_ERROR;
+ task_plugin_list = task_plugin_type;
+ while ((task_plugin = strtok_r(task_plugin_list, ",", &last))) {
+ i = g_task_context_num++;
+ xrealloc(g_task_context,
+ (sizeof(slurmd_task_context_t *) * g_task_context_num));
+ if (strncmp(task_plugin, "task/", 5) == 0)
+ task_plugin += 5; /* backward compatibility */
+ task_plugin = xstrdup_printf("task/%s", task_plugin);
+ g_task_context[i] = _slurmd_task_context_create( task_plugin );
+ if ( g_task_context[i] == NULL ) {
+ error( "cannot create task context for %s",
+ task_plugin );
+ goto error;
+ }
+
+ if ( _slurmd_task_get_ops( g_task_context[i] ) == NULL ) {
+ error( "cannot resolve task plugin operations for %s",
+ task_plugin );
+ goto error;
+ }
+ xfree(task_plugin);
+ task_plugin_list = NULL; /* for next iteration */
}
done:
slurm_mutex_unlock( &g_task_context_lock );
xfree(task_plugin_type);
return retval;
+
+error:
+ xfree(task_plugin);
+ retval = SLURM_ERROR;
+ for (i = 0; i < g_task_context_num; i++)
+ if (g_task_context[i])
+ _slurmd_task_context_destroy(g_task_context[i]);
+ xfree(g_task_context);
+ goto done;
}
/*
@@ -232,13 +256,21 @@ extern int slurmd_task_init(void)
*/
extern int slurmd_task_fini(void)
{
- int rc;
+ int i, rc = SLURM_SUCCESS;
+ slurm_mutex_lock( &g_task_context_lock );
if (!g_task_context)
- return SLURM_SUCCESS;
+ goto done;
- rc = _slurmd_task_context_destroy(g_task_context);
- g_task_context = NULL;
+ for (i = 0; i < g_task_context_num; i++)
+ if (_slurmd_task_context_destroy(g_task_context[i]) != SLURM_SUCCESS)
+ rc = SLURM_ERROR;
+
+ xfree(g_task_context);
+ g_task_context_num = -1;
+
+done:
+ slurm_mutex_unlock( &g_task_context_lock );
return rc;
}
@@ -249,10 +281,17 @@ extern int slurmd_task_fini(void)
*/
extern int slurmd_batch_request(uint32_t job_id, batch_job_launch_msg_t *req)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.slurmd_batch_request))(job_id, req);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.slurmd_batch_request))(job_id, req);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -264,10 +303,18 @@ extern int slurmd_launch_request(uint32_
launch_tasks_request_msg_t *req,
uint32_t node_id)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.slurmd_launch_request))(job_id, req, node_id);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.slurmd_launch_request))(job_id,
+ req, node_id);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -279,10 +326,18 @@ extern int slurmd_reserve_resources(uint
launch_tasks_request_msg_t *req,
uint32_t node_id )
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.slurmd_reserve_resources))(job_id, req, node_id);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.slurmd_reserve_resources))(job_id,
+ req, node_id);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -292,10 +347,17 @@ extern int slurmd_reserve_resources(uint
*/
extern int slurmd_suspend_job(uint32_t job_id)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.slurmd_suspend_job))(job_id);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.slurmd_suspend_job))(job_id);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -305,10 +367,17 @@ extern int slurmd_suspend_job(uint32_t j
*/
extern int slurmd_resume_job(uint32_t job_id)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.slurmd_resume_job))(job_id);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.slurmd_resume_job))(job_id);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -318,10 +387,17 @@ extern int slurmd_resume_job(uint32_t jo
*/
extern int slurmd_release_resources(uint32_t job_id)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.slurmd_release_resources))(job_id);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.slurmd_release_resources))(job_id);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -332,10 +408,17 @@ extern int slurmd_release_resources(uint
*/
extern int pre_setuid(slurmd_job_t *job)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.pre_setuid))(job);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.pre_setuid))(job);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -345,10 +428,17 @@ extern int pre_setuid(slurmd_job_t *job)
*/
extern int pre_launch(slurmd_job_t *job)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.pre_launch))(job);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.pre_launch))(job);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
/*
@@ -358,8 +448,15 @@ extern int pre_launch(slurmd_job_t *job)
*/
extern int post_term(slurmd_job_t *job)
{
+ int i, rc = SLURM_SUCCESS;
+
if (slurmd_task_init())
return SLURM_ERROR;
- return (*(g_task_context->ops.post_term))(job);
+ slurm_mutex_lock( &g_task_context_lock );
+ for (i = 0; ((i < g_task_context_num) && (rc == SLURM_SUCCESS)); i++)
+ rc = (*(g_task_context[i]->ops.post_term))(job);
+ slurm_mutex_unlock( &g_task_context_lock );
+
+ return (rc);
}
