diff --git a/etc/slurm.conf.example b/etc/slurm.conf.example index 425f4ff..fce49de 100644 --- a/etc/slurm.conf.example +++ b/etc/slurm.conf.example @@ -22,6 +22,7 @@ AuthType=auth/munge #JobCredentialPrivateKey= #JobCredentialPublicCertificate= StateSaveLocation=/tmp +EnableStateSavePartition=0 SlurmdSpoolDir=/tmp/slurmd SwitchType=switch/none MpiDefault=none diff --git a/src/api/config_info.c b/src/api/config_info.c index 34ecb84..40305fe 100644 --- a/src/api/config_info.c +++ b/src/api/config_info.c @@ -979,6 +979,13 @@ extern void *slurm_ctl_conf_2_key_pairs (slurm_ctl_conf_t* slurm_ctl_conf_ptr) key_pair->value = xstrdup(slurm_ctl_conf_ptr->state_save_location); list_append(ret_list, key_pair); + snprintf(tmp_str, sizeof(tmp_str), "%u", + slurm_ctl_conf_ptr->enable_state_save_partition); + key_pair = xmalloc(sizeof(config_key_pair_t)); + key_pair->name = xstrdup("EnableStateSavePartition"); + key_pair->value = xstrdup(tmp_str); + list_append(ret_list, key_pair); + key_pair = xmalloc(sizeof(config_key_pair_t)); key_pair->name = xstrdup("SuspendExcNodes"); key_pair->value = xstrdup(slurm_ctl_conf_ptr->suspend_exc_nodes); diff --git a/src/common/read_config.c b/src/common/read_config.c index f3a7dc2..f078d48 100644 --- a/src/common/read_config.c +++ b/src/common/read_config.c @@ -275,6 +275,7 @@ s_p_options_t slurm_conf_options[] = { {"SrunEpilog", S_P_STRING}, {"SrunProlog", S_P_STRING}, {"StateSaveLocation", S_P_STRING}, + {"EnableStateSavePartition", S_P_BOOLEAN}, {"SuspendExcNodes", S_P_STRING}, {"SuspendExcParts", S_P_STRING}, {"SuspendProgram", S_P_STRING}, @@ -1873,6 +1874,7 @@ free_slurm_conf (slurm_ctl_conf_t *ctl_conf_ptr, bool purge_node_hash) xfree (ctl_conf_ptr->srun_epilog); xfree (ctl_conf_ptr->srun_prolog); xfree (ctl_conf_ptr->state_save_location); + xfree (ctl_conf_ptr->enable_state_save_partition); xfree (ctl_conf_ptr->suspend_exc_nodes); xfree (ctl_conf_ptr->suspend_exc_parts); xfree (ctl_conf_ptr->suspend_program); @@ -2007,6 +2009,7 @@ init_slurm_conf (slurm_ctl_conf_t *ctl_conf_ptr) xfree (ctl_conf_ptr->srun_prolog); xfree (ctl_conf_ptr->srun_epilog); xfree (ctl_conf_ptr->state_save_location); + ctl_conf_ptr->enable_state_save_partition = 0; xfree (ctl_conf_ptr->suspend_exc_nodes); xfree (ctl_conf_ptr->suspend_exc_parts); xfree (ctl_conf_ptr->suspend_program); @@ -3173,6 +3176,10 @@ _validate_and_set_defaults(slurm_ctl_conf_t *conf, s_p_hashtbl_t *hashtbl) "StateSaveLocation", hashtbl)) conf->state_save_location = xstrdup(DEFAULT_SAVE_STATE_LOC); + if(!s_p_get_boolean((bool *)&conf->enable_state_save_partition, + "EnableStateSavePartition", hashtbl)) + conf->enable_state_save_partition = false; + s_p_get_string(&conf->suspend_exc_nodes, "SuspendExcNodes", hashtbl); s_p_get_string(&conf->suspend_exc_parts, "SuspendExcParts", hashtbl); s_p_get_string(&conf->suspend_program, "SuspendProgram", hashtbl); diff --git a/src/common/read_config.h b/src/common/read_config.h index 0fe1362..85e027c 100644 --- a/src/common/read_config.h +++ b/src/common/read_config.h @@ -121,6 +121,7 @@ extern char *default_plugstack; #define DEFAULT_RESUME_RATE 300 #define DEFAULT_RESUME_TIMEOUT 60 #define DEFAULT_SAVE_STATE_LOC "/var/spool" +#define DEFAULT_ENABLE_SAVE_STATE_PART 0 #define DEFAULT_SCHEDROOTFILTER 1 #define DEFAULT_SCHEDULER_PORT 7321 #define DEFAULT_SCHED_LOG_LEVEL 0 diff --git a/src/common/slurm_protocol_api.c b/src/common/slurm_protocol_api.c index 26458f7..8a24e73 100644 --- a/src/common/slurm_protocol_api.c +++ b/src/common/slurm_protocol_api.c @@ -722,6 +722,26 @@ char *slurm_get_state_save_location(void) return state_save_loc; } +/* slurm_get_enable_state_save_partition + * get enable_state_save_partition from slurmctld_conf object + */ +bool slurm_get_enable_state_save_partition(void) +{ + + bool enable_state_save_partition = 0; + slurm_ctl_conf_t *conf; + + if (slurmdbd_conf) { + } else { + conf = slurm_conf_lock(); + enable_state_save_partition = conf->enable_state_save_partition; + slurm_conf_unlock(); + } + + return enable_state_save_partition; +} + + /* slurm_get_auth_type * returns the authentication type from slurmctld_conf object * RET char * - auth type, MUST be xfreed by caller diff --git a/src/common/slurm_protocol_api.h b/src/common/slurm_protocol_api.h index 509cf3f..40894bf 100644 --- a/src/common/slurm_protocol_api.h +++ b/src/common/slurm_protocol_api.h @@ -304,6 +304,13 @@ uint16_t slurm_get_private_data(void); */ char *slurm_get_state_save_location(void); +/* slurm_get_enable_state_save_partition + * get enable_state_save_partition from slurmctld_conf object + * RET bool + */ +bool slurm_get_enable_state_save_partition(void); + + /* slurm_get_auth_type * returns the authentication type from slurmctld_conf object * RET char * - auth type, MUST be xfreed by caller diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c index cbc2f17..3c1018a 100644 --- a/src/common/slurm_protocol_pack.c +++ b/src/common/slurm_protocol_pack.c @@ -4705,6 +4705,7 @@ _pack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t * build_ptr, Buf buffer, packstr(build_ptr->srun_epilog, buffer); packstr(build_ptr->srun_prolog, buffer); packstr(build_ptr->state_save_location, buffer); + pack16(build_ptr->enable_state_save_partition, buffer); packstr(build_ptr->suspend_exc_nodes, buffer); packstr(build_ptr->suspend_exc_parts, buffer); packstr(build_ptr->suspend_program, buffer); @@ -4905,6 +4906,7 @@ _pack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t * build_ptr, Buf buffer, packstr(build_ptr->srun_epilog, buffer); packstr(build_ptr->srun_prolog, buffer); packstr(build_ptr->state_save_location, buffer); + pack16(build_ptr->enable_state_save_partition, buffer); packstr(build_ptr->suspend_exc_nodes, buffer); packstr(build_ptr->suspend_exc_parts, buffer); packstr(build_ptr->suspend_program, buffer); @@ -5102,6 +5104,8 @@ _pack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t * build_ptr, Buf buffer, packstr(build_ptr->srun_epilog, buffer); packstr(build_ptr->srun_prolog, buffer); packstr(build_ptr->state_save_location, buffer); + pack16(build_ptr->enable_state_save_partition, buffer); + packstr(build_ptr->suspend_exc_nodes, buffer); packstr(build_ptr->suspend_exc_parts, buffer); packstr(build_ptr->suspend_program, buffer); @@ -5296,6 +5300,7 @@ _pack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t * build_ptr, Buf buffer, packstr(build_ptr->srun_epilog, buffer); packstr(build_ptr->srun_prolog, buffer); packstr(build_ptr->state_save_location, buffer); + pack16(build_ptr->enable_state_save_partition, buffer); packstr(build_ptr->suspend_exc_nodes, buffer); packstr(build_ptr->suspend_exc_parts, buffer); packstr(build_ptr->suspend_program, buffer); @@ -5489,6 +5494,7 @@ _pack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t * build_ptr, Buf buffer, packstr(build_ptr->srun_epilog, buffer); packstr(build_ptr->srun_prolog, buffer); packstr(build_ptr->state_save_location, buffer); + pack16(build_ptr->enable_state_save_partition, buffer); packstr(build_ptr->suspend_exc_nodes, buffer); packstr(build_ptr->suspend_exc_parts, buffer); packstr(build_ptr->suspend_program, buffer); @@ -5775,6 +5781,7 @@ _unpack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t **build_buffer_ptr, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->state_save_location, &uint32_tmp, buffer); + safe_unpack16(&build_ptr->enable_state_save_partition, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_nodes, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_parts, @@ -6049,6 +6056,7 @@ _unpack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t **build_buffer_ptr, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->state_save_location, &uint32_tmp, buffer); + safe_unpack16(&build_ptr->enable_state_save_partition, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_nodes, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_parts, @@ -6319,6 +6327,7 @@ _unpack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t **build_buffer_ptr, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->state_save_location, &uint32_tmp, buffer); + safe_unpack16(&build_ptr->enable_state_save_partition, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_nodes, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_parts, @@ -6588,6 +6597,7 @@ _unpack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t **build_buffer_ptr, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->state_save_location, &uint32_tmp, buffer); + safe_unpack16(&build_ptr->enable_state_save_partition, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_nodes, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_parts, @@ -6858,6 +6868,7 @@ _unpack_slurm_ctl_conf_msg(slurm_ctl_conf_info_msg_t **build_buffer_ptr, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->state_save_location, &uint32_tmp, buffer); + safe_unpack16(&build_ptr->enable_state_save_partition, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_nodes, &uint32_tmp, buffer); safe_unpackstr_xmalloc(&build_ptr->suspend_exc_parts, diff --git a/src/slurmctld/controller.c b/src/slurmctld/controller.c index 8f31671..15147d1 100644 --- a/src/slurmctld/controller.c +++ b/src/slurmctld/controller.c @@ -49,6 +49,7 @@ # include #endif +#include #include #include #include @@ -58,6 +59,8 @@ #include #include #include +#include +#include #include "slurm/slurm_errno.h" @@ -2020,6 +2023,27 @@ extern void set_slurmctld_state_loc(void) int rc; struct stat st; const char *path = slurmctld_conf.state_save_location; + const bool partition_job_state = slurmctld_conf.enable_state_save_partition; + + char job_dir[200]; + char job_partition_dir[200]; + char job_dir_orig[200]; + char *partition; + + int match; + int match_job_dir; + regex_t re; + regex_t re_job_dir; + + struct dirent *direntStateDirectory; + struct dirent *direntJobDirectory; + struct dirent *direntPartitionDirectory; + + DIR *dirStateDirectory; + DIR *dirJobDirectory; + DIR *dirPartitionDirectory; + + /* * If state save location does not exist, try to create it. @@ -2036,8 +2060,96 @@ extern void set_slurmctld_state_loc(void) fatal("State save loc: %s: Not a directory!", path); else if (access(path, R_OK|W_OK|X_OK) < 0) fatal("Incorrect permissions on state save loc: %s", path); + + /* + * The addition of the enable_state_save_partition allows job states + * to be saved in either + * - /${job_state_save_location}/job_#{id}/ + * - /${job_state_save_location}/#{id[-2:]}/job_#{id}/ + * By switching the config flag, the jobs need to move to the correct + * location + */ + + if ( regcomp(&re_job_dir, "job\\.[0-9]+", REG_EXTENDED|REG_NOSUB) != 0 ) { + fatal("State save loc: couldn't compile regex to move job states into partition"); + } + + if ( slurm_get_enable_state_save_partition() ) { + debug3("set_slurmctld_state_loc: partitions are enabled", getpid()); + verbose("Using save state partitions"); + + partition = (char *) xmalloc(2); + dirStateDirectory = opendir(path); + if (dirStateDirectory) { + while (( direntStateDirectory = readdir(dirStateDirectory) ) != NULL ) { + match = regexec(&re_job_dir, direntStateDirectory->d_name, (size_t) 0, NULL, 0); + if (match == 0) { + strncpy(partition, direntStateDirectory->d_name + strlen(direntStateDirectory->d_name) - 2, 2); + + sprintf(job_dir, "%s/%s/%s", path, partition, direntStateDirectory->d_name); + sprintf(job_partition_dir, "%s/%s", path, partition); + sprintf(job_dir_orig, "%s/%s", path, direntStateDirectory->d_name); + + if (((rc = stat(job_partition_dir, &st)) < 0) && (errno == ENOENT)) { + if (mkdir(job_partition_dir, 0755) < 0) + fatal("Could not create partition directory"); + } + else if (rc < 0) + fatal("Unable to stat partition directory loc: %s", job_partition_dir); + else if (!S_ISDIR(st.st_mode)) + fatal("partition directory: %s: Not a directory!", job_partition_dir); + + rename(job_dir_orig, job_dir); + debug3("moving job state from: %s to: %s", job_dir_orig, job_dir); + verbose("moving job state from: %s to: %s", job_dir_orig, job_dir); + } + } + } + xfree (partition); + closedir(dirStateDirectory); + + } else { + debug3("set_slurmctld_state_loc: partitions are not enabled", getpid()); + verbose("Not using save state partitions"); + + if (regcomp(&re, "^[0-9]{2}$b", REG_EXTENDED|REG_NOSUB) != 0) { + fatal("State save loc: couldn't compile regex to move job states out of partitions"); + } + dirStateDirectory = opendir(path); + if( dirStateDirectory ){ + + + while ( (direntStateDirectory = readdir(dirStateDirectory) ) != NULL ) { + + match = regexec(&re, direntStateDirectory->d_name, (size_t) 0, NULL, 0); + if ( match == 0) { + sprintf(job_partition_dir, "%s/%s", path, direntStateDirectory->d_name); + dirPartitionDirectory = opendir(job_partition_dir); + if ( dirPartitionDirectory ) { + while (( direntPartitionDirectory = readdir(dirPartitionDirectory)) != NULL ){ + match_job_dir = regexec(&re_job_dir, direntPartitionDirectory->d_name, (size_t) 0, NULL, 0); + if (match_job_dir == 0){ + sprintf(job_dir_orig, "%s/%s", job_partition_dir, direntPartitionDirectory->d_name); + sprintf(job_dir, "%s/%s", path, direntPartitionDirectory->d_name); + debug3("moving job state from: %s to: %s", job_dir_orig, job_dir); + verbose("moving job state from: %s to: %s", job_dir_orig, job_dir); + rename (job_dir_orig, job_dir); + } + } + if (rmdir(job_partition_dir)) + debug3("Removing job partition directory: %s", job_partition_dir); + else + debug3("Unable to remove job partition directory: %s", job_partition_dir); + } + } + } + } + } + regfree(&re_job_dir); + } + /* _assoc_cache_mgr - hold out until we have real data from the * database so we can reset the job ptr's assoc ptr's */ static void *_assoc_cache_mgr(void *no_data) diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c index 23d7f05..07085dc 100644 --- a/src/slurmctld/job_mgr.c +++ b/src/slurmctld/job_mgr.c @@ -295,8 +295,16 @@ static void _delete_job_desc_files(uint32_t job_id) struct stat sbuf; dir_name = slurm_get_state_save_location(); + + + + /// if some config + if ( slurm_get_enable_state_save_partition() ) { + sprintf(job_dir, "/%02d/job.%d", job_id % 100, job_id); + } else { + sprintf(job_dir, "/job.%d", job_id); + } - sprintf(job_dir, "/job.%d", job_id); xstrcat(dir_name, job_dir); file_name = xstrdup(dir_name); @@ -4327,7 +4335,13 @@ _copy_job_desc_to_file(job_desc_msg_t * job_desc, uint32_t job_id) dir_name = slurm_get_state_save_location(); /* Create job_id specific directory */ - sprintf(job_dir, "/job.%u", job_id); + + if ( slurm_get_enable_state_save_partition() ) { + sprintf(job_dir, "/%02d/job.%u", job_id % 100, job_id); + } else { + sprintf(job_dir, "/job.%u", job_id); + } + xstrcat(dir_name, job_dir); if (mkdir(dir_name, 0700)) { error("mkdir(%s) error %m", dir_name); @@ -4451,7 +4465,11 @@ char **get_job_env(struct job_record *job_ptr, uint32_t * env_size) char job_dir[30], *file_name, **environment = NULL; file_name = slurm_get_state_save_location(); - sprintf(job_dir, "/job.%d/environment", job_ptr->job_id); + if ( slurm_get_enable_state_save_partition() ) { + sprintf(job_dir, "/%02d/job.%d/environment", job_ptr->job_id % 100, job_ptr->job_id); + } else { + sprintf(job_dir, "/job.%d/environment", job_ptr->job_id); + } xstrcat(file_name, job_dir); _read_data_array_from_file(file_name, &environment, env_size, job_ptr); @@ -4472,8 +4490,12 @@ char *get_job_script(struct job_record *job_ptr) if (job_ptr->batch_flag) { char *file_name = slurm_get_state_save_location(); char job_dir[30]; - - sprintf(job_dir, "/job.%d/script", job_ptr->job_id); + if ( slurm_get_enable_state_save_partition() ) { + sprintf(job_dir, "/%02d/job.%d/script", job_ptr->job_id % 100, job_ptr->job_id); + } else { + sprintf(job_dir, "/job.%d/script", job_ptr->job_id); + } + xstrcat(file_name, job_dir); _read_data_from_file(file_name, &script); diff --git a/src/slurmctld/proc_req.c b/src/slurmctld/proc_req.c index 92d7483..65c5ba9 100644 --- a/src/slurmctld/proc_req.c +++ b/src/slurmctld/proc_req.c @@ -622,6 +622,7 @@ void _fill_ctld_conf(slurm_ctl_conf_t * conf_ptr) conf_ptr->srun_prolog = xstrdup(conf->srun_prolog); conf_ptr->srun_epilog = xstrdup(conf->srun_epilog); conf_ptr->state_save_location = xstrdup(conf->state_save_location); + conf_ptr->enable_state_save_partition = conf->enable_state_save_partition; conf_ptr->suspend_exc_nodes = xstrdup(conf->suspend_exc_nodes); conf_ptr->suspend_exc_parts = xstrdup(conf->suspend_exc_parts); conf_ptr->suspend_program = xstrdup(conf->suspend_program);