The problem was that a reservation's times may advance a short time after it
completes. This could permit a job to be scheduled in the time period after
it completes and before the reservation records time stamps get advanced.
The attached patch will advance a reservation's time stamps in several
additional places, which should prevent any job from being scheduled at
the time of a future reservation.
I expect to get this change in SLURM version 2.2.3, which we can release
later this week.
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Morris "Moe" Jette [email protected] 925-423-4856
Integrated Computational Resource Management Group fax 925-423-6961
Livermore Computing Lawrence Livermore National Laboratory
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
________________________________________
From: [email protected] [[email protected]] On Behalf
Of Per Lundqvist [[email protected]]
Sent: Thursday, February 24, 2011 6:31 AM
To: [email protected]
Subject: [slurm-dev] too long jobs slips in between daily occurring reservation
At our site we have daily occurring reservation like:
ReservationName=devel StartTime=2011-02-24T08:00:00 EndTime=2011-02-24T18:00:00
Duration=10:00:00
Nodes=n[1-4] NodeCnt=4 Features=(null) PartitionName=(null)
Flags=IGNORE_JOBS,DAILY,SPEC_NODES
Users=user1,...
I.e. every day between 0800-1800 on nodes n[1-4]. To only allow jobs
below a certain walltime within the reservation we have a partition
called devel on those nodes n[1-4] with this walltime limit.
In order to make use of n[1-4] when the reservation is not in effect
(all days 18:00:01 - 07:59:59), we add them to another partition
during that time and remove them again from the latter partition when
the reservation is in effect, and so on.
When the reservation is not in effect, jobs with an expected endtime
including and after the next reservation start shouldn't be able to
start on n[1-4] - but this might actually happen.
Here is such a case where job 130973 with TimeLimit=1-16:00:00 starts
on these nodes on the other partition (after the nodes have been added
to it, after the reservation has ended and before the reservation has
been updated):
[2011-02-19T09:34:54] _slurm_rpc_submit_batch_job JobId=130973 usec=1503
[2011-02-23T18:00:01] update_part: setting nodes to n[1-8,93-364] for
partition kappa
[2011-02-23T18:00:11] sched: Allocate JobId=130973
NodeList=n[1-4,117,119,167,184] #CPUs=64
[2011-02-23T18:00:15] sched: _slurm_rpc_job_step_create: StepId=130973.0
n[1-4,117,119,167,184] usec=248
[2011-02-23T18:00:27] Advance reservation devel one day
All reservations should ideally advance atomically when they end I
guess, before any jobs could be scheduled on the nodes (in this case
at 18:00:01)?
regards,
Per Lundqvist
--
Per Lundqvist
National Supercomputer Centre
Linköping University, Sweden
http://www.nsc.liu.se
Index: src/slurmctld/reservation.c
===================================================================
--- src/slurmctld/reservation.c (revision 22578)
+++ src/slurmctld/reservation.c (working copy)
@@ -90,6 +90,7 @@
uint32_t cnodes_per_bp = 0;
#endif
+static void _advance_resv_time(slurmctld_resv_t *resv_ptr);
static void _advance_time(time_t *res_time, int day_cnt);
static int _build_account_list(char *accounts, int *account_cnt,
char ***account_list);
@@ -2365,6 +2366,7 @@
bitstr_t *node_bitmap;
ListIterator iter;
int i, rc = SLURM_SUCCESS;
+ time_t now = time(NULL);
if (*part_ptr == NULL) {
*part_ptr = default_part_loc;
@@ -2383,6 +2385,8 @@
if (!iter)
fatal("malloc: list_iterator_create");
while ((resv_ptr = (slurmctld_resv_t *) list_next(iter))) {
+ if (resv_ptr->end_time <= now)
+ _advance_resv_time(resv_ptr);
if ((resv_ptr->node_bitmap == NULL) ||
(resv_ptr->start_time >= resv_desc_ptr->end_time) ||
(resv_ptr->end_time <= resv_desc_ptr->start_time))
@@ -2675,6 +2679,8 @@
if (!iter)
fatal("malloc: list_iterator_create");
while ((resv_ptr = (slurmctld_resv_t *) list_next(iter))) {
+ if (resv_ptr->end_time <= now)
+ _advance_resv_time(resv_ptr);
if ((job_ptr->resv_ptr == resv_ptr) ||
(resv_ptr->start_time <= now))
continue; /* authorized user of reservation */
@@ -2755,7 +2761,7 @@
time_t when)
{
slurmctld_resv_t * resv_ptr;
- time_t job_start_time, job_end_time;
+ time_t job_start_time, job_end_time, now = time(NULL);
ListIterator iter;
int resv_cnt = 0;
@@ -2765,6 +2771,8 @@
if (!iter)
fatal("malloc: list_iterator_create");
while ((resv_ptr = (slurmctld_resv_t *) list_next(iter))) {
+ if (resv_ptr->end_time <= now)
+ _advance_resv_time(resv_ptr);
if ((resv_ptr->start_time >= job_end_time) ||
(resv_ptr->end_time <= job_start_time))
continue; /* reservation at different time */
@@ -2802,6 +2810,7 @@
{
slurmctld_resv_t * resv_ptr, *res2_ptr;
time_t job_start_time, job_end_time, lic_resv_time;
+ time_t now = time(NULL);
ListIterator iter;
int i, rc = SLURM_SUCCESS;
@@ -2818,6 +2827,8 @@
return ESLURM_RESERVATION_INVALID;
if (_valid_job_access_resv(job_ptr, resv_ptr) != SLURM_SUCCESS)
return ESLURM_RESERVATION_ACCESS;
+ if (resv_ptr->end_time <= now)
+ _advance_resv_time(resv_ptr);
if (*when < resv_ptr->start_time) {
/* reservation starts later */
*when = resv_ptr->start_time;
@@ -2825,7 +2836,7 @@
}
if (resv_ptr->node_cnt == 0) {
/* empty reservation treated like it will start later */
- *when = time(NULL) + 600;
+ *when = now + 600;
return ESLURM_INVALID_TIME_VALUE;
}
if (*when > resv_ptr->end_time) {
@@ -2885,6 +2896,8 @@
if (!iter)
fatal("malloc: list_iterator_create");
while ((resv_ptr = (slurmctld_resv_t *) list_next(iter))) {
+ if (resv_ptr->end_time <= now)
+ _advance_resv_time(resv_ptr);
if ((resv_ptr->node_bitmap == NULL) ||
(resv_ptr->start_time >= job_end_time) ||
(resv_ptr->end_time <= job_start_time))
@@ -2989,6 +3002,35 @@
return SLURM_SUCCESS;
}
+/* Advance a expired reservation's time stamps one day or one week
+ * as appropriate. */
+static void _advance_resv_time(slurmctld_resv_t *resv_ptr)
+{
+ int day_cnt = 0;
+ char *interval = "";
+
+ if (resv_ptr->flags & RESERVE_FLAG_DAILY) {
+ day_cnt = 1;
+ interval = "day";
+ } else if (resv_ptr->flags & RESERVE_FLAG_WEEKLY) {
+ day_cnt = 7;
+ interval = "week";
+ }
+
+ if (day_cnt) {
+ verbose("Advance reservation %s one %s", resv_ptr->name,
+ interval);
+ resv_ptr->start_time = resv_ptr->start_time_first;
+ _advance_time(&resv_ptr->start_time, day_cnt);
+ resv_ptr->start_time_prev = resv_ptr->start_time;
+ resv_ptr->start_time_first = resv_ptr->start_time;
+ _advance_time(&resv_ptr->end_time, day_cnt);
+ _post_resv_create(resv_ptr);
+ last_resv_update = time(NULL);
+ schedule_resv_save();
+ }
+}
+
/* Finish scan of all jobs for valid reservations
*
* Purge vestigial reservation records.
@@ -3012,35 +3054,7 @@
_validate_node_choice(resv_ptr);
continue;
}
-
- if ((resv_ptr->job_run_cnt == 0) &&
- (resv_ptr->flags & RESERVE_FLAG_DAILY)) {
- verbose("Advance reservation %s one day",
- resv_ptr->name);
- resv_ptr->start_time = resv_ptr->start_time_first;
- _advance_time(&resv_ptr->start_time, 1);
- resv_ptr->start_time_prev = resv_ptr->start_time;
- resv_ptr->start_time_first = resv_ptr->start_time;
- _advance_time(&resv_ptr->end_time, 1);
- _post_resv_create(resv_ptr);
- last_resv_update = now;
- schedule_resv_save();
- continue;
- }
- if ((resv_ptr->job_run_cnt == 0) &&
- (resv_ptr->flags & RESERVE_FLAG_WEEKLY)) {
- verbose("Advance reservation %s one week",
- resv_ptr->name);
- resv_ptr->start_time = resv_ptr->start_time_first;
- _advance_time(&resv_ptr->start_time, 7);
- resv_ptr->start_time_prev = resv_ptr->start_time;
- resv_ptr->start_time_first = resv_ptr->start_time;
- _advance_time(&resv_ptr->end_time, 7);
- _post_resv_create(resv_ptr);
- last_resv_update = now;
- schedule_resv_save();
- continue;
- }
+ _advance_resv_time(resv_ptr);
if ((resv_ptr->job_pend_cnt == 0) &&
(resv_ptr->job_run_cnt == 0) &&
(resv_ptr->maint_set_node == 0) &&
