El vie, 05-04-2013 a las 08:19 -0600, Danny Auble escribió:
> Without a backtrace this would be very difficult to solve. a
> slurmdbd.conf file would also be helpful.
Hello Danny,
Our slurmdbd.conf file is:
#
# Example slurmdbd.conf file.
#
# See the slurmdbd.conf man page for more information.
#
# Archive info
#ArchiveJobs=yes
#ArchiveDir="/tmp"
#ArchiveSteps=yes
#ArchiveScript=
#JobPurge=12
#StepPurge=1
#
# Authentication info
AuthType=auth/munge
#AuthInfo=/var/run/munge/munge.socket.2
#
# slurmDBD info
DbdHost=localhost
DbdPort=6819
SlurmUser=slurm
#MessageTimeout=300
DebugLevel=7
#DefaultQOS=normal,standby
LogFile=/usr/local/slurm-2.4.3/log/slurmdbd.log
PidFile=/var/run/slurmdbd.pid
#PidFile=/usr/local/slurm-2.4.3/pid/slurmdbd.pid
#PluginDir=/usr/lib/slurm
#PrivateData=accounts,users,usage,jobs
#TrackWCKey=yes
#
# Database info
StorageType=accounting_storage/mysql
StorageHost=localhost
#StoragePort=1234
#StoragePass=password
StorageUser=slurm
StorageLoc=slurm_acct_db
TrackWCKey=yes
ArchiveDir="/tmp"
ArchiveEvents=yes
ArchiveJobs=yes
ArchiveSteps=yes
ArchiveSuspend=yes
PurgeEventAfter=2months
PurgeJobAfter=12months
PurgeStepAfter=2days
PurgeSuspendAfter=2hours
We have tried also with:
ArchiveSteps=no
ArchiveSuspend=no
The problem is the same.
The log file is huge, the final lines are:
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:RUNNING
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369643-time_suspended,
state=1 where job_db_inx=62441;update "(null)_suspend_table" set
time_end=1364369643 where job_db_inx=62441 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:137780.4294967294
NAME:batch SUBMIT:1364291749
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=137780 and
time_submit=1364291749 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 137780 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:137780.4294967294
SUBMIT:1364291749
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=137780 and
time_submit=1364291749 and time_eligible=0 and time_start=1364362597;
[2013-04-05T21:26:37] error: couldn't add job 137780 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_COMPLETE: ID:137780
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_complete() called
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=137780 and
time_submit=1364291749 and time_eligible=0 and time_start=1364362597;
[2013-04-05T21:26:37] error: couldn't add job 137780 at job completion
[2013-04-05T21:26:37] DBD_JOB_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:SUSPENDED
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369678-time_suspended,
state=2 where job_db_inx=62441;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62441, 1325, 1364369678, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369678-time_suspended,
state=1 where job_db_inx=62420;update "(null)_suspend_table" set
time_end=1364369678 where job_db_inx=62420 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369708-time_suspended,
state=2 where job_db_inx=62420;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62420, 1325, 1364369708, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:RUNNING
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369708-time_suspended,
state=1 where job_db_inx=62441;update "(null)_suspend_table" set
time_end=1364369708 where job_db_inx=62441 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_START: START CALL ID:137920
NAME:launchazuleno_6CC_S.1339.com.sh INX:0
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 1526
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc, account, partition, node_inx) values
(137920, 1526, 27, 0, 888, 424, 'bladi23', 0, 7200, 1364291808,
1364291808, 1364369713, 'launchazuleno_6CC_S.1339.com.sh', 0, 1, 11212,
1, 1, 1, 'quiralidad_serv', 'ccc', '26') on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=888,
id_group=424, nodelist='bladi23', id_resv=0, timelimit=7200,
time_submit=1364291808, time_start=1364369713,
job_name='launchazuleno_6CC_S.1339.com.sh', track_steps=0, id_qos=27,
state=greatest(state, 1), priority=11212, cpus_req=1, cpus_alloc=1,
nodes_alloc=1, account='quiralidad_serv', partition='ccc', node_inx='26'
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] DBD_JOB_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:SUSPENDED
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369738-time_suspended,
state=2 where job_db_inx=62441;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62441, 1325, 1364369738, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369738-time_suspended,
state=1 where job_db_inx=62420;update "(null)_suspend_table" set
time_end=1364369738 where job_db_inx=62420 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369768-time_suspended,
state=2 where job_db_inx=62420;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62420, 1325, 1364369768, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:RUNNING
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369768-time_suspended,
state=1 where job_db_inx=62441;update "(null)_suspend_table" set
time_end=1364369768 where job_db_inx=62441 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:SUSPENDED
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369800-time_suspended,
state=2 where job_db_inx=62441;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62441, 1325, 1364369800, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369801-time_suspended,
state=1 where job_db_inx=62420;update "(null)_suspend_table" set
time_end=1364369801 where job_db_inx=62420 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369831-time_suspended,
state=2 where job_db_inx=62420;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62420, 1325, 1364369831, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:RUNNING
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369831-time_suspended,
state=1 where job_db_inx=62441;update "(null)_suspend_table" set
time_end=1364369831 where job_db_inx=62441 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:SUSPENDED
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369861-time_suspended,
state=2 where job_db_inx=62441;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62441, 1325, 1364369861, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369861-time_suspended,
state=1 where job_db_inx=62420;update "(null)_suspend_table" set
time_end=1364369861 where job_db_inx=62420 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369891-time_suspended,
state=2 where job_db_inx=62420;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62420, 1325, 1364369891, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:RUNNING
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369891-time_suspended,
state=1 where job_db_inx=62441;update "(null)_suspend_table" set
time_end=1364369891 where job_db_inx=62441 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.56
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364368527, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364368527, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.57 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.57
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369903, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369903, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.58 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.58
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369906, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369906, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.59 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.59
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369909, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369909, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.60 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.60
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369911, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369911, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.61 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.61
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369914, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369914, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.62 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:SUSPENDED
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369923-time_suspended,
state=2 where job_db_inx=62441;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62441, 1325, 1364369923, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369923-time_suspended,
state=1 where job_db_inx=62420;update "(null)_suspend_table" set
time_end=1364369923 where job_db_inx=62420 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.62
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369916, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369916, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.63 NAME:orted
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.63
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364369923, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364369923, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:141917.4294967294
NAME:batch SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=141917 and
time_submit=1364295090 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 141917 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:141917.4294967294
SUBMIT:1364295090
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'None assigned', 0, 0, 1364295090, 1364295090, 1364367465, 'allocation',
1, 0, 0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='None assigned', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364367465, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 0), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_COMPLETE: ID:141917
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_complete() called
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] error: No user for associd 511
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:481) query
insert into "(null)_job_table" (id_job, id_assoc, id_qos, id_wckey,
id_user, id_group, nodelist, id_resv, timelimit, time_eligible,
time_submit, time_start, job_name, track_steps, state, priority,
cpus_req, cpus_alloc, nodes_alloc) values (141917, 511, 0, 0, 0, 0,
'rodio', 0, 0, 1364295090, 1364295090, 1364367465, 'allocation', 1, 3,
0, 0, 0, 0) on duplicate key update
job_db_inx=LAST_INSERT_ID(job_db_inx), id_wckey=0, id_user=0,
id_group=0, nodelist='rodio', id_resv=0, timelimit=0,
time_submit=1364295090, time_start=1364367465, job_name='allocation',
track_steps=1, id_qos=0, state=greatest(state, 3), priority=0,
cpus_req=0, cpus_alloc=0, nodes_alloc=0
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: It looks like the storage has gone away
trying to reconnect
[2013-04-05T21:26:37] error: We should have gotten a new id: Table
'slurm_acct_db.(null)_job_table' doesn't exist
[2013-04-05T21:26:37] error: couldn't add job 141917 at job completion
[2013-04-05T21:26:37] DBD_JOB_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_START: ID:137774.4294967294
NAME:batch SUBMIT:1364291747
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=137774 and
time_submit=1364291747 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't add job 137774 at step start
[2013-04-05T21:26:37] DBD_STEP_START: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_STEP_COMPLETE: ID:137774.4294967294
SUBMIT:1364291747
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=137774 and
time_submit=1364291747 and time_eligible=0 and time_start=1364361641;
[2013-04-05T21:26:37] error: couldn't add job 137774 at step completion
[2013-04-05T21:26:37] DBD_STEP_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_COMPLETE: ID:137774
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_complete() called
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=137774 and
time_submit=1364291747 and time_eligible=0 and time_start=1364361641;
[2013-04-05T21:26:37] error: couldn't add job 137774 at job completion
[2013-04-05T21:26:37] DBD_JOB_COMPLETE: cluster not registered
[2013-04-05T21:26:37] error: accounting_storage_mysql.c:2612 no cluster
name
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55930 STATE:SUSPENDED
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369958-time_suspended,
state=2 where job_db_inx=62420;insert into
"(null)_suspend_table" (job_db_inx, id_assoc, time_start, time_end)
values (62420, 1325, 1364369958, 0);
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:136360 STATE:RUNNING
[2013-04-05T21:26:37] debug2: as_mysql_slurmdb_job_start() called
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:299) query
select job_db_inx from "(null)_job_table" where id_job=136360 and
time_submit=1364221033 and time_eligible=0 and time_start=0;
[2013-04-05T21:26:37] error: couldn't suspend job 136360
[2013-04-05T21:26:37] debug2: DBD_JOB_SUSPEND: ID:55951 STATE:RUNNING
[2013-04-05T21:26:37] debug3: 9(as_mysql_job.c:1146) query
update "(null)_job_table" set time_suspended=1364369958-time_suspended,
state=1 where job_db_inx=62441;update "(null)_suspend_table" set
time_end=1364369958 where job_db_inx=62441 && time_end=0;
[2013-04-05T21:26:37] debug2: DBD_REGISTER_CTLD: called for (null)(6820)
[2013-04-05T21:26:37] debug2: slurmctld at ip:192.168.22.18, port:6820
Thanks in advance,
Pablo Sanz
--
Pablo Sanz Mercado
Director Técnico
Centro de Computación Científica
Universidad Autónoma de Madrid
