Changeset: b3ad7b2f063f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b3ad7b2f063f
Added Files:
monetdb5/modules/mal/sysmon.c
monetdb5/modules/mal/sysmon.h
sql/test/BugTracker-2013/Tests/oid_handling.sql
sql/test/BugTracker-2013/Tests/oid_handling.stable.err
sql/test/BugTracker-2013/Tests/oid_handling.stable.out
Modified Files:
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_instruction.h
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_runtime.c
monetdb5/mal/mal_runtime.h
monetdb5/mal/mal_stack.h
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/querylog.c
monetdb5/modules/mal/querylog.h
monetdb5/modules/mal/querylog.mal
monetdb5/modules/mal/sysmon.mal
monetdb5/optimizer/opt_querylog.c
sql/backends/monet5/sql.mx
sql/backends/monet5/sql_gencode.c
sql/scripts/26_sysmon.sql
sql/test/BugTracker-2009/Tests/POWER_vs_prod.SF-2596114.stable.out
sql/test/BugTracker-2009/Tests/mclient-lsql-D.stable.out
sql/test/BugTracker-2009/Tests/name_clash_with_dump.SF-2780395.stable.out
sql/test/BugTracker-2010/Tests/LIMIT_OFFSET_big-endian.Bug-2622.stable.out
sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out
sql/test/BugTracker-2011/Tests/func_iter_vs_bulk.Bug-2826.stable.out
sql/test/BugTracker-2011/Tests/interrupted-initialization.Bug-2875.stable.out
sql/test/BugTracker-2012/Tests/predicate_select.Bug-3090.stable.err
sql/test/BugTracker-2012/Tests/predicate_select.Bug-3090.stable.out
sql/test/BugTracker-2012/Tests/rewrite_like_into_likesubselect.Bug-3179.stable.out
sql/test/BugTracker-2013/Tests/All
sql/test/BugTracker/Tests/explain.SF-1739353.stable.out
sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out
sql/test/BugTracker/Tests/multi-column-constraint.SF-1964587.stable.out
sql/test/Dump/Tests/dump-empty.stable.out
sql/test/Dump/Tests/dump.stable.out
sql/test/Tests/systemfunctions.stable.out
sql/test/mapi/Tests/php_monetdb.stable.out
Branch: default
Log Message:
Introduction of the active query queue monitor
The MonetDB kernel maintains an active queue of all running queries.
This queue is available for all users to inspect the status of
his own queries. The system administrator can inspect it to
overlook the complete workload on the system.
The queue is made visible as a table producing function, called sys.queue().
sql>select * from sys.queue();
+------+---------+----------------------------+----------------------------+----------+---------+-----------+----------------------------+
| qtag | user | started | estimate |
progress | status | tag | query |
+======+=========+============================+============================+==========+=========+===========+============================+
| 12 | monetdb | 2013-03-17 15:55:33.000000 | 2013-03-17 15:55:48.585000 |
0 | running | 1223889@0 | select * from sys.queue(); |
+------+---------+----------------------------+----------------------------+----------+---------+-----------+----------------------------+
1 tuple (0.412ms)
The schema structure is largely self-explanotory.
If the query (template) is ran multiple times,
then the system can derived a progress indicator and calculate an estimated
time of completion.
The 'tag' column references the query log tables, provided this facility has
been turned on.
The initial column 'qtag' provides a key to each active query
and can be used to sys.pause(tag), sys.resume(tag) and sys.stop(tag) the query.
Note that pause takes effect at the first safe point within the query plan,
which often is after the current MAL instruciton has been finished.
diffs (truncated from 1688 to 300 lines):
diff --git a/monetdb5/mal/mal_debugger.c b/monetdb5/mal/mal_debugger.c
--- a/monetdb5/mal/mal_debugger.c
+++ b/monetdb5/mal/mal_debugger.c
@@ -44,6 +44,10 @@ static void printStackHdr(stream *f, Mal
static mdbStateRecord *mdbTable;
+/*
+ * The debugger flags overview
+ */
+
void
mdbInit(void)
{
@@ -470,10 +474,6 @@ retryRead:
stk->cmd = 'C';
break;
}
- if (strncmp("call", b, 3) == 0) {
- showException(cntxt->fdout, MAL, "mdb.command",
"call instruction not yet implemented");
- break;
- }
stk->cmd = 'c';
skipWord(cntxt, b);
m = 0;
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -114,6 +114,7 @@ newMalBlk(int maxvars, int maxstmts)
mb->vtop = 0;
mb->vsize = maxvars;
mb->help = mb->binding = NULL;
+ mb->tag = 0;
mb->errors = 0;
mb->alternative = NULL;
mb->history = NULL;
@@ -178,6 +179,7 @@ freeMalBlk(MalBlkPtr mb)
if (mb->binding)
GDKfree(mb->binding);
mb->binding = 0;
+ mb->tag = 0;
if (mb->help)
GDKfree(mb->help);
mb->help = 0;
@@ -238,6 +240,7 @@ copyMalBlk(MalBlkPtr old)
mb->help = old->help ? GDKstrdup(old->help) : NULL;
mb->binding = old->binding ? GDKstrdup(old->binding) : NULL;
mb->errors = old->errors;
+ mb->tag = old->tag;
mb->typefixed = old->typefixed;
mb->flowfixed = old->flowfixed;
mb->recycle = old->recycle;
diff --git a/monetdb5/mal/mal_instruction.h b/monetdb5/mal/mal_instruction.h
--- a/monetdb5/mal/mal_instruction.h
+++ b/monetdb5/mal/mal_instruction.h
@@ -119,6 +119,7 @@ typedef struct PERF {
typedef struct MALBLK {
str binding; /* related C-function */
str help; /* supportive
commentary */
+ oid tag; /* unique block tag */
struct MALBLK *alternative;
int vtop; /* next free slot */
int vsize; /* size of variable
arena */
diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c
--- a/monetdb5/mal/mal_interpreter.c
+++ b/monetdb5/mal/mal_interpreter.c
@@ -319,7 +319,6 @@ str runMAL(Client cntxt, MalBlkPtr mb, M
RuntimeProfileRecord runtimeProfile;
(void) mbcaller;
- runtimeProfileInit(cntxt, mb, &runtimeProfile, cntxt->flags &
memoryFlag);
if (mb->errors) {
if (cntxt->itrace == 0) /* permit debugger analysis */
throw( MAL, "mal.interpreter", "Syntax error in
script");
@@ -370,6 +369,7 @@ str runMAL(Client cntxt, MalBlkPtr mb, M
* observed due the small size of the function).
*/
}
+ runtimeProfileInit(cntxt, mb, stk, &runtimeProfile, cntxt->flags &
memoryFlag);
if (stk->cmd && env && stk->cmd != 'f')
stk->cmd = env->cmd;
@@ -441,7 +441,6 @@ callMAL(Client cntxt, MalBlkPtr mb, MalS
* number of cores, which may be too coarse.
*/
MT_sema_down(&mal_parallelism,"callMAL");
- runtimeProfileInit(cntxt, mb, &runtimeProfile, cntxt->flags &
memoryFlag);
#ifdef DEBUG_CALLMAL
mnstr_printf(cntxt->fdout, "callMAL\n");
printInstruction(cntxt->fdout, mb, 0, pci, LIST_MAL_ALL);
@@ -514,11 +513,11 @@ str runMALsequence(Client cntxt, MalBlkP
//int tid = 0;
RuntimeProfileRecord runtimeProfile, runtimeProfileFunction;
- runtimeProfileInit(cntxt, mb, &runtimeProfile, cntxt->flags &
memoryFlag);
if (stk == NULL)
throw(MAL, "mal.interpreter", MAL_STACK_FAIL);
if (cntxt->flags & timerFlag)
oldtimer = cntxt->timer = GDKusec();
+ runtimeProfileInit(cntxt, mb, stk, &runtimeProfile, cntxt->flags &
memoryFlag);
/* prepare extended backup and garbage structures */
if ( mb->maxarg > 16 ){
@@ -532,21 +531,28 @@ str runMALsequence(Client cntxt, MalBlkP
/* also produce event record for start of function */
if ( startpc == 1 )
- runtimeProfileInit(cntxt, mb, &runtimeProfileFunction,
cntxt->flags & memoryFlag);
+ runtimeProfileInit(cntxt, mb, stk, &runtimeProfileFunction,
cntxt->flags & memoryFlag);
stkpc = startpc;
exceptionVar = -1;
while (stkpc < mb->stop && stkpc != stoppc) {
pci = getInstrPtr(mb, stkpc);
- if (cntxt->itrace || mb->trap) {
+ if (cntxt->itrace || mb->trap || stk->status) {
lng t = 0;
+ if (stk->status == 'p'){
+ // execution is paused
+ while ( stk->status == 'p')
+ MT_sleep_ms(50);
+ continue;
+ }
+ if ( stk->status == 'q')
+ stk->cmd = 'q';
+
if (stk->cmd == 0)
stk->cmd = cntxt->itrace;
if (oldtimer)
t = GDKusec();
- if (cntxt->flags & bbpFlag)
- BBPTraceCall(cntxt, mb, stk, prevpc);
prevpc = stkpc;
mdbStep(cntxt, mb, stk, stkpc);
if (stk->cmd == 'x' || cntxt->mode == FINISHING) {
diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c
--- a/monetdb5/mal/mal_runtime.c
+++ b/monetdb5/mal/mal_runtime.c
@@ -35,17 +35,9 @@
#define hashinfo(X) (((X) && (X)->mask)? ((X)->mask + (X)->lim + 1) *
sizeof(int) + sizeof(*(X)) + cnt * sizeof(int): 0)
// Keep a queue of running queries
-struct RUN {
- Client cntxt;
- MalBlkPtr mb;
- lng tag;
- str query;
- str status;
- lng start;
- lng runtime;
-} *running;
+QueryQueue QRYqueue;
static int qtop, qsize;
-static int qtag;
+static int qtag= 1;
static str isaSQLquery(MalBlkPtr mb){
@@ -57,7 +49,7 @@ static str isaSQLquery(MalBlkPtr mb){
if ( p->token == ENDsymbol)
break;
if ( getModuleId(p) && idcmp(getModuleId(p), "querylog") == 0
&& idcmp(getFunctionId(p),"define")==0)
- return getVarConstant(mb,getArg(p,2)).val.sval;
+ return getVarConstant(mb,getArg(p,1)).val.sval;
}
return 0;
}
@@ -66,21 +58,23 @@ static str isaSQLquery(MalBlkPtr mb){
* Manage the runtime profiling information
*/
void
-runtimeProfileInit(Client cntxt, MalBlkPtr mb, RuntimeProfile prof, int
initmemory)
+runtimeProfileInit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, RuntimeProfile
prof, int initmemory)
{
int i;
str q;
MT_lock_set(&mal_delayLock, "sysmon");
- if ( running == 0)
- running = (struct RUN *) GDKzalloc( sizeof (struct RUN) *
(qsize= 256));
+ if ( QRYqueue == 0)
+ QRYqueue = (QueryQueue) GDKzalloc( sizeof (struct QRYQUEUE) *
(qsize= 256));
else
if ( qtop +1 == qsize )
- running = (struct RUN *) GDKrealloc( running, sizeof (struct
RUN) * (qsize +=256));
+ QRYqueue = (QueryQueue) GDKrealloc( QRYqueue, sizeof (struct
QRYQUEUE) * (qsize +=256));
for( i = 0; i < qtop; i++)
- if ( running[i].mb == mb)
+ if ( QRYqueue[i].mb == mb)
break;
+ if ( mb->tag == 0)
+ mb->tag = OIDnew(1);
prof->newclk = 0;
prof->ppc = -2;
prof->tcs = 0;
@@ -88,14 +82,15 @@ runtimeProfileInit(Client cntxt, MalBlkP
prof->oublock = 0;
if ( i == qtop ) {
- running[i].mb = mb; // for detecting duplicates
- running[i].tag = qtag++;
- running[i].start = GDKusec();
- running[i].runtime = mb->runtime;
+ QRYqueue[i].mb = mb; // for detecting duplicates
+ QRYqueue[i].stk = stk; // for status pause 'p'/running '0'/
quiting 'q'
+ QRYqueue[i].tag = qtag++;
+ QRYqueue[i].start = (lng)time(0);
+ QRYqueue[i].runtime = mb->runtime;
q = isaSQLquery(mb);
- running[i].query = q? GDKstrdup(q):0;
- running[i].status = "running";
- running[i].cntxt = cntxt;
+ QRYqueue[i].query = q? GDKstrdup(q):0;
+ QRYqueue[i].status = "running";
+ QRYqueue[i].cntxt = cntxt;
}
if (initmemory)
prof->memory = MT_mallinfo();
@@ -122,17 +117,21 @@ runtimeProfileFinish(Client cntxt, MalBl
MT_lock_set(&mal_delayLock, "sysmon");
for( i=j=0; i< qtop; i++)
- if ( running[i].mb != mb)
- running[j++] = running[i];
+ if ( QRYqueue[i].mb != mb)
+ QRYqueue[j++] = QRYqueue[i];
else {
- if (running[i].query)
- GDKfree(running[i].query);
- running[i].cntxt = 0;
- running[i].tag = 0;
- running[i].query = 0;
- running[i].status =0;
- mb->calls++;
- mb->runtime += ((GDKusec() - running[i].start)-
running[i].runtime)/mb->calls;
+ QRYqueue[i].mb->calls++;
+ QRYqueue[i].mb->runtime += ((lng)time(0) - QRYqueue[i].start) *
1000.0/QRYqueue[i].mb->calls;
+
+ // reset entry
+ if (QRYqueue[i].query)
+ GDKfree(QRYqueue[i].query);
+ QRYqueue[i].cntxt = 0;
+ QRYqueue[i].tag = 0;
+ QRYqueue[i].query = 0;
+ QRYqueue[i].status =0;
+ QRYqueue[i].stk =0;
+ QRYqueue[i].mb =0;
}
qtop = j;
@@ -284,139 +283,3 @@ updateFootPrint(MalBlkPtr mb, MalStkPtr
stk->tmpspace += total/1024/1024; // keep it in MBs
}
}
-
-str
-runtimeSQLqueue(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
-{
- BAT *tag, *user, *query, *estimate, *started, *progress, *activity;
- int *t = (int*) getArgReference(stk,pci,0);
- int *u = (int*) getArgReference(stk,pci,1);
- int *s = (int*) getArgReference(stk,pci,2);
- int *e = (int*) getArgReference(stk,pci,3);
- int *p = (int*) getArgReference(stk,pci,4);
- int *a = (int*) getArgReference(stk,pci,5);
- int *q = (int*) getArgReference(stk,pci,6);
- lng now;
- int i, prog;
- str usr;
-
- (void) cntxt;
- (void) mb;
- tag = BATnew(TYPE_void, TYPE_lng, qsize);
- user = BATnew(TYPE_void, TYPE_str, qsize);
- started = BATnew(TYPE_void, TYPE_lng, qsize);
- estimate = BATnew(TYPE_void, TYPE_lng, qsize);
- progress = BATnew(TYPE_void, TYPE_int, qsize);
- activity = BATnew(TYPE_void, TYPE_str, qsize);
- query = BATnew(TYPE_void, TYPE_str, qsize);
- if ( tag == NULL || query == NULL || started == NULL || estimate ==
NULL || progress == NULL || activity == NULL){
- if (tag) BBPreleaseref(tag->batCacheid);
- if (user) BBPreleaseref(user->batCacheid);
- if (query) BBPreleaseref(query->batCacheid);
- if (activity) BBPreleaseref(activity->batCacheid);
- if (started) BBPreleaseref(started->batCacheid);
- if (estimate) BBPreleaseref(estimate->batCacheid);
- if (progress) BBPreleaseref(progress->batCacheid);
- throw(MAL, "runtimeSQLqueue", MAL_MALLOC_FAIL);
- }
- BATseqbase(tag, 0);
- BATkey(tag, TRUE);
-
- BATseqbase(user, 0);
- BATkey(user, TRUE);
-
- BATseqbase(query, 0);
- BATkey(query, TRUE);
-
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
