Changeset: 7fc299e39776 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7fc299e39776 Added Files: monetdb5/modules/mal/Tests/cpustat.mal monetdb5/modules/mal/querylog.c monetdb5/modules/mal/querylog.h monetdb5/modules/mal/querylog.mal monetdb5/optimizer/opt_querylog.c monetdb5/optimizer/opt_querylog.h sql/scripts/15_querylog.sql Removed Files: monetdb5/optimizer/opt_history.c monetdb5/optimizer/opt_history.h sql/scripts/15_history.sql Modified Files: monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/Tests/All monetdb5/modules/mal/Tests/clients01.mal monetdb5/modules/mal/mal_init.mal monetdb5/modules/mal/profiler.c monetdb5/modules/mal/profiler.h monetdb5/modules/mal/profiler.mal monetdb5/optimizer/Makefile.ag monetdb5/optimizer/Tests/dataflow3.mal monetdb5/optimizer/opt_pipes.c monetdb5/optimizer/opt_prelude.c monetdb5/optimizer/opt_prelude.h monetdb5/optimizer/opt_wrapper.c monetdb5/optimizer/optimizer.mal sql/backends/monet5/sql.mx sql/backends/monet5/sql_gencode.c sql/scripts/Makefile.ag sql/server/sql_env.c sql/server/sql_env.h Branch: default Log Message:
Replace 15_history with 15_querylog The history script maintained a query log using a patch of the SQL queries calling for it. Aside from the expensive step to perform these updates thru inlined SQL-MAL code, it can not support concurrent use. Each update would invalidate the complete concurrent query. The complete functionality has been replaced by the querylog schema, which uses hardwired tables and BAT transaction mgmt to keep the query log. The functionality of the log has been changed also, see http://www.monetdb.org/Documentation/Cookbooks/SQLrecipies/QueryHistoryNew diffs (truncated from 1708 to 300 lines): diff --git a/monetdb5/modules/mal/Makefile.ag b/monetdb5/modules/mal/Makefile.ag --- a/monetdb5/modules/mal/Makefile.ag +++ b/monetdb5/modules/mal/Makefile.ag @@ -51,6 +51,7 @@ lib_mal = { pcre.c \ pqueue.c pqueue.h \ profiler.c profiler.h \ + querylog.c querylog.h \ recycle.c recycle.h \ remote.c remote.h \ sabaoth.c sabaoth.h \ @@ -78,7 +79,7 @@ headers_mal = { txtsim.mal recycle.mal \ cluster.mal trader.mal \ tokenizer.mal zorder.mal sample.mal \ - calc.mal batcalc.mal batmtime.mal + calc.mal batcalc.mal batmtime.mal querylog.mal } EXTRA_DIST = batExtensions.mal iterator.mal constraints.mal \ @@ -86,6 +87,6 @@ EXTRA_DIST = batExtensions.mal iterator. profiler.mal recycle.mal remote.mal sabaoth.mal trader.mal \ transaction.mal txtsim.mal tablet.mal tablet.h sample.mal \ mal_mapi.mal mat.mal tokenizer.mal pqueue.mal calc.mal \ - batcalc.mal batmtime.mal + batcalc.mal batmtime.mal querylog.mal EXTRA_DIST_DIR = Tests diff --git a/monetdb5/modules/mal/Tests/All b/monetdb5/modules/mal/Tests/All --- a/monetdb5/modules/mal/Tests/All +++ b/monetdb5/modules/mal/Tests/All @@ -72,3 +72,5 @@ clients-hashes # shows different output if the password backend is different (or # monetdb password has been changed) clients-getpasswd + +#cpustat too system dependent to always run. diff --git a/monetdb5/modules/mal/Tests/clients01.mal b/monetdb5/modules/mal/Tests/clients01.mal --- a/monetdb5/modules/mal/Tests/clients01.mal +++ b/monetdb5/modules/mal/Tests/clients01.mal @@ -8,7 +8,7 @@ io.print(d); #u:= clients.getUsers(); #io.print(u); -#l:= clients.getLogins(); -#io.print(l); +#(u,l):= clients.getLogins(); +#io.print(u,l); diff --git a/monetdb5/modules/mal/Tests/cpustat.mal b/monetdb5/modules/mal/Tests/cpustat.mal new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/Tests/cpustat.mal @@ -0,0 +1,16 @@ +(user,nice,sys,idle,iowait) := profiler.cpustats(); +io.print(user, nice, sys, idle, iowait); + +b:= bat.new(:oid,:lng); +r:= mmath.srand(0); +barrier i:= 0:lng; + k:= mmath.rand(); + l:= calc.lng(k); + bat.append(b,l); + redo i:= iterator.next(1:lng,1000000:lng); +exit i; + +(load,io):= profiler.cpuload(user,nice,sys,idle,iowait); +io.print(load,io); +(user,nice,sys,idle,iowait) := profiler.cpustats(); +io.print(user, nice, sys, idle, iowait); diff --git a/monetdb5/modules/mal/mal_init.mal b/monetdb5/modules/mal/mal_init.mal --- a/monetdb5/modules/mal/mal_init.mal +++ b/monetdb5/modules/mal/mal_init.mal @@ -98,6 +98,7 @@ include constraints; # experimental include mal_mapi; include profiler; +include querylog; include sample; include optimizer; diff --git a/monetdb5/modules/mal/profiler.c b/monetdb5/modules/mal/profiler.c --- a/monetdb5/modules/mal/profiler.c +++ b/monetdb5/modules/mal/profiler.c @@ -371,7 +371,9 @@ CMDgetFootprint( Client cntxt, MalBlkPtr lng *l= getArgReference(stk,pci,0); (void) cntxt; - *l = getFootPrint(mb,stk); + (void) mb; + *l = stk->tmpspace; + cntxt->flags &= ~footprintFlag; // clear it as well return MAL_SUCCEED; } @@ -390,7 +392,7 @@ CMDgetMemory( Client cntxt, MalBlkPtr mb (void) mb; *l = cntxt->memory; - cntxt->flags &= ~memoryFlag; + cntxt->flags &= ~memoryFlag; //clear it as well cntxt->memory= 0; return MAL_SUCCEED; } @@ -401,3 +403,25 @@ CMDtomograph(int *ret) (void) ret; return MAL_SUCCEED; } + +str +CMDcpustats(lng *user, lng *nice, lng *sys, lng *idle, lng *iowait) +{ + profilerGetCPUStat(user,nice,sys,idle,iowait); + return MAL_SUCCEED; +} + +str +CMDcpuloadPercentage(int *cycles, int *io, lng *user, lng *nice, lng *sys, lng *idle, lng *iowait) +{ + lng userN, niceN, sysN, idleN, iowaitN, N; + *cycles = 0; + *io = 0; + profilerGetCPUStat(&userN,&niceN,&sysN,&idleN,&iowaitN); + N = (userN - *user + niceN - *nice + sysN - *sys); + if ( N){ + *cycles = (int) ( ((double) N) / (N + idleN - *idle + iowaitN - *iowait) *100); + *io = (int) ( ((double) iowaitN- *iowait) / (N + idleN - *idle + iowaitN - *iowait) *100); + } + return MAL_SUCCEED; +} diff --git a/monetdb5/modules/mal/profiler.h b/monetdb5/modules/mal/profiler.h --- a/monetdb5/modules/mal/profiler.h +++ b/monetdb5/modules/mal/profiler.h @@ -82,4 +82,6 @@ profiler_export str CMDsetMemoryFlag( Cl profiler_export str CMDgetMemory( Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); profiler_export str CMDgetFootprint( Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); profiler_export str CMDtomograph(int *ret); +profiler_export str CMDcpustats(lng *user, lng *nice, lng *sys, lng *idle, lng *iowait); +profiler_export str CMDcpuloadPercentage(int *cycles, int *io, lng *user, lng *nice, lng *sys, lng *idle, lng *iowait); #endif /* _PROFILER_*/ diff --git a/monetdb5/modules/mal/profiler.mal b/monetdb5/modules/mal/profiler.mal --- a/monetdb5/modules/mal/profiler.mal +++ b/monetdb5/modules/mal/profiler.mal @@ -136,3 +136,11 @@ comment "Get the amount of memory claime command tomograph() address CMDtomograph comment "No-op operation, merely a signal to the tomograph package"; + +command cpustats()(user:lng, nice:lng, sys:lng,idle:lng,iowait:lng) +address CMDcpustats +comment "Extract cpu statistics from the kernel"; + +command cpuload(user:lng, nice:lng, sys:lng,idle:lng,iowait:lng)(cycles:int,io:int) +address CMDcpuloadPercentage +comment "Calculate the average cpu load percentage and io waiting times"; diff --git a/monetdb5/modules/mal/querylog.c b/monetdb5/modules/mal/querylog.c new file mode 100644 --- /dev/null +++ b/monetdb5/modules/mal/querylog.c @@ -0,0 +1,329 @@ +/* + * The contents of this file are subject to the MonetDB Public License + * Version 1.1 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://www.monetdb.org/Legal/MonetDBLicense + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + * License for the specific language governing rights and limitations + * under the License. + * + * The Original Code is the MonetDB Database System. + * + * The Initial Developer of the Original Code is CWI. + * Portions created by CWI are Copyright (C) 1997-July 2008 CWI. + * Copyright August 2008-2013 MonetDB B.V. + * All Rights Reserved. +*/ +/* + * Martin Kersten + * Language Extensions + * Iterators over scalar ranges are often needed, also at the MAL level. + * The barrier and control primitives are sufficient to mimic them directly. + * + * The modules located in the kernel directory should not + * rely on the MAL datastructures. That's why we have to deal with + * some bat operations here and delegate the signature to the + * proper module upon loading. + * + * Running a script is typically used to initialize a context. + * Therefore we need access to the runtime context. + * For the call variants we have + * to determine an easy way to exchange the parameter/return values. + */ + +#include "monetdb_config.h" +#include "querylog.h" + +/* (c) M.L. Kersten + * The query logger facility is hardwired to avoid interference + * with the SQL transaction manager. + * + * The events being captured are stored in separate BATs. + * They are made persistent to accumulate information over + * multiple sessions. This means it has to be explicitly reset + * to avoid disc overflow using querylog.reset(). +create table querylog.catalog( + id oid, + "user" string, -- owner of the query + defined timestamp, -- when entered into the cache + query string, + pipe string, -- optimizer pipe-line deployed + optimize bigint -- time in usec +); +create table querylog.calls( + id oid, + "start" timestamp, -- time the statement was started + "stop" timestamp, -- time the statement was completely finished + arguments string, + tuples wrd, -- number of tuples in the result set + exec bigint, -- time spent (in usec) until the result export + result bigint, -- time spent (in usec) to ship the result set + cpuload int, -- average cpu load percentage during execution + iowait int, -- time waiting for IO to finish in usec + space bigint -- total storage size of intermediates created (in MB) +); +*/ + +static int QLOGtrace = 0; +static int QLOG_init = 0; +static int QLOGthreshold = 0; + +static BAT *QLOG_cat_id = 0; +static BAT *QLOG_cat_user = 0; +static BAT *QLOG_cat_defined = 0; +static BAT *QLOG_cat_query = 0; +static BAT *QLOG_cat_pipe = 0; +static BAT *QLOG_cat_optimize = 0; + +static BAT *QLOG_calls_id = 0; +static BAT *QLOG_calls_start = 0; +static BAT *QLOG_calls_stop = 0; +static BAT *QLOG_calls_arguments = 0; +static BAT *QLOG_calls_tuples = 0; +static BAT *QLOG_calls_exec = 0; +static BAT *QLOG_calls_result = 0; +static BAT *QLOG_calls_cpuload = 0; +static BAT *QLOG_calls_iowait = 0; +static BAT *QLOG_calls_space = 0; + +void +QLOGcatalog(BAT **r) +{ + int i; + for ( i=0;i < 6; i++) + r[i]=0; + if (initQlog()) + return ; + MT_lock_set(&mal_profileLock, "querylogLock"); + r[0] = BATcopy(QLOG_cat_id, TYPE_oid, QLOG_cat_id->ttype, 0); + r[1] = BATcopy(QLOG_cat_user, TYPE_oid, QLOG_cat_user->ttype,0); + r[2] = BATcopy(QLOG_cat_defined, TYPE_oid, QLOG_cat_defined->ttype,0); + r[3] = BATcopy(QLOG_cat_query, TYPE_oid, QLOG_cat_query->ttype,0); + r[4] = BATcopy(QLOG_cat_pipe, TYPE_oid, QLOG_cat_pipe->ttype,0); + r[5] = BATcopy(QLOG_cat_optimize, TYPE_oid, QLOG_cat_optimize->ttype,0); + MT_lock_unset(&mal_profileLock, "querylogLock"); +} + +void +QLOGcalls(BAT **r) +{ + int i; + for ( i=0;i < 10; i++) + r[i]=0; + if (initQlog()) + return ; + MT_lock_set(&mal_profileLock, "querylogLock"); + r[0] = BATcopy(QLOG_calls_id, TYPE_oid, QLOG_calls_id->ttype, 0); + r[1] = BATcopy(QLOG_calls_start, TYPE_oid, QLOG_calls_start->ttype,0); + r[2] = BATcopy(QLOG_calls_stop, TYPE_oid, QLOG_calls_stop->ttype,0); + r[3] = BATcopy(QLOG_calls_arguments, TYPE_oid, QLOG_calls_arguments->ttype,0); + r[4] = BATcopy(QLOG_calls_tuples, TYPE_oid, QLOG_calls_tuples->ttype,0); + r[5] = BATcopy(QLOG_calls_exec, TYPE_oid, QLOG_calls_exec->ttype,0); + r[6] = BATcopy(QLOG_calls_result, TYPE_oid, QLOG_calls_result->ttype,0); + r[7] = BATcopy(QLOG_calls_cpuload, TYPE_oid, QLOG_calls_cpuload->ttype,0); + r[8] = BATcopy(QLOG_calls_iowait, TYPE_oid, QLOG_calls_iowait->ttype,0); + r[9] = BATcopy(QLOG_calls_space, TYPE_oid, QLOG_calls_space->ttype,0); + MT_lock_unset(&mal_profileLock, "querylogLock"); +} + +static bat commitlist[32]; +static int committop=1; + +static BAT * +QLOGcreate(str hnme, str tnme, int tt) +{ + BAT *b; + char buf[128]; + _______________________________________________ checkin-list mailing list [email protected] http://mail.monetdb.org/mailman/listinfo/checkin-list
