Changeset: 2b0530250203 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2b0530250203
Modified Files:
monetdb5/mal/mal_client.h
monetdb5/mal/mal_parser.c
monetdb5/modules/mal/wlcr.c
monetdb5/modules/mal/wlcr.h
monetdb5/modules/mal/wlcr.mal
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
sql/backends/monet5/sql_wlcr.c
sql/backends/monet5/sql_wlcr.h
sql/backends/monet5/sql_wlcr.mal
sql/scripts/60_wlcr.sql
Branch: wlcr
Log Message:
Get the WLCR log ready for execution at the clone
diffs (truncated from 786 to 300 lines):
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -175,6 +175,7 @@ typedef struct CLIENT {
* The workload for replication/replay is saved initially as a MAL
block.
*/
int wlcr_kind;
+ int wlcr_mode;
MalBlkPtr wlcr;
/*
* Errors during copy into are collected in a user specific column
set
diff --git a/monetdb5/mal/mal_parser.c b/monetdb5/mal/mal_parser.c
--- a/monetdb5/mal/mal_parser.c
+++ b/monetdb5/mal/mal_parser.c
@@ -1753,9 +1753,10 @@ parseMAL(Client cntxt, Symbol curPrg, in
echoInput(cntxt);
/* here the work takes place */
- while ((c = currChar(cntxt)) && lines-- > 0) {
+ while ((c = currChar(cntxt)) && lines > 0) {
switch (c) {
case '\n': case '\r': case '\f':
+ lines -= c =='\n';
nextChar(cntxt);
echoInput(cntxt);
continue;
diff --git a/monetdb5/modules/mal/wlcr.c b/monetdb5/modules/mal/wlcr.c
--- a/monetdb5/modules/mal/wlcr.c
+++ b/monetdb5/modules/mal/wlcr.c
@@ -71,7 +71,8 @@ static MT_Lock wlcr_lock MT_LOCK_INI
int wlcr_threshold = 0; // threshold (milliseconds) for keeping readonly
queries
-int wlcr_batch = 0; // last batch jon identifier
+int wlcr_batch = 0; // last batch job identifier
+int wlcr_start = 0; // first batch to check next
int wlcr_tid = 0; // last transaction id
static char *wlcr_name[]= {"","query","update","catalog"};
@@ -95,14 +96,14 @@ WLCRloggerfile(Client cntxt)
FILE *fd;
(void) cntxt;
- wlcr_batch++;
- wlcr_tid = 0;
snprintf(path,PATHLENGTH,"%s%cwlcr_%06d",wlcr_dir,DIR_SEP,wlcr_batch);
mnstr_printf(cntxt->fdout,"#WLCRloggerfile batch %s\n",path);
wlcr_fd = open_wastream(path);
if( wlcr_fd == 0)
throw(MAL,"wlcr.logger","Could not create %s\n",path);
+ wlcr_batch++;
+ wlcr_tid = 0;
snprintf(path,PATHLENGTH,"%s%cwlcr",wlcr_dir, DIR_SEP);
mnstr_printf(cntxt->fdout,"#WLCRloggerfile %s\n",wlcr_dir);
fd = fopen(path,"w");
@@ -116,6 +117,7 @@ WLCRloggerfile(Client cntxt)
/*
* The existence of the master directory should be checked upon server restart.
* A new batch file should be created as a result.
+ * We also have to keep track on the files that have been read by the clone
from the parent.
*/
str
WLCRinit(Client cntxt)
@@ -135,6 +137,7 @@ WLCRinit(Client cntxt)
dir = GDKfilepath(0,0,"master",0);
snprintf(path, PATHLENGTH,"%s%cwlcr",dir, DIR_SEP);
mnstr_printf(cntxt->fdout,"#Testing WLCR %s\n", path);
+ wlcr_start = 0;
fd = fopen(path,"r");
if( fd){
// database is in master tracking mode
@@ -202,9 +205,13 @@ static InstrPtr
WLCRaddtime(Client cntxt, InstrPtr pci, InstrPtr p)
{
char tbuf[26];
- time_t clk = pci->clock.tv_sec;
+ struct timeval clock;
+ time_t clk ;
struct tm ctm;
+ (void) pci;
+ gettimeofday(&clock,NULL);
+ clk = clock.tv_sec;
ctm = *localtime(&clk);
strftime(tbuf, 26, "%Y-%m-%dT%H:%M:%S",&ctm);
return pushStr(cntxt->wlcr, p, tbuf);
@@ -219,7 +226,7 @@ WLCRaddtime(Client cntxt, InstrPtr pci,
s->def = NULL;\
} \
if( cntxt->wlcr->stop == 0){\
- p = newStmt(cntxt->wlcr,"wlreplay","job");\
+ p = newStmt(cntxt->wlcr,"clone","job");\
p = pushStr(cntxt->wlcr,p, cntxt->username);\
p = pushInt(cntxt->wlcr,p, wlcr_tid);\
p = WLCRaddtime(cntxt,pci, p); \
@@ -240,7 +247,7 @@ WLCRjob(Client cntxt, MalBlkPtr mb, MalS
}
str
-WLCRfin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+WLCRexec(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
(void) cntxt;
(void) mb;
@@ -257,7 +264,7 @@ WLCRquery(Client cntxt, MalBlkPtr mb, Ma
if ( strcmp("-- no query",getVarConstant(mb, getArg(pci,1)).val.sval)
== 0)
return MAL_SUCCEED; // ignore system internal queries.
WLCR_start();
- p = newStmt(cntxt->wlcr, "wlreplay","query");
+ p = newStmt(cntxt->wlcr, "clone","query");
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,1)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,2)).val.sval);
p->ticks = GDKms();
@@ -271,7 +278,7 @@ WLCRgeneric(Client cntxt, MalBlkPtr mb,
(void) stk;
WLCR_start();
- p = newStmt(cntxt->wlcr, "wlreplay",getFunctionId(pci));
+ p = newStmt(cntxt->wlcr, "clone",getFunctionId(pci));
for( i = pci->retc; i< pci->argc; i++){
tpe =getArgType(mb, pci, i);
switch(tpe){
@@ -294,7 +301,7 @@ WLCRgeneric(Client cntxt, MalBlkPtr mb,
int k=0; \
for( ; p < q; p++, k++){\
if( k % 32 == 31){\
- pci = newStmt(cntxt->wlcr,
"wlreplay",getFunctionId(pci));\
+ pci = newStmt(cntxt->wlcr, "clone",getFunctionId(pci));\
pci = pushStr(cntxt->wlcr, pci, sch);\
pci = pushStr(cntxt->wlcr, pci, tbl);\
pci = pushStr(cntxt->wlcr, pci, col);\
@@ -339,7 +346,7 @@ WLCRdatashipping(Client cntxt, MalBlkPtr
bi= bat_iterator(b);
BATloop(b,p,q){
if( k % 32 == 31){
- pci = newStmt(cntxt->wlcr,
"wlreplay",getFunctionId(pci));
+ pci = newStmt(cntxt->wlcr,
"clone",getFunctionId(pci));
pci = pushStr(cntxt->wlcr, pci, sch);
pci = pushStr(cntxt->wlcr, pci, tbl);
pci = pushStr(cntxt->wlcr, pci, col);
@@ -365,7 +372,7 @@ WLCRappend(Client cntxt, MalBlkPtr mb, M
(void) mb;
WLCR_start();
- p = newStmt(cntxt->wlcr, "wlreplay","append");
+ p = newStmt(cntxt->wlcr, "clone","append");
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,1)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,2)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,3)).val.sval);
@@ -399,7 +406,7 @@ WLCRdelete(Client cntxt, MalBlkPtr mb, M
(void) mb;
WLCR_start();
- p = newStmt(cntxt->wlcr, "wlreplay","delete");
+ p = newStmt(cntxt->wlcr, "clone","delete");
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,1)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,2)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,3)).val.sval);
@@ -428,7 +435,7 @@ WLCRupdate(Client cntxt, MalBlkPtr mb, M
(void) stk;
WLCR_start();
- p = newStmt(cntxt->wlcr, "wlreplay","updateOID");
+ p = newStmt(cntxt->wlcr, "clone","updateOID");
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,1)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,2)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,3)).val.sval);
@@ -445,7 +452,7 @@ WLCRupdate(Client cntxt, MalBlkPtr mb, M
}
}
- p = newStmt(cntxt->wlcr, "wlreplay","updateVALUE");
+ p = newStmt(cntxt->wlcr, "clone","updateVALUE");
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,1)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,2)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,3)).val.sval);
@@ -473,7 +480,7 @@ WLCRclear_table(Client cntxt, MalBlkPtr
(void) stk;
WLCR_start();
- p = newStmt(cntxt->wlcr, "wlr","clear_table");
+ p = newStmt(cntxt->wlcr, "clone","clear_table");
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,1)).val.sval);
p = pushStr(cntxt->wlcr, p, getVarConstant(mb, getArg(pci,2)).val.sval);
p->ticks = GDKms();
@@ -504,7 +511,7 @@ WLCRwrite(Client cntxt, str kind)
if(cntxt->wlcr->stop == 0)
return MAL_SUCCEED;
- newStmt(cntxt->wlcr,"wlreplay","fin");
+ newStmt(cntxt->wlcr,"clone","exec");
wlcr_tid++;
MT_lock_set(&wlcr_lock);
p = getInstrPtr(cntxt->wlcr,0);
diff --git a/monetdb5/modules/mal/wlcr.h b/monetdb5/modules/mal/wlcr.h
--- a/monetdb5/modules/mal/wlcr.h
+++ b/monetdb5/modules/mal/wlcr.h
@@ -28,7 +28,7 @@ mal_export str WLCRinit(Client cntxt);
mal_export str WLCRinitCmd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
mal_export str WLCRmaster(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
mal_export str WLCRjob(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
-mal_export str WLCRfin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
+mal_export str WLCRexec(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
mal_export str WLCRquery(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
mal_export str WLCRgeneric(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
mal_export str WLCRappend(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
diff --git a/monetdb5/modules/mal/wlcr.mal b/monetdb5/modules/mal/wlcr.mal
--- a/monetdb5/modules/mal/wlcr.mal
+++ b/monetdb5/modules/mal/wlcr.mal
@@ -18,12 +18,12 @@ pattern master(threshold:int)
address WLCRmaster
comment "Activate the workload-capture-replay. Only queries surpassing the
threshold are kept for replay.";
-pattern job(user:str, tid:int, started:str, runtime:int)
+pattern job(user:str, tid:int, started:str, action:str, kind:str, runtime:int)
address WLCRjob
comment "Mark the beginning of the work unit which can be a compound
transaction";
-pattern fin()
-address WLCRfinished
+pattern exec()
+address WLCRexec
comment "Mark the end of the work unit";
pattern query(q:str,pipe:str)
diff --git a/monetdb5/optimizer/opt_prelude.c b/monetdb5/optimizer/opt_prelude.c
--- a/monetdb5/optimizer/opt_prelude.c
+++ b/monetdb5/optimizer/opt_prelude.c
@@ -56,6 +56,7 @@ str betweensymmetricRef;
str blockRef;
str bbpRef;
str tidRef;
+str cloneRef;
str dateRef;
str deltaRef;
str subdeltaRef;
@@ -378,6 +379,7 @@ void optimizerInit(void)
create_indexRef = putName("create_index");
create_functionRef = putName("create_function");
create_triggerRef = putName("create_trigger");
+ cloneRef = putName("clone");
dateRef = putName("date");
dataflowRef = putName("dataflow");
dblRef = putName("dbl");
diff --git a/monetdb5/optimizer/opt_prelude.h b/monetdb5/optimizer/opt_prelude.h
--- a/monetdb5/optimizer/opt_prelude.h
+++ b/monetdb5/optimizer/opt_prelude.h
@@ -67,6 +67,7 @@ mal_export str bpmRef;
mal_export str bstreamRef;
mal_export str calcRef;
mal_export str clear_tableRef;
+mal_export str cloneRef;
mal_export str closeRef;
mal_export str columnRef;
mal_export str commitRef;
diff --git a/sql/backends/monet5/sql_wlcr.c b/sql/backends/monet5/sql_wlcr.c
--- a/sql/backends/monet5/sql_wlcr.c
+++ b/sql/backends/monet5/sql_wlcr.c
@@ -23,12 +23,13 @@
#include "sql.h"
#include "wlcr.h"
#include "sql_wlcr.h"
+#include "sql_scenario.h"
+#include "opt_prelude.h"
#include "mal_parser.h"
#include "mal_client.h"
#define WLCR_REPLAY 1
-#define WLCR_SYNC 2
-static int wlcr_mode;
+#define WLCR_CLONE 2
static str wlcr_master;
static int wlcr_replaythreshold;
@@ -75,14 +76,22 @@ WLCRreplayinit(Client cntxt, MalBlkPtr m
return MAL_SUCCEED;
}
+/*
+ * Run the clone actions under a new client
+ * and safe debugging in a tmp file.
+ */
void
WLCRprocess(void *arg)
{
Client cntxt = (Client) arg;
- int i;
+ int i, pc;
char path[PATHLENGTH];
stream *fd;
Client c;
+ size_t sz;
+ MalBlkPtr mb;
+ InstrPtr q;
+ str msg;
c =MCforkClient(cntxt);
if( c == 0){
@@ -92,30 +101,64 @@ WLCRprocess(void *arg)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
