Changeset: 9ddd4ae15f5e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9ddd4ae15f5e
Added Files:
sql/test/wlcr/Tests/wlcr4.py
Modified Files:
monetdb5/modules/mal/wlcr.c
monetdb5/modules/mal/wlcr.h
monetdb5/modules/mal/wlcr.mal
sql/backends/monet5/sql_wlcr.c
sql/backends/monet5/sql_wlcr.h
sql/backends/monet5/sql_wlcr.mal
sql/scripts/60_wlcr.sql
sql/test/wlcr/Tests/All
sql/test/wlcr/Tests/wlcr1.py
sql/test/wlcr/Tests/wlcr2.py
sql/test/wlcr/Tests/wlcr2.stable.out
sql/test/wlcr/Tests/wlcr3.py
Branch: wlcr
Log Message:
Minor changes
Open issue, properly continue the replica when more log
files are available
diffs (truncated from 365 to 300 lines):
diff --git a/monetdb5/modules/mal/wlcr.c b/monetdb5/modules/mal/wlcr.c
--- a/monetdb5/modules/mal/wlcr.c
+++ b/monetdb5/modules/mal/wlcr.c
@@ -24,7 +24,7 @@
* global file system.
*
* A database can be set once into 'master' mode only once using the SQL
command:
- * CALL setmaster()
+ * CALL master()
*
* It creates a directory .../dbfarm/dbname/master to hold all necessary
information
* for the creation and maintenance of replicas.
@@ -276,7 +276,7 @@ WLCthreshold(Client cntxt, MalBlkPtr mb,
}
str
-WLCsetmaster(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+WLCmaster(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
char path[PATHLENGTH];
str msg = MAL_SUCCEED;
@@ -295,7 +295,7 @@ WLCsetmaster(Client cntxt, MalBlkPtr mb,
wlcr_archive = GDKfilepath(0,0,"master",0);
snprintf(path, PATHLENGTH,"%s%c%s_wlcr", wlcr_archive, DIR_SEP,
wlcr_dbname);
if( GDKcreatedir(path) == GDK_FAIL)
- throw(SQL,"wlcr.setmaster","Could not create %s\n",
wlcr_archive);
+ throw(SQL,"wlcr.master","Could not create %s\n",
wlcr_archive);
#ifdef _WLC_DEBUG_
mnstr_printf(cntxt->fdout,"#Snapshot directory '%s'\n",
wlcr_archive);
#endif
diff --git a/monetdb5/modules/mal/wlcr.h b/monetdb5/modules/mal/wlcr.h
--- a/monetdb5/modules/mal/wlcr.h
+++ b/monetdb5/modules/mal/wlcr.h
@@ -31,7 +31,7 @@ mal_export str WLCexit(void);
mal_export int WLCused(void);
mal_export str WLCgetConfig(void);
mal_export str WLCinitCmd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
-mal_export str WLCsetmaster(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
+mal_export str WLCmaster(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
mal_export str WLCstopmaster(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
mal_export str WLCthreshold(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
mal_export str WLCjob(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
diff --git a/monetdb5/modules/mal/wlcr.mal b/monetdb5/modules/mal/wlcr.mal
--- a/monetdb5/modules/mal/wlcr.mal
+++ b/monetdb5/modules/mal/wlcr.mal
@@ -10,8 +10,8 @@ pattern init()
address WLCinitCmd
comment "Test for running as master";
-pattern setmaster()
-address WLCsetmaster
+pattern master()
+address WLCmaster
comment "Activate the workload-capture-replay";
pattern commit()
diff --git a/sql/backends/monet5/sql_wlcr.c b/sql/backends/monet5/sql_wlcr.c
--- a/sql/backends/monet5/sql_wlcr.c
+++ b/sql/backends/monet5/sql_wlcr.c
@@ -105,7 +105,7 @@ WLRgetMaster(str dbname)
fd = fopen(dir,"r");
if( fd == NULL){
GDKfree(dir);
- throw(SQL,"getMaster","Database '%s' not acting as a
master",dbname);
+ throw(SQL,"getMaster","Database '%s' not acting as a
master",dir);
}
(void) fclose(fd);
@@ -150,7 +150,7 @@ WLRinitReplica(str dbname)
fd = fopen(dir,"r");
if( fd ){
(void) fclose(fd);
- throw(SQL,"setreplica","Already in replica mode '%s'",dbname);
+ throw(SQL,"setreplica","Already in replica mode for
'%s'",dbname);
}
GDKfree(dir);
@@ -186,6 +186,10 @@ WLRgetThreshold( Client cntxt, MalBlkPtr
return MAL_SUCCEED;
}
+/*
+ * Run once through the list of pending WLCR logs
+ * Continuing where you left off the previous time.
+ */
void
WLCRprocess(void *arg)
{
@@ -228,11 +232,13 @@ WLCRprocess(void *arg)
wlcr_archive, wlr_firstbatch, wlr_lastbatch, wlr_threshold);
#endif
wlr_tag = 0;
- for( i= wlr_lastbatch; i < wlcr_lastbatch; i++){
+ for( i= wlr_lastbatch; i < wlcr_lastbatch && ! GDKexiting(); i++){
snprintf(path,PATHLENGTH,"%s%c%s_%012d", wlr_master, DIR_SEP,
wlr_dbname, i);
fd= open_rstream(path);
if( fd == NULL){
mnstr_printf(GDKerr,"#wlcr.process:'%s' can not be
accessed \n",path);
+ // Be careful not to miss log files.
+ // In the future wait for more files becoming available.
continue;
}
sz = getFileSize(fd);
@@ -297,21 +303,28 @@ WLCRprocess(void *arg)
}
str
-WLCRsetreplica(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+WLCRreplicate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{ str msg;
+ bit waitfor= 0;
MT_Id wlcr_thread;
(void) mb;
if( cntxt->wlcr_mode == WLCR_REPLICATE || cntxt->wlcr_mode ==
WLCR_REPLAY){
- throw(SQL,"wlcr.setreplica","System already in synchronization
mode");
+ throw(SQL,"wlcr.replicate","System already in synchronization
mode");
}
msg = WLRinitReplica( *getArgReference_str(stk,pci,1));
if( msg)
return msg;
+ if ( pci->argc ==3)
+ waitfor = *getArgReference_bit(stk,pci,2);
cntxt->wlcr_mode = WLCR_REPLICATE;
+ // For testing it is helpful to wait for completion of the replication
process
+ if( waitfor)
+ WLCRprocess(cntxt);
+ // start the process for continual integration in the background
if (MT_create_thread(&wlcr_thread, WLCRprocess, (void*) cntxt,
MT_THR_JOINABLE) < 0) {
- throw(SQL,"wlcr.setreplica","replay process can not be
started\n");
+ throw(SQL,"wlcr.replicate","replay process can not be
started\n");
}
return MAL_SUCCEED;
}
diff --git a/sql/backends/monet5/sql_wlcr.h b/sql/backends/monet5/sql_wlcr.h
--- a/sql/backends/monet5/sql_wlcr.h
+++ b/sql/backends/monet5/sql_wlcr.h
@@ -19,7 +19,7 @@
*/
extern void WLCRprocess(void *arg);
extern void WLRinit(Client cntxt);
-extern str WLCRsetreplica(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
+extern str WLCRreplicate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
extern str WLRjob(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
extern str WLRexec(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
diff --git a/sql/backends/monet5/sql_wlcr.mal b/sql/backends/monet5/sql_wlcr.mal
--- a/sql/backends/monet5/sql_wlcr.mal
+++ b/sql/backends/monet5/sql_wlcr.mal
@@ -6,9 +6,13 @@
module wlcr;
-pattern wlcr.setreplica(dbname:str)
-address WLCRsetreplica
-comment "Roll the snapshot forward to a uptodate clone";
+pattern wlcr.replicate(dbname:str)
+address WLCRreplicate
+comment "Roll the snapshot forward to an up-to-date clone";
+
+pattern wlcr.replicate(dbname:str, waitformaster:bit)
+address WLCRreplicate
+comment "Roll the snapshot forward to an up-to-date clone and wait for it";
module wlr;
pattern job(user:str, tid:int, started:str, action:str, kind:str, runtime:lng)
diff --git a/sql/scripts/60_wlcr.sql b/sql/scripts/60_wlcr.sql
--- a/sql/scripts/60_wlcr.sql
+++ b/sql/scripts/60_wlcr.sql
@@ -9,12 +9,15 @@
declare replaythreshold integer;
set replaythreshold = -1; -- don't replay
-create procedure setmaster()
-external name wlcr.setmaster;
+create procedure master()
+external name wlcr.master;
create procedure stopmaster()
external name wlcr.stopmaster;
-create procedure setreplica(dbname string)
-external name wlcr.setreplica;
+create procedure replicate(dbname string)
+external name wlcr.replicate;
+create procedure replicate(dbname string, waitforsync bool)
+external name wlcr.replicate;
+
diff --git a/sql/test/wlcr/Tests/All b/sql/test/wlcr/Tests/All
--- a/sql/test/wlcr/Tests/All
+++ b/sql/test/wlcr/Tests/All
@@ -1,3 +1,4 @@
wlcr1
wlcr2
wlcr3
+wlcr4
diff --git a/sql/test/wlcr/Tests/wlcr1.py b/sql/test/wlcr/Tests/wlcr1.py
--- a/sql/test/wlcr/Tests/wlcr1.py
+++ b/sql/test/wlcr/Tests/wlcr1.py
@@ -18,7 +18,7 @@ s = process.server(dbname = dbname, stdi
c = process.client('sql', dbname = dbname, stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE)
cout, cerr = c.communicate('''\
-call setmaster();
+call master();
select 1;
create table tmp(i int, s string);
insert into tmp values(1,'hello'), (2,'world');
diff --git a/sql/test/wlcr/Tests/wlcr2.py b/sql/test/wlcr/Tests/wlcr2.py
--- a/sql/test/wlcr/Tests/wlcr2.py
+++ b/sql/test/wlcr/Tests/wlcr2.py
@@ -18,8 +18,7 @@ s = process.server(dbname = dbname, stdi
c = process.client('sql', dbname = dbname, stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE)
cout, cerr = c.communicate('''\
-update tmp set i = 3 where i = 1;
-update tmp set s = 'blah';
+insert into tmp values(3,'blah'),(4,'bloh');
select * from tmp;
''')
diff --git a/sql/test/wlcr/Tests/wlcr2.stable.out
b/sql/test/wlcr/Tests/wlcr2.stable.out
--- a/sql/test/wlcr/Tests/wlcr2.stable.out
+++ b/sql/test/wlcr/Tests/wlcr2.stable.out
@@ -17,17 +17,17 @@ stdout of test 'wlcr2` in directory 'sql
# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-25896/.s.monetdb.37590
# MonetDB/GIS module loaded
# MonetDB/SQL module loaded
-#update tmp set i = 3 where i = 1;
-[ 1 ]
-#update tmp set s = 'blah';
+#insert into tmp values(3,'blah'),(4,'bloh');
[ 2 ]
#select * from tmp;
% sys.tmp, sys.tmp # table_name
% i, s # name
% int, clob # type
-% 1, 4 # length
+% 1, 5 # length
+[ 1, "hello" ]
+[ 2, "world" ]
[ 3, "blah" ]
-[ 2, "blah" ]
+[ 4, "bloh" ]
# 11:51:22 >
# 11:51:22 > "Done."
diff --git a/sql/test/wlcr/Tests/wlcr3.py b/sql/test/wlcr/Tests/wlcr3.py
--- a/sql/test/wlcr/Tests/wlcr3.py
+++ b/sql/test/wlcr/Tests/wlcr3.py
@@ -2,9 +2,8 @@ try:
from MonetDBtesting import process
except ImportError:
import process
-import os, sys, socket
+import os, sys
-process.verbose = True
dbfarm = os.getenv('GDK_DBFARM')
tstdb = os.getenv('TSTDB')
@@ -12,39 +11,21 @@ if not tstdb or not dbfarm:
print 'No TSTDB or GDK_DBFARM in environment'
sys.exit(1)
-def freeport():
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- sock.bind(('', 0))
- port = sock.getsockname()[1]
- sock.close()
- return port
+dbname = tstdb
-cloneport = freeport()
+s = process.server(dbname = dbname, stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE)
-dbname = tstdb
-dbnameclone = tstdb + '-clone'
-
-# clean up before we start
-if os.path.exists(os.path.join(dbfarm, dbnameclone)):
- import shutil
- shutil.rmtree(os.path.join(dbfarm, dbnameclone))
-
-master = process.server(dbname = dbname, stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE)
-slave = process.server(dbname = dbnameclone, mapiport = cloneport, stdin =
process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
-
-c = process.client('sql', dbname = dbnameclone, port = cloneport, stdin =
process.PIPE, stdout = process.PIPE, stderr = process.PIPE)
+c = process.client('sql', dbname = dbname, stdin = process.PIPE, stdout =
process.PIPE, stderr = process.PIPE)
cout, cerr = c.communicate('''\
-call clone('%s');
+update tmp set i = 3 where i = 1;
+update tmp set s = 'blah';
select * from tmp;
-''' % dbname)
+''')
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
