Changeset: 532040f4f0ff for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=532040f4f0ff
Added Files:
monetdb5/modules/mal/wlc.c
monetdb5/modules/mal/wlc.h
monetdb5/modules/mal/wlc.mal
monetdb5/optimizer/opt_wlc.c
monetdb5/optimizer/opt_wlc.h
sql/backends/monet5/wlr.c
sql/backends/monet5/wlr.h
sql/backends/monet5/wlr.mal
Removed Files:
monetdb5/modules/mal/wlcr.c
monetdb5/modules/mal/wlcr.h
monetdb5/modules/mal/wlcr.mal
monetdb5/optimizer/opt_wlcr.c
monetdb5/optimizer/opt_wlcr.h
sql/backends/monet5/sql_wlcr.c
sql/backends/monet5/sql_wlcr.h
sql/backends/monet5/sql_wlcr.mal
Modified Files:
monetdb5/mal/mal.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/mal_init.mal
monetdb5/optimizer/Makefile.ag
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_wrapper.c
monetdb5/optimizer/optimizer.mal
sql/backends/monet5/40_sql.mal
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql_scenario.c
sql/scripts/60_wlcr.sql
sql/server/sql_mvc.c
sql/test/wlcr/Tests/wlc01.py
Branch: wlcr
Log Message:
Simplify the interface
- use a heartbeat approach
- split wlcr naming according to their role
- update documentation
diffs (truncated from 1626 to 300 lines):
diff --git a/monetdb5/mal/mal.c b/monetdb5/mal/mal.c
--- a/monetdb5/mal/mal.c
+++ b/monetdb5/mal/mal.c
@@ -36,7 +36,7 @@ int have_hge;
#include "mal_runtime.h"
#include "mal_resource.h"
#include "opt_statistics.h"
-#include "wlcr.h"
+#include "wlc.h"
MT_Lock mal_contextLock MT_LOCK_INITIALIZER("mal_contextLock");
MT_Lock mal_namespaceLock MT_LOCK_INITIALIZER("mal_namespaceLock");
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -242,8 +242,8 @@ MCinitClientRecord(Client c, oid user, b
/* create a recycler cache */
c->exception_buf_initialized = 0;
c->error_row = c->error_fld = c->error_msg = c->error_input = NULL;
- c->wlcr_kind = 0;
- c->wlcr = NULL;
+ c->wlc_kind = 0;
+ c->wlc = NULL;
#ifndef HAVE_EMBEDDED /* no authentication in embedded mode */
{
str msg = AUTHgetUsername(&c->username, c);
@@ -396,10 +396,10 @@ freeClient(Client c)
BBPrelease(c->error_msg->batCacheid);
BBPrelease(c->error_input->batCacheid);
c->error_row = c->error_fld = c->error_msg = c->error_input =
NULL;
- if( c->wlcr)
- freeMalBlk(c->wlcr);
- c->wlcr_kind = 0;
- c->wlcr = NULL;
+ if( c->wlc)
+ freeMalBlk(c->wlc);
+ c->wlc_kind = 0;
+ c->wlc = NULL;
}
if (t)
THRdel(t); /* you may perform suicide */
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -176,8 +176,8 @@ typedef struct CLIENT {
* It is split into the capturing part (wlc) and the replay part (wlr).
* This allows a single server to act as both a master and a replica.
*/
- int wlcr_kind; // used by master to characterise the compound
transaction
- MalBlkPtr wlcr;
+ int wlc_kind; // used by master to characterise the compound
transaction
+ MalBlkPtr wlc;
/*
* Errors during copy into are collected in a user specific column
set
diff --git a/monetdb5/modules/mal/Makefile.ag b/monetdb5/modules/mal/Makefile.ag
--- a/monetdb5/modules/mal/Makefile.ag
+++ b/monetdb5/modules/mal/Makefile.ag
@@ -36,7 +36,7 @@ lib_mal = {
mkey.c mkey.h \
manifold.c manifold.h \
oltp.c oltp.h \
- wlcr.c wlcr.h \
+ wlc.c wlc.h \
pcre.c \
profiler.c profiler.h \
querylog.c querylog.h \
@@ -60,7 +60,7 @@ headers_mal = {
inspect.mal manual.mal mal_io.mal mkey.mal manifold.mal \
iterator.mal clients.mal \
factories.mal groupby.mal mdb.mal pcre.mal mat.mal \
- transaction.mal oltp.mal wlcr.mal \
+ transaction.mal oltp.mal wlc.mal \
mal_mapi.mal sabaoth.mal remote.mal \
txtsim.mal \
tokenizer.mal sample.mal json_util.mal \
diff --git a/monetdb5/modules/mal/mal_init.mal
b/monetdb5/modules/mal/mal_init.mal
--- a/monetdb5/modules/mal/mal_init.mal
+++ b/monetdb5/modules/mal/mal_init.mal
@@ -94,7 +94,7 @@ include srvpool;
include mal_mapi;
include oltp;
-include wlcr;
+include wlc;
# Any extensions (MAL scripts) that should be automatically loaded upon
# startup can be placed in the autoload directory. One typically finds
diff --git a/monetdb5/modules/mal/wlcr.c b/monetdb5/modules/mal/wlc.c
rename from monetdb5/modules/mal/wlcr.c
rename to monetdb5/modules/mal/wlc.c
--- a/monetdb5/modules/mal/wlcr.c
+++ b/monetdb5/modules/mal/wlc.c
@@ -23,16 +23,18 @@
* the snapshot until a specific time point or transaction id is reached.
*
* Some systems also use the logical logs to REPLAY all (expensive) queries
- * against the database. We skip this for the time being.
+ * against the database. We skip this for the time being, as those queries
+ * can be captured already in the server.
*
* The goal of this module is to ease BACKUP and REPPLICATION of a master
database
- * with a time-bounded delay.
- * Such a clone is a database replica that aid in query workload sharing,
- * database versioning, and (re-)partitioning.
- * Tables taken from the master version are not protected against local
updates.
- * However, any transaction being replay that fails finalizes the cloning
process.
+ * with a time-bounded delay. This means that both master and replica run at a
certain beat
+ * (in seconds) by which information is made available or read by the replica.
+ *
+ * Such a replica in query workload sharing, database versioning, and
(re-)partitioning.
+ * Tables taken from the master are not protected against local updates.
+ * However, any replay transaction that fails stops the cloning process.
* Furthermore, only persistent tables are considered for replication.
- * Tables under the 'tmp' schema are ignored.
+ * Updates under the 'tmp' schema are ignored.
*
* Simplicity and ease of end-user control has been the driving argument here.
*
@@ -45,46 +47,46 @@
* CALL master()
* An alternative path to the log records can be given to reduce the IO
latency,
* e.g. a nearby SSD.
- * By default, it creates a directory .../dbfarm/dbname/wlcr_logs to hold all
+ * By default, it creates a directory .../dbfarm/dbname/wlc_logs to hold all
* necessary information for the creation of a database clone.
*
* A master configuration file is added to the database directory to keep the
state/
* It contains the following key=value pairs:
* snapshot=<path to a snapshot directory>
- * logs=<path to the wlcr log directory>
+ * logs=<path to the wlc log directory>
* state=<started, stopped>
* batches=<next available batch file to be applied>
- * drift=<maximal delay before transactions are published as a
separate log, in seconds>
+ * beat=<maximal delay between log files, in seconds>
* write=<timestamp of the last transaction recorded>
*
* A missing path to the snapshot denotes that we can start the clone with an
empty database.
* The log files are stored as master/<dbname>_<batchnumber>. They belong to
the snapshot.
*
- * Each wlcr log file contains a serial log of committed compound transactions.
+ * Each wlc log file contains a serial log of committed compound transactions.
* The log records are represented as ordinary MAL statement blocks, which
* are executed in serial mode. (parallelism can be considered for large
updates later)
* Each transaction job is identified by a unique id, its starting time, and
the user responsible..
* The log-record should end with a commit.
*
- * A transaction log is created by the master. He decides when the log may be
globally used.
- * The trigger for this is the allowed 'drift'. A new transaction log file is
published when
- * the system has been collecting logs for some time (drift in seconds).
- * The drift determines the maximal window of transactions loss that is
permitted.
- * The maximum drift can be set using a SQL command, e.g.
- * CALL setmasterdrift(duration)
+ * A transaction log is created by the master using a heartbeat (in seconds).
+ * A new transaction log file is published when the system has been
+ * collecting logs for some time.
+ * The cycle time determines the drift of the replica, excluding its own speed
to catch up.
+ * The beat can be set using a SQL command, e.g.
+ * CALL masterbeat(duration)
* Setting it to zero leads to a log file per transaction and may cause a
large log directory.
* A default of 5 minutes should balance polling overhead in most practical
situations.
*
* A minor problem here is that we should ensure that the log file is closed
even if there
* are no transactions running. It is solved with a separate monitor thread,
which ensures
- * that the logs are flushed at least after 'drift' seconds since the first
logrecord was created.
+ * that the logs are flushed at least after 'beat' seconds since the first
logrecord was created.
* After closing, the replicas can see from the master configuration file that
a new log batch is available.
*
* The final step is to close stop transaction logging with the command
* CALL stopmaster().
* It typically is the end-of-life-time for a snapshot. For example, when
planning to do
* a large bulk load of the database, stopping logging avoids a double write
into the
- * database. The database can be brought back into wlcr mode using a fresh
snapshot.
+ * database. The database can be brought back into wlc mode using a fresh
snapshot.
*
*[TODO] A more secure way to set a database into master mode is to use the
command
* monetdb master <dbname> [ <optional snapshot path>]
@@ -122,30 +124,30 @@
*
* SELECT replicaClock();
* returns the timestamp of the last replicated transaction.
- * SELECT replicaBacklog();
- * returns the number of pending transactions to be in sink with master.
+ * SELECT replicaTick();
+ * returns the transaction id of the last replicated transaction.
* SELECT masterClock();
* return the timestamp of the last committed transaction in the master.
+ * SELECT masterTick();
+ * return the transaction id of the last committed transaction in the master.
*
* Any failure encountered during a log replay terminates the replication
process,
* leaving a message in the merovingian log configuration.
*
- * The wlcr files purposely have a textual format derived from the MAL
statements.
+ * The wlc files purposely have a textual format derived from the MAL
statements.
* This provides a stepping stone for remote execution later.
*
- * [TODO] consider the roll forward of SQL session variables, i.e.
optimizer_pipe (for now assume default pipe).
+ * [TODO] consider the roll forward of SQL session variables, i.e.
optimizer_pipe
* For updates we don't need special care for this.
- * [TODO] limit replication logs to persistent tables. Temporary tables should
never be forwarded to the clone.
*/
#include "monetdb_config.h"
#include <time.h>
#include "mal_builder.h"
-#include "wlcr.h"
+#include "wlc.h"
-MT_Lock wlcr_lock MT_LOCK_INITIALIZER("wlcr_lock");
+MT_Lock wlc_lock MT_LOCK_INITIALIZER("wlc_lock");
static char wlc_snapshot[PATHLENGTH]; // The location of the snapshot against
which the logs work
-static lng wlc_start= 0; // Start time of first
transaction
static stream *wlc_fd = 0;
// These properties are needed by the replica to direct the roll-forward.
@@ -155,13 +157,13 @@ lng wlc_id = 0; // next transaction
int wlc_state = 0; // The current status of the in the
life cycle
char wlc_write[26]; // The timestamp of the last committed
transaction
int wlc_batches = 0; // identifier of next batch
-int wlc_drift = 10; // maximal period covered by a single log file
in seconds
+int wlc_beat = 10; // maximal period covered by a single log file
in seconds
/* The database snapshots are binary copies of the dbfarm/database/bat
* New snapshots are created currently using the 'monetdb snapshot <db>'
command
* or a SQL procedure.
*
- * The wlcr logs are stored in the snapshot directory as a time-stamped list
+ * The wlc logs are stored in the snapshot directory as a time-stamped list
*/
int
@@ -186,8 +188,8 @@ WLCreadConfig(FILE *fd)
strncpy(wlc_write, path + 6, 26);
if( strncmp("batches=", path, 8) == 0)
wlc_batches = atoi(path+ 8);
- if( strncmp("drift=", path, 6) == 0)
- wlc_drift = atoi(path+ 6);
+ if( strncmp("beat=", path, 5) == 0)
+ wlc_beat = atoi(path+ 5);
if( strncmp("state=", path, 6) == 0)
wlc_state = atoi(path+ 6);
}
@@ -203,7 +205,7 @@ WLCgetConfig(void){
fd = fopen(l,"r");
GDKfree(l);
if( fd == NULL)
- throw(MAL,"wlcr.getConfig","Could not access %s\n",l);
+ throw(MAL,"wlc.getConfig","Could not access %s\n",l);
WLCreadConfig(fd);
return MAL_SUCCEED;
}
@@ -218,7 +220,7 @@ str WLCsetConfig(void){
fd = open_wastream(path);
GDKfree(path);
if( fd == NULL)
- throw(MAL,"wlcr.setConfig","Could not access wlc.config\n");
+ throw(MAL,"wlc.setConfig","Could not access wlc.config\n");
if( wlc_snapshot[0] )
mnstr_printf(fd,"snapshot=%s\n", wlc_snapshot);
mnstr_printf(fd,"logs=%s\n", wlc_dir);
@@ -226,7 +228,7 @@ str WLCsetConfig(void){
mnstr_printf(fd,"write=%s\n", wlc_write );
mnstr_printf(fd,"state=%d\n", wlc_state );
mnstr_printf(fd,"batches=%d\n", wlc_batches );
- mnstr_printf(fd,"drift=%d\n", wlc_drift );
+ mnstr_printf(fd,"beat=%d\n", wlc_beat );
close_stream(fd);
return MAL_SUCCEED;
}
@@ -239,20 +241,19 @@ WLCsetlogger(void)
char path[PATHLENGTH];
if( wlc_dir[0] == 0)
- throw(MAL,"wlcr.setlogger","Path not initalized");
- MT_lock_set(&wlcr_lock);
+ throw(MAL,"wlc.setlogger","Path not initalized");
+ MT_lock_set(&wlc_lock);
snprintf(path,PATHLENGTH,"%s%c%s_%012d", wlc_dir, DIR_SEP, wlc_name,
wlc_batches);
wlc_fd = open_wastream(path);
if( wlc_fd == 0){
- MT_lock_unset(&wlcr_lock);
- GDKerror("wlcr.logger:Could not create %s\n",path);
- throw(MAL,"wlcr.logger","Could not create %s\n",path);
+ MT_lock_unset(&wlc_lock);
+ GDKerror("wlc.logger:Could not create %s\n",path);
+ throw(MAL,"wlc.logger","Could not create %s\n",path);
}
wlc_batches++;
- wlc_start = GDKms()/1000;
WLCsetConfig();
- MT_lock_unset(&wlcr_lock);
+ MT_lock_unset(&wlc_lock);
return MAL_SUCCEED;
}
@@ -265,42 +266,39 @@ WLCcloselogger(void)
mnstr_fsync(wlc_fd);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
