MonetDB: newstorage - cleanup
Changeset: 4b84361882a0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4b84361882a0
Modified Files:
sql/benchmarks/tpch/load.sh
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_storage.h
sql/storage/store.c
Branch: newstorage
Log Message:
cleanup
diffs (149 lines):
diff --git a/sql/benchmarks/tpch/load.sh b/sql/benchmarks/tpch/load.sh
--- a/sql/benchmarks/tpch/load.sh
+++ b/sql/benchmarks/tpch/load.sh
@@ -35,7 +35,7 @@ SF='sf-0.01'
#SF='sf-5'
dir=`echo $SF | tr '[a-z]' '[A-Z]'`
-$SQL c.sql-dec-primary-foreign
+#$SQL c.sql-dec-primary-foreign
#$SQL c.sql-dec
#cat load-$SF-LOCKED.sql$T | sed -e s+PWD+$PWD/$dir+ | $SQL
cat load-$SF.sql$T | sed -e s+PWD+$PWD/$dir+ | $SQL
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -45,16 +45,16 @@ bind_ts_ins(sql_timestamps *ts, int acce
}
static BAT *
-bind_ts_insert(sql_trans *tr, sql_table *t, int access) {
-
+bind_ts_insert(sql_trans *tr, sql_table *t, int access)
+{
assert(t-data);
t-s-base.rtime = t-base.rtime = tr-stime;
return bind_ts_ins(t-data, access);
}
static BAT *
-bind_ts_del(sql_timestamps *ts, int access) {
-
+bind_ts_del(sql_timestamps *ts, int access)
+{
BAT *b;
#ifdef NDEBUG
@@ -273,7 +273,7 @@ dup_delta(sql_trans *tr, sql_delta *obat
return LOG_OK;
}
-int
+static int
dup_bat(sql_trans *tr, sql_table *t, sql_delta *obat, sql_delta *bat, int
type, int oc_isnew, int c_isnew)
{
return dup_delta( tr, obat, bat, type, oc_isnew, c_isnew,
isTempTable(t), t-sz);
@@ -405,7 +405,7 @@ timestamps_insert(sql_trans *tr, sql_tab
wrd i;
int nts=0;
- BUN first = ts-slot;
+ oid first = ts-slot;
assert(!insb || insb-htype == TYPE_void);
assert(!delb || delb-htype == TYPE_void);
@@ -419,8 +419,8 @@ timestamps_insert(sql_trans *tr, sql_tab
/* insert negative timestamps until commit */
for(i=cnt; i0; i--) {
- BUNinplace(insb, first, insb-H, (ptr)nts, TRUE);
- BUNinplace(delb, first, delb-H, int_nil, TRUE);
+ BUNinplace(insb, first, NULL, (ptr)nts, TRUE);
+ BUNinplace(delb, first, NULL, int_nil, TRUE);
first++;
}
@@ -464,8 +464,8 @@ insert_bat2col(sql_timestamps *ts, sql_d
}
static void
-claim_tab(sql_trans *tr, sql_table *t, wrd cnt) {
-
+claim_tab(sql_trans *tr, sql_table *t, wrd cnt)
+{
sql_timestamps *ts;
node *n;
@@ -650,7 +650,7 @@ timestamps_delete_bat(sql_trans *tr, sql
BATloop(i, p, q) {
BUN slot = *(BUN*)Tloc(i,p);
ts-deleted = slot;
- BUNinplace(delb, slot, delb-H, (ptr)nts, TRUE);
+ BUNinplace(delb, slot, NULL, (ptr)nts, TRUE);
}
bat_destroy(delb);
}
@@ -674,7 +674,6 @@ delete_tab(sql_trans *tr, sql_table *t,
sql_timestamps *ts;
node *n;
-
if (bat_debug==2)
printf(#in delete_tab\n);
@@ -1057,8 +1056,8 @@ load_timestamps(sql_trans *tr, sql_times
}
static int
-create_timestamps(sql_trans *tr, sql_table *t) {
-
+create_timestamps(sql_trans *tr, sql_table *t)
+{
int ok = LOG_OK;
BAT *b;
sql_timestamps *ts = t-data;
@@ -1624,7 +1623,7 @@ gtr_minmax( sql_trans *tr )
return _gtr_update(tr, gtr_minmax_table);
}
-int
+static int
tr_update_delta( sql_trans *tr, sql_delta *obat, sql_delta *cbat)
{
int ok = LOG_OK;
@@ -1669,7 +1668,7 @@ tr_update_delta( sql_trans *tr, sql_delt
return ok;
}
-int
+static int
tr_update_tsbats(sql_trans *tr, sql_timestamps *tts, sql_timestamps *fts) {
int ok = LOG_OK;
diff --git a/sql/storage/bat/bat_storage.h b/sql/storage/bat/bat_storage.h
--- a/sql/storage/bat/bat_storage.h
+++ b/sql/storage/bat/bat_storage.h
@@ -54,11 +54,5 @@ typedef struct sql_timestamps {
/* initialize bat storage call back functions interface */
extern int bat_storage_init( store_functions *sf );
-extern int tr_update_delta( sql_trans *tr, sql_delta *obat, sql_delta *cbat );
-extern int tr_update_tsbats(sql_trans *tr, sql_timestamps *fts, sql_timestamps
*tts);
-
-extern int dup_bat(sql_trans *tr, sql_table *t, sql_delta *obat, sql_delta
*bat, int type, int oc_isnew, int c_isnew);
-extern sql_delta * timestamp_delta( sql_delta *d, int ts);
-
#endif /*BATSTORAGE_H */
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -1600,7 +1600,7 @@ store_manager(void)
return;
}
MT_lock_set(bs_lock, store_manager);
- if (store_nr_active || GDKexiting() ||
+ if (1 || store_nr_active || GDKexiting() ||
logger_funcs.changes() 1000) {
MonetDB: newstorage - cleanup
Changeset: b81ddec11ae7 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b81ddec11ae7
Modified Files:
gdk/gdk_logger.c
gdk/gdk_logger.h
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_pipes.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.mx
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/include/sql_catalog.h
sql/server/sql_mvc.c
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_storage.h
sql/storage/bat/bat_table.c
sql/storage/sql_storage.h
sql/storage/store.c
Branch: newstorage
Log Message:
cleanup
) use log_delta (and new log_delta_slice) to log the inserts as updates
) store slot/inserted in the sql_timestamp, with multiple inserts keep
them in a list of inserts
) keep deleted first location in the sql_timestamp structure
) removed insert part from the delta interface, disabled push_select optimizer
and adapted mergetable for this
Open issue, on restart some of the deletes aren't handled properly, leading
to broken allready drop tables.
diffs (truncated from 4033 to 300 lines):
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1721,7 +1721,7 @@ log_delta(logger *lg, BAT *b, char *name
}
l.tid = lg-tid;
- nr = (BUNlast(b) - BUNfirst(b));
+ nr = BATcount(b);
assert(nr = GDK_int_max);
l.nr = (int) nr;
lg-changes += l.nr;
@@ -1736,7 +1736,7 @@ log_delta(logger *lg, BAT *b, char *name
log_write_string(lg, name) == LOG_ERR)
return LOG_ERR;
- for (p = BUNfirst(b); p BUNlast(b) ok == GDK_SUCCEED; p++)
{
+ for (p = 0; p nr ok == GDK_SUCCEED; p++) {
const void *h = BUNhead(bi, p);
const void *t = BUNtail(bi, p);
@@ -1744,7 +1744,55 @@ log_delta(logger *lg, BAT *b, char *name
ok = (ok == GDK_FAIL) ? ok : wt(t, lg-log, 1);
}
- //if (lg-debug 1)
+ if (lg-debug 1)
+ fprintf(stderr, Logged %s %d inserts\n, name, l.nr);
+ }
+ return (ok == GDK_SUCCEED) ? LOG_OK : LOG_ERR;
+}
+
+int
+log_delta_slice(logger *lg, BUN first, size_t cnt, BAT *b, char *name)
+{
+ int ok = GDK_SUCCEED;
+ logformat l;
+ BUN p;
+ BUN nr;
+
+ if (lg-debug 128) {
+ /* logging is switched off */
+ return LOG_OK;
+ }
+
+ l.tid = lg-tid;
+ nr = cnt;
+ assert(nr = GDK_int_max);
+ l.nr = (int) nr;
+ lg-changes += l.nr;
+
+ if (l.nr) {
+ BATiter bi = bat_iterator(b);
+ int (*wh) (const void *, stream *, size_t) = b-htype ==
TYPE_void ? BATatoms[TYPE_oid].atomWrite : BATatoms[b-htype].atomWrite;
+ int (*wt) (const void *, stream *, size_t) =
BATatoms[b-ttype].atomWrite;
+
+ l.flag = LOG_UPDATE;
+ if (log_write_format(lg, l) == LOG_ERR ||
+ log_write_string(lg, name) == LOG_ERR)
+ return LOG_ERR;
+
+ for (p = first; p first+cnt ok == GDK_SUCCEED; p++) {
+ oid id = p;
+ const void *t = BUNtail(bi, p);
+
+ if (b-htype == TYPE_void) {
+ ok = wh(id, lg-log, 1);
+ } else {
+ const void *h = BUNhead(bi, p);
+ ok = wh(h, lg-log, 1);
+ }
+ ok = (ok == GDK_FAIL) ? ok : wt(t, lg-log, 1);
+ }
+
+ if (lg-debug 1)
fprintf(stderr, Logged %s %d inserts\n, name, l.nr);
}
return (ok == GDK_SUCCEED) ? LOG_OK : LOG_ERR;
@@ -1793,7 +1841,7 @@ log_bat(logger *lg, BAT *b, char *name)
}
}
-// if (lg-debug 1)
+ if (lg-debug 1)
fprintf(stderr, Logged %s %d inserts\n, name, l.nr);
}
l.nr = (int) (b-batFirst - b-batDeleted);
@@ -1817,7 +1865,7 @@ log_bat(logger *lg, BAT *b, char *name)
ok = (ok == GDK_FAIL) ? ok : wt(t, lg-log, 1);
}
-// if (lg-debug 1)
+ if (lg-debug 1)
fprintf(stderr, Logged %s %d deletes\n, name, l.nr);
}
return (ok == GDK_SUCCEED) ? LOG_OK : LOG_ERR;
diff --git a/gdk/gdk_logger.h b/gdk/gdk_logger.h
--- a/gdk/gdk_logger.h
+++ b/gdk/gdk_logger.h
@@ -113,6 +113,7 @@ gdk_export int log_bat_clear(logger *lg,
gdk_export int log_bat_persists(logger *lg, BAT *b, char *n);
gdk_export int log_bat_transient(logger *lg, char *n);
MonetDB: newstorage - cleanup
Changeset: d602c8268b62 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d602c8268b62
Modified Files:
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_storage.h
Branch: newstorage
Log Message:
cleanup
diffs (223 lines):
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -395,8 +395,7 @@ dup_delta(sql_trans *tr, sql_delta *obat
}
}
- //if(bat-bid)
- temp_dup(bat-bid);
+ temp_dup(bat-bid);
return LOG_OK;
}
@@ -462,55 +461,6 @@ update_idx(sql_trans *tr, sql_idx * i, v
else
assert(0);
}
-/**
-static void
-delta_append_bat( sql_delta *bat, BAT *i )
-{
- int id = i-batCacheid;
- BAT *b;
-#ifndef NDEBUG
- BAT *c = BBPquickdesc(bat-bid, 0);
- assert(!c || c-htype == TYPE_void);
-#endif
-
- if (bat_debug==2)
- printf(#in delta_append_bat\n);
-
- if (!BATcount(i))
- return ;
- b = temp_descriptor(bat-ibid);
- assert(b-htype == TYPE_void);
-
- if (bat-cached) {
- bat_destroy(bat-cached);
- bat-cached = NULL;
- }
- assert(!c || BATcount(c) == bat-ibase);
- if (!bat-ibase !BATcount(b) BBP_refs(id) == 1 BBP_lrefs(id)
== 1 !isVIEW(i) i-ttype*/ /* we need info if this is comming from copy
into, like role == PERSISTENT *//*){
- temp_destroy(bat-ibid);
- bat-ibid = id;
- temp_dup(id);
- bat_destroy(b);
- } else {
- if (!isEbat(b)){
- *//* try to use mmap() *//*
- if (BATcount(b)+BATcount(i) (BUN) REMAP_PAGE_MAXSIZE)
{
- BATmmap(b, STORE_MMAP, STORE_MMAP,
STORE_MMAP, STORE_MMAP, 1);
- }
- assert(b-T-heap.storage != STORE_PRIV);
- } else {
- temp_destroy(bat-ibid);
- bat-ibid = ebat2real(b-batCacheid, bat-ibase);
- bat_destroy(b);
- b = temp_descriptor(bat-ibid);
- }
- BATappend(b, i, TRUE);
- assert(BUNlast(b) b-batInserted);
- bat_destroy(b);
- }
- bat-cnt += BATcount(i);
-}
-*/
static void
insert_val2col(sql_delta *col, void *i) {
@@ -523,7 +473,6 @@ insert_val2col(sql_delta *col, void *i)
col-cached = NULL;
}
- //assert(!c || BATcount(c) == col-ibase);
assert(c-T-heap.storage != STORE_PRIV);
BUNinplace(c, (BATcount(c) -1), c-H, i, TRUE);
@@ -532,37 +481,6 @@ insert_val2col(sql_delta *col, void *i)
bat_destroy(c);
}
-/**static void
-delta_append_val( sql_delta *bat, void *i )
-{
-
- BAT *b = temp_descriptor(bat-ibid);
-#ifndef NDEBUG
- BAT *c = BBPquickdesc(bat-bid, 0);
- assert(!c || c-htype == TYPE_void);
-#endif
-
- if (bat_debug==2)
- printf(#in delta append val\n);
-
- assert(b-htype == TYPE_void);
- if (bat-cached) {
- bat_destroy(bat-cached);
- bat-cached = NULL;
- }
- assert(!c || BATcount(c) == bat-ibase);
- if (isEbat(b)) {
- bat_destroy(b);
- temp_destroy(bat-ibid);
- bat-ibid = ebat2real(bat-ibid, bat-ibase);
- b = temp_descriptor(bat-ibid);
- }
- BUNappend(b, i, TRUE);
- assert(BUNlast(b) b-batInserted);
- bat-cnt ++;
- bat_destroy(b);
-}*/
-
static int
dup_col(sql_trans *tr, sql_column *oc, sql_column *c )
{
@@ -593,38 +511,6 @@ dup_idx(sql_trans *tr, sql_idx *i, sql_i
}
return ok;
}
-/**
-static int
-dup_dbat( sql_trans *tr, sql_dbat *obat, sql_dbat *bat, int is_new, int temp)
-{
- if (bat_debug==2)
- printf(#in dup_dbat\n);
-
- bat-dbid = obat-dbid;
- bat-cnt = obat-cnt;
- bat-dname = _STRDUP(obat-dname);
- bat-wtime = obat-wtime;
-
- if (bat-dbid) {
- if (is_new)
- obat-dbid = temp_copy(bat-dbid, temp);
- else
- bat-dbid = ebat_copy(bat-dbid, 0, temp);
- }
- (void)tr;
- return LOG_OK;
-}
-
-static int
-dup_del(sql_trans *tr, sql_table *ot, sql_table *t)
-{
- sql_dbat *bat = t-data = ZNEW(sql_dbat), *obat = ot-data;
- int ok = dup_dbat( tr, obat, bat, isNew(t), isTempTable(t));
- assert(t-base.allocated == 0);
- //t-base.allocated = 1;
- return ok;
-}
-*/
static int
dup_tsbats(sql_trans *tr, sql_timestamps *ots, sql_timestamps *ts, int is_new,
int temp) {
@@ -728,7 +614,6 @@ static void
claim_tab(sql_trans *tr, sql_table *t, wrd cnt) {
sql_timestamps *ts;
- //sql_dbat *bat;
node *n, *in;
int growF=0;
@@
