MonetDB: unlock - improved handling of deletes

[Niels Nes]

Changeset: 1471c4f087fc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1471c4f087fc
Modified Files:
gdk/gdk_logger.c
gdk/gdk_logger.h
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_scenario.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_storage.h
sql/storage/bat/bat_table.c
sql/storage/sql_storage.h
sql/storage/store.c
sql/test/BugTracker-2009/Tests/copy_multiple_files.SF-2902320.stable.out

sql/test/BugTracker-2009/Tests/copy_multiple_files.SF-2902320.stable.out.Windows
sql/test/BugTracker-2009/Tests/join_topn.SF-2654133.test
testing/Mtest.py.in
Branch: unlock
Log Message:

improved handling of deletes


diffs (truncated from 2523 to 300 lines):

diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -27,13 +27,14 @@ static gdk_return logger_del_bat(logger 
 
 #define LOG_START  0
 #define LOG_END1
-#define LOG_UPDATE_BULK2
-#define LOG_UPDATE 3
-#define LOG_CREATE 4
-#define LOG_DESTROY5
-#define LOG_SEQ6
-#define LOG_CLEAR  7
-#define LOG_ROW8 /* per row relative small log entry */
+#define LOG_UPDATE_CONST   2
+#define LOG_UPDATE_BULK3
+#define LOG_UPDATE 4
+#define LOG_CREATE 5
+#define LOG_DESTROY6
+#define LOG_SEQ7
+#define LOG_CLEAR  8
+#define LOG_ROW9 /* per row relative small log entry */
 
 #ifdef NATIVE_WIN32
 #define getfilepos _ftelli64
@@ -52,6 +53,7 @@ static gdk_return logger_del_bat(logger 
 static const char *log_commands[] = {
"LOG_START",
"LOG_END",
+   "LOG_UPDATE_CONST",
"LOG_UPDATE_BULK",
"LOG_UPDATE",
"LOG_CREATE",
@@ -354,7 +356,22 @@ log_read_updates(logger *lg, trans *tr, 
}
}
 
-   if (l->flag == LOG_UPDATE_BULK) {
+   if (l->flag == LOG_UPDATE_CONST) {
+   if (!mnstr_readLng(lg->input_log, )) {
+   if (r)
+   BBPreclaim(r);
+   return LOG_ERR;
+   }
+   void *t = rt(tv, lg->input_log, 1);
+   if (t == NULL) {
+   res = LOG_ERR;
+   } else {
+   for(BUN p = 0; p<(BUN) nr; p++) {
+   if (r && BUNappend(r, t, true) != 
GDK_SUCCEED)
+   res = LOG_ERR;
+   }
+   }
+   } else if (l->flag == LOG_UPDATE_BULK) {
if (!mnstr_readLng(lg->input_log, )) {
if (r)
BBPreclaim(r);
@@ -462,7 +479,8 @@ log_read_updates(logger *lg, trans *tr, 
 
if (res == LOG_OK) {
if (tr_grow(tr) == GDK_SUCCEED) {
-   tr->changes[tr->nr].type = l->flag;
+   tr->changes[tr->nr].type =
+   
l->flag==LOG_UPDATE_CONST?LOG_UPDATE_BULK:l->flag;
tr->changes[tr->nr].nr = pnr;
tr->changes[tr->nr].tt = tpe;
tr->changes[tr->nr].cid = id;
@@ -1020,6 +1038,7 @@ logger_read_transaction(logger *lg)
case LOG_SEQ:
err = log_read_seq(lg, );
break;
+   case LOG_UPDATE_CONST:
case LOG_UPDATE_BULK:
case LOG_UPDATE:
if (tr == NULL)
@@ -2063,6 +2082,48 @@ logger_sequence(logger *lg, int seq, lng
return 0;
 }
 
+gdk_return
+log_constant(logger *lg, int type, ptr val, log_id id, lng offset, lng cnt)
+{
+   char tpe = find_type(lg, type);
+   gdk_return ok = GDK_SUCCEED;
+   logformat l;
+   lng nr;
+   int is_row = 0;
+
+   if (lg->row_insert_nrcols != 0) {
+   lg->row_insert_nrcols--;
+   is_row = 1;
+   }
+   l.flag = LOG_UPDATE_CONST;
+   l.id = id;
+   nr = cnt;
+
+   if (LOG_DISABLED(lg) || !nr) {
+   /* logging is switched off */
+   return GDK_SUCCEED;
+   }
+
+   gdk_return (*wt) (const void *, stream *, size_t) = 
BATatoms[type].atomWrite;
+
+   if (is_row)
+   l.flag = tpe;
+   if (log_write_format(lg, ) != GDK_SUCCEED ||
+   (!is_row && !mnstr_writeLng(lg->output_log, nr)) ||
+   (!is_row && mnstr_write(lg->output_log, , 1, 1) != 1) ||
+   (!is_row && !mnstr_writeLng(lg->output_log, offset)))
+   return GDK_FAIL;
+
+   ok 

MonetDB: unlock - improved handling of deletes

[Niels Nes]

Changeset: 60b956883f97 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=60b956883f97
Modified Files:
gdk/gdk_logger.c
gdk/gdk_logger.h
gdk/gdk_logger_internals.h
sql/storage/bat/bat_storage.c
Branch: unlock
Log Message:

improved handling of deletes
initial commit, needs more work.


diffs (206 lines):

diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -625,13 +625,14 @@ log_read_updates(logger *lg, trans *tr, 
 static log_return
 log_read_batgroup(logger *lg, trans *tr, logformat *l, char bg_tpe, oid id)
 {
-   lng offset = 0, nr_deleted = 0, nr_inserted = l->nr;
-   oid rid;
+   lng nr_inserted = l->nr, offset_inserted = 0, nr_deleted = 0, 
offset_deleted = 0;
+   oid rid = 0;
log_return res = LOG_OK;
char tpe = 0, *name = NULL;
 
-   if (mnstr_readLng(lg->input_log, ) != 1 ||
-   mnstr_readLng(lg->input_log, _deleted) != 1) {
+   if (mnstr_readLng(lg->input_log, _inserted) != 1 ||
+   mnstr_readLng(lg->input_log, _deleted) != 1 ||
+   mnstr_readLng(lg->input_log, _deleted) != 1) {
fprintf(stderr, "!ERROR: log_read_batgroup: read failed\n");
return LOG_EOF;
}
@@ -640,7 +641,7 @@ log_read_batgroup(logger *lg, trans *tr,
if (nr_deleted) {
l->nr = nr_deleted;
if (bg_tpe == LOG_BATGROUP_ID && log_read_id(lg, , ) == 
LOG_OK) {
-   res = log_read_updates(lg, tr, l, name, tpe, rid, 
bg_tpe == LOG_BATGROUP_ID, 0);
+   res = log_read_updates(lg, tr, l, name, tpe, rid, 
bg_tpe == LOG_BATGROUP_ID, offset_deleted);
} else {
res = LOG_ERR;
} 
@@ -651,7 +652,7 @@ log_read_batgroup(logger *lg, trans *tr,
if (bg_tpe == LOG_BATGROUP_ID && log_read_id(lg, , 
) == LOG_OK) { 
if (tpe == LOG_BATGROUP_END)
break;
-   res = log_read_updates(lg, tr, l, name, tpe, 
rid, bg_tpe == LOG_BATGROUP_ID, offset);
+   res = log_read_updates(lg, tr, l, name, tpe, 
rid, bg_tpe == LOG_BATGROUP_ID, offset_inserted);
} else {
res = LOG_ERR;
}
@@ -2044,6 +2045,8 @@ logger_load(int debug, const char *fn, c
if (BBPrename(lg->catalog_cnt->batCacheid, bak) < 0) {
goto error;
}
+   if (bm_get_counts(lg) == GDK_FAIL)
+   goto error;
 
lg->freed = logbat_new(TYPE_int, 1, TRANSIENT);
if (lg->freed == NULL) {
@@ -2175,7 +2178,7 @@ logger_load(int debug, const char *fn, c
if (lg->postfuncp && (*lg->postfuncp)(lg) != GDK_SUCCEED)
goto error;
}
-   return bm_get_counts(lg);
+   return GDK_SUCCEED;
   error:
if (fp)
fclose(fp);
@@ -2471,7 +2474,6 @@ logger_flush(logger *lg)
lg->changes -= 1000;
return GDK_SUCCEED;
}
-   // get lock 
if (!lg->input_log) {
char *filename;
char id[BUFSIZ];
@@ -2505,11 +2507,11 @@ logger_flush(logger *lg)
lg->flush_id++;
}
if (res != LOG_ERR) {
-   /* probably done in read_trans already */
-   // commit changes
-   lg->saved_tid++;
+   if (logger_commit(lg) != GDK_SUCCEED) {
+   fprintf(stderr, "!ERROR: logger_flush: logger_commit 
failed\n");
+   res = LOG_ERR;
+   }
}
-   // release lock 
return res == LOG_ERR ? GDK_FAIL : GDK_SUCCEED;
 }
 
@@ -2836,7 +2838,7 @@ log_bat(logger *lg, BAT *b, const char *
 
l.flag = tpe?LOG_INSERT_ID:LOG_INSERT;
assert(!offset || is_bg || name);
-   assert(!is_bg || l.nr == lg->bg.nr_inserted); /* or these are 
the deleted !*/
+   assert(!is_bg || l.nr == lg->bg.nr_inserted || l.nr == 
lg->bg.nr_deleted);
if (name && offset && !tpe)
l.flag = LOG_INSERT_OFFSET;
if ((!is_bg && log_write_format(lg, ) != GDK_SUCCEED) ||
@@ -2897,7 +2899,7 @@ log_bat_clear(logger *lg, const char *na
 }
 
 gdk_return 
-log_batgroup(logger *lg, char tpe_id, oid id, bool cleared, lng nr_inserted, 
lng offset, lng nr_deleted)
+log_batgroup(logger *lg, char tpe_id, oid id, bool cleared, lng nr_inserted, 
lng offset_inserted, lng nr_deleted, lng offset_deleted)
 {
logformat l;
char tpe = (tpe_id)?LOG_BATGROUP_ID:LOG_BATGROUP; 
@@ -2923,14 +2925,17 @@ log_batgroup(logger *lg, char tpe_id, oi
log_write_id(lg, tpe, id) != GDK_SUCCEED)
return GDK_FAIL;

-   assert(offset == 0 || !cleared);
+   assert(offset_inserted ==