MonetDB: default - Approve output
Changeset: 8ed17956f2f4 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8ed17956f2f4 Modified Files: sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out Branch: default Log Message: Approve output diffs (37 lines): diff --git a/sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out b/sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out --- a/sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out +++ b/sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out @@ -38,9 +38,7 @@ Ready. % 112 # length function user.s4_1():void; X_2:void := querylog.define("explain select count(*) from tmp where i = 20160222;", "default_pipe", 12:int); -X_12 := bat.new(nil:int); -C_14 := algebra.thetaselect(X_12, 20160222:int, "=="); -X_16 := algebra.projection(C_14, X_12); +X_16 := bat.new(nil:int); X_17 := aggr.count(X_16); sql.resultSet("sys.L4", "L3", "bigint", 64:int, 0:int, 7:int, X_17); end user.s4_1; @@ -53,9 +51,7 @@ end user.s4_1; % 118 # length function user.s6_1():void; X_2:void := querylog.define("explain select count(*) from tmp where i = \\'20160222\\';", "default_pipe", 13:int); -X_12 := bat.new(nil:int); -C_16 := algebra.thetaselect(X_12, 20160222:int, "=="); -X_18 := algebra.projection(C_16, X_12); +X_18 := bat.new(nil:int); X_19 := aggr.count(X_18); sql.resultSet("sys.L4", "L3", "bigint", 64:int, 0:int, 7:int, X_19); end user.s6_1; @@ -68,10 +64,7 @@ end user.s6_1; % 113 # length function user.s8_1():void; X_2:void := querylog.define("explain select count(*) from tmp where i = 201602221;", "default_pipe", 13:int); -X_12 := bat.new(nil:int); -X_13:bat[:int] := batcalc.int(0:int, X_12, 9:int, 0:int); -C_18 := algebra.thetaselect(X_13, 201602221:int, "=="); -X_20 := algebra.projection(C_18, X_12); +X_20 := bat.new(nil:int); X_21 := aggr.count(X_20); sql.resultSet("sys.L4", "L3", "bigint", 64:int, 0:int, 7:int, X_21); end user.s8_1; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Add thetaselect and bat updates
Changeset: f8afc241d5a2 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f8afc241d5a2 Modified Files: monetdb5/optimizer/opt_emptybind.c Branch: default Log Message: Add thetaselect and bat updates Empty sets can be propagated. diffs (240 lines): diff --git a/monetdb5/optimizer/opt_emptybind.c b/monetdb5/optimizer/opt_emptybind.c --- a/monetdb5/optimizer/opt_emptybind.c +++ b/monetdb5/optimizer/opt_emptybind.c @@ -20,6 +20,15 @@ #include "opt_deadcode.h" #include "mal_builder.h" +#define addresult(I) \ + do { \ + int tpe = getVarType(mb,getArg(p,I)); \ + q= newStmt(mb, batRef, newRef); \ + getArg(q,0)= getArg(p,I); \ + q = pushType(mb, q, getBatType(tpe)); \ + empty[getArg(q,0)]= 1; \ + } while (0) + #define emptyresult(I) \ do { \ int tpe = getVarType(mb,getArg(p,I)); \ @@ -30,6 +39,7 @@ p = pushType(mb,p, getBatType(tpe));\ setVarType(mb, getArg(p,0), tpe); \ setVarFixed(mb, getArg(p,0)); \ + empty[getArg(p,0)]= 1; \ } while (0) @@ -37,7 +47,7 @@ int OPTemptybindImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { int i,j, actions =0; - int *marked; + int *empty; int limit = mb->stop, slimit = mb->ssize; InstrPtr p, q, *old = mb->stmt, *updated; char buf[256]; @@ -55,19 +65,19 @@ OPTemptybindImplementation(Client cntxt, goto wrapup; // track of where 'emptybind' results are produced - marked = (int *) GDKzalloc(mb->vsize * sizeof(int)); - if ( marked == NULL) + empty = (int *) GDKzalloc(mb->vsize * sizeof(int)); + if ( empty == NULL) return 0; updated= (InstrPtr *) GDKzalloc(esize * sizeof(InstrPtr)); if( updated == 0){ - GDKfree(marked); + GDKfree(empty); return 0; } (void) stk; /* Got an instructions V:= bat.new(:tpe) -* The form the initial family of marked sets. +* The form the initial family of empty sets. */ (void) cntxt; @@ -102,7 +112,7 @@ OPTemptybindImplementation(Client cntxt, #ifdef DEBUG_OPT_EMPTYBIND mnstr_printf(cntxt->fdout, "#empty bat pc %d var %d\n",i , getArg(p,0) ); #endif - marked[getArg(p,0)] = i; + empty[getArg(p,0)] = i; continue; } @@ -111,14 +121,14 @@ OPTemptybindImplementation(Client cntxt, if ( etop == esize){ updated = (InstrPtr*) GDKrealloc( updated, (esize += 256) * sizeof(InstrPtr)); if( updated == NULL){ - GDKfree(marked); + GDKfree(empty); goto wrapup; } } updated[etop++]= p; } - /* restore the naming, dropping the runtime property 'marked' */ + /* restore the naming, dropping the runtime property 'empty' */ if (getFunctionId(p) == emptybindRef) { #ifdef DEBUG_OPT_EMPTYBIND mnstr_printf(cntxt->fdout, "#empty bind pc %d var %d\n",i , getArg(p,0) ); @@ -126,9 +136,9 @@ OPTemptybindImplementation(Client cntxt, actions++; setFunctionId(p,bindRef); p->typechk= TYPE_UNKNOWN; - marked[getArg(p,0)] = i; + empty[getArg(p,0)] = i; if( p->retc == 2){ - marked[getArg(p,1)] = i; + empty[getArg(p,1)] = i; #ifdef DEBUG_OPT_EMPTYBIND mnstr_printf(cntxt->fdout, "#empty update bind pc %d var %d\n",i , getArg(p,1) ); #endif @@ -144,30 +154,30 @@ OPTemptybindImplementation(Client cntxt, #ifdef DEBUG_OPT_EMPTYBIND mnstr_printf(cntxt->fdout, "#reset mark empty variable pc %d var %d\n",i , getArg(p,0) ); #endif -
MonetDB: Dec2016 - Fix or crash reported in bug 6220.
Changeset: 9fdb3a8b4c32 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9fdb3a8b4c32 Modified Files: sql/server/rel_select.c Branch: Dec2016 Log Message: Fix or crash reported in bug 6220. diffs (12 lines): diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c --- a/sql/server/rel_select.c +++ b/sql/server/rel_select.c @@ -4881,7 +4881,7 @@ rel_query(mvc *sql, sql_rel *rel, symbol /* reset error */ sql->session->status = 0; sql->errstr[0] = 0; - if (used) + if (used && rel) rel = rel_dup(rel); if (!used && (!sn->lateral && !lateral) && rel) { sql_rel *o = rel; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Simplify code for debugging
Changeset: 7401c070465b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7401c070465b Modified Files: monetdb5/mal/mal.h monetdb5/mal/mal_debugger.c monetdb5/mal/mal_function.c monetdb5/mal/mal_instruction.c monetdb5/mal/mal_instruction.h Branch: default Log Message: Simplify code for debugging diffs (138 lines): diff --git a/monetdb5/mal/mal.h b/monetdb5/mal/mal.h --- a/monetdb5/mal/mal.h +++ b/monetdb5/mal/mal.h @@ -146,7 +146,14 @@ typedef struct SYMDEF { typedef struct VARRECORD { char id[IDLENGTH]; /* use the space for the full name */ malType type; /* internal type signature */ - short flags;/* see below, reserve some space */ +unsigned short constant:1, +typevar:1, +fixedtype:1, +udftype:1, +cleanup:1, +initialized:1, +used:1, +disabled:1; short depth;/* scope block depth, set to -1 if not used */ short worker; /* thread id of last worker producing it */ ValRecord value; diff --git a/monetdb5/mal/mal_debugger.c b/monetdb5/mal/mal_debugger.c --- a/monetdb5/mal/mal_debugger.c +++ b/monetdb5/mal/mal_debugger.c @@ -1228,7 +1228,7 @@ printStackElm(stream *f, MalBlkPtr mb, V if (strcmp(nmeOnStk, nme) && strncmp(nmeOnStk, "BAT", 3)) mnstr_printf(f, "!%s ", nmeOnStk); mnstr_printf(f, " %s", (isVarConstant(mb, index) ? " constant" : "")); - /* mnstr_printf(f, " %s", (isVarUsed(mb,index) ? "": " not used" ));*/ + mnstr_printf(f, " %s", (isVarUsed(mb,index) ? "": " not used" )); mnstr_printf(f, " %s", (isVarTypedef(mb, index) ? " type variable" : "")); GDKfree(nme); mnstr_printf(f, "\n"); diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c --- a/monetdb5/mal/mal_function.c +++ b/monetdb5/mal/mal_function.c @@ -629,6 +629,10 @@ setVariableScope(MalBlkPtr mb) dflow= -1; else depth--; } + if( blockReturn(p)){ + for (k = 0; k < p->argc; k++) + setVarEolife(mb,getArg(p,k),pc); + } } for (k = 0; k < mb->vtop; k++) if( getVarEolife(mb,k) == 0) diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -768,7 +768,13 @@ clearVariable(MalBlkPtr mb, int varid) if (isVarConstant(mb, varid) || isVarDisabled(mb, varid)) VALclear(>value); v->type = 0; - v->flags = 0; + v->constant= 0; + v->typevar= 0; + v->fixedtype= 0; + v->udftype= 0; + v->cleanup= 0; + v->initialized= 0; + v->used= 0; v->rowcnt = 0; v->eolife = 0; v->stc = 0; diff --git a/monetdb5/mal/mal_instruction.h b/monetdb5/mal/mal_instruction.h --- a/monetdb5/mal/mal_instruction.h +++ b/monetdb5/mal/mal_instruction.h @@ -51,38 +51,39 @@ #define getVarGDKType(M,I) getGDKType((M)->var[I].type) #define setVarType(M,I,V) (M)->var[I].type = V -#define clrVarFixed(M,I) ((M)->var[I].flags &= ~VAR_FIXTYPE) -#define setVarFixed(M,I) ((M)->var[I].flags |= VAR_FIXTYPE) -#define isVarFixed(M,I)((M)->var[I].flags & VAR_FIXTYPE) +#define clrVarFixed(M,I) ((M)->var[I].fixedtype = 0) +#define setVarFixed(M,I) ((M)->var[I].fixedtype =1) +#define isVarFixed(M,I)((M)->var[I].fixedtype) -#define clrVarCleanup(M,I) ((M)->var[I].flags &= ~VAR_CLEANUP) -#define setVarCleanup(M,I) ((M)->var[I].flags |= VAR_CLEANUP) -#define isVarCleanup(M,I) ((M)->var[I].flags & VAR_CLEANUP) +#define clrVarCleanup(M,I) ((M)->var[I].cleanup = 0) +#define setVarCleanup(M,I) ((M)->var[I].cleanup = 1) +#define isVarCleanup(M,I) ((M)->var[I].cleanup ) + #define isTmpVar(M,I) (*getVarName(M,I) == REFMARKER && *(getVarName(M,I)+1) == TMPMARKER) -#define clrVarUsed(M,I)((M)->var[I].flags &= ~VAR_USED) -#define setVarUsed(M,I)((M)->var[I].flags |= VAR_USED) -#define isVarUsed(M,I) ((M)->var[I].flags & VAR_USED) +#define clrVarUsed(M,I)((M)->var[I].used = 0) +#define setVarUsed(M,I)((M)->var[I].used = 1) +#define isVarUsed(M,I) ((M)->var[I].used) -#define clrVarDisabled(M,I)((M)->var[I].flags &= ~VAR_DISABLED) -#define setVarDisabled(M,I)((M)->var[I].flags |= VAR_DISABLED) -#define isVarDisabled(M,I) ((M)->var[I].flags & VAR_DISABLED) +#define clrVarDisabled(M,I)((M)->var[I].disabled= 0 )
MonetDB: default - Push aside incomplete code
Changeset: d350ab890b8e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d350ab890b8e Modified Files: monetdb5/optimizer/opt_garbageCollector.c Branch: default Log Message: Push aside incomplete code diffs (30 lines): diff --git a/monetdb5/optimizer/opt_garbageCollector.c b/monetdb5/optimizer/opt_garbageCollector.c --- a/monetdb5/optimizer/opt_garbageCollector.c +++ b/monetdb5/optimizer/opt_garbageCollector.c @@ -25,8 +25,8 @@ int OPTgarbageCollectorImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { - int i, j, k, limit, slimit; - InstrPtr p, q, *old; + int i, j, limit, slimit; + InstrPtr p, *old; int actions = 0; char buf[256]; lng usec = GDKusec(); @@ -99,6 +99,7 @@ OPTgarbageCollectorImplementation(Client * otherwise we could not inject the assignment */ if (blockExit(p) ){ /* force garbage collection of all declared within output block and ending here */ +/* ignore for the time being, it requires a more thorough analysis of dependencies. for( k = stmtlnk[i]; k; k = varlnk[k]) if( isaBatType(getVarType(mb,k)) ){ q = newAssignment(mb); @@ -110,6 +111,7 @@ OPTgarbageCollectorImplementation(Client setVarEolife(mb,k,mb->stop-1); actions++; } +*/ } } assert(p); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Approve output
Changeset: b297a9fcaa44 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b297a9fcaa44 Modified Files: monetdb5/optimizer/Tests/GCexample01.stable.out monetdb5/optimizer/Tests/tst4006.stable.out Branch: default Log Message: Approve output diffs (102 lines): diff --git a/monetdb5/optimizer/Tests/GCexample01.stable.out b/monetdb5/optimizer/Tests/GCexample01.stable.out --- a/monetdb5/optimizer/Tests/GCexample01.stable.out +++ b/monetdb5/optimizer/Tests/GCexample01.stable.out @@ -37,20 +37,17 @@ function user.qry():void; barrier (x:oid, z:int) := iterator.new(b:bat[:int]); #[5] (0) ITRbunIterator 7 8 <- 1 io.print(x:oid); #[6] (0) IOprint_val 9 <- 7 exit (x:oid, z:int); #[7] (0) 7 8 -b:bat[:int] := nil:bat[:int]; #[8] (0) 1 <- 16 -barrier (x:oid, z:int) := iterator.new(c:bat[:int]); #[9] (0) ITRbunIterator 7 8 <- 3 -io.print(x:oid); #[10] (0) IOprint_val 10 <- 7 -redo (x:oid, z:int) := iterator.next(c:bat[:int]); #[11] (0) ITRbunNext 7 8 <- 3 -exit (x:oid, z:int); #[12] (0) 7 8 -c:bat[:int] := nil:bat[:int]; #[13] (0) 3 <- 18 +barrier (x:oid, z:int) := iterator.new(c:bat[:int]); #[8] (0) ITRbunIterator 7 8 <- 3 +io.print(x:oid); #[9] (0) IOprint_val 10 <- 7 +redo (x:oid, z:int) := iterator.next(c:bat[:int]); #[10] (0) ITRbunNext 7 8 <- 3 +exit (x:oid, z:int); #[11] (0) 7 8 # do not close d, it may be reused -barrier go:bit := true:bit;#[15] (0) 12 <- 13 -io.print(d:bat[:int]); #[16] (0) IOprint_val 14 <- 4 -redo go:bit := true:bit; #[17] (0) 12 <- 13 -exit go:bit; #[18] (0) 12 -d:bat[:int] := nil:bat[:int]; #[19] (0) 4 <- 20 -end user.qry; #[20] (0) -#garbagecollector actions= 4 time=31 usec +barrier go:bit := true:bit;#[13] (0) 12 <- 13 +io.print(d:bat[:int]); #[14] (0) IOprint_val 14 <- 4 +redo go:bit := true:bit; #[15] (0) 12 <- 13 +exit go:bit; #[16] (0) 12 +end user.qry; #[17] (0) +#garbagecollector actions= 1 time=21 usec function user.qry():void; #[0] (0) 0 b:bat[:int] := bat.new(:int); #[1] (0) CMDBATnew 1 <- 2 c:bat[:int] := bat.new(:int); #[2] (0) CMDBATnew 3 <- 2 @@ -59,20 +56,17 @@ function user.qry():void; barrier (x:oid, z:int) := iterator.new(b:bat[:int]); #[5] (0) ITRbunIterator 7 8 <- 1 io.print(x:oid); #[6] (0) IOprint_val 9 <- 7 exit (x:oid, z:int); #[7] (0) 7 8 -b:bat[:int] := nil:bat[:int]; #[8] (0) 1 <- 16 -barrier (x:oid, z:int) := iterator.new(c:bat[:int]); #[9] (0) ITRbunIterator 7 8 <- 3 -io.print(x:oid); #[10] (0) IOprint_val 10 <- 7 -redo (x:oid, z:int) := iterator.next(c:bat[:int]); #[11] (0) ITRbunNext 7 8 <- 3 -exit (x:oid, z:int); #[12] (0) 7 8 -c:bat[:int] := nil:bat[:int]; #[13] (0) 3 <- 18 +barrier (x:oid, z:int) := iterator.new(c:bat[:int]); #[8] (0) ITRbunIterator 7 8 <- 3 +io.print(x:oid); #[9] (0) IOprint_val 10 <- 7 +redo (x:oid, z:int) := iterator.next(c:bat[:int]); #[10] (0) ITRbunNext 7 8 <- 3 +exit (x:oid, z:int); #[11] (0) 7 8 # do not close d, it may be reused -barrier go:bit := true:bit;#[15] (0) 12 <- 13 -io.print(d:bat[:int]); #[16] (0) IOprint_val 14 <- 4 -redo go:bit := true:bit; #[17] (0) 12 <- 13 -exit go:bit; #[18] (0) 12 -d:bat[:int] := nil:bat[:int]; #[19] (0) 4 <- 20 -end user.qry; #[20] (0) -#garbagecollector actions= 4 time=31 usec +barrier go:bit := true:bit;#[13] (0) 12 <- 13 +io.print(d:bat[:int]); #[14] (0) IOprint_val 14 <- 4 +redo go:bit := true:bit; #[15] (0) 12 <- 13 +exit go:bit; #[16] (0) 12 +end user.qry; #[17] (0) +#garbagecollector actions= 1 time=21 usec # 23:17:49 > # 23:17:49 > Done. diff --git a/monetdb5/optimizer/Tests/tst4006.stable.out b/monetdb5/optimizer/Tests/tst4006.stable.out --- a/monetdb5/optimizer/Tests/tst4006.stable.out +++ b/monetdb5/optimizer/Tests/tst4006.stable.out @@ -34,12 +34,10 @@ barrier V12:bit := true:bit; V13:any := algebra.union(x:bat[:int], z:bat[:int]);
MonetDB: default - Proper count of dataflow blocks
Changeset: fcbbcb416c4f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fcbbcb416c4f Modified Files: monetdb5/optimizer/opt_dataflow.c Branch: default Log Message: Proper count of dataflow blocks diffs (25 lines): diff --git a/monetdb5/optimizer/opt_dataflow.c b/monetdb5/optimizer/opt_dataflow.c --- a/monetdb5/optimizer/opt_dataflow.c +++ b/monetdb5/optimizer/opt_dataflow.c @@ -210,9 +210,10 @@ OPTdataflowImplementation(Client cntxt, } } flowblock = newTmpVariable(mb,TYPE_bit); - q= newFcnCall(mb,languageRef,dataflowRef);\ - q->barrier= BARRIERsymbol;\ - getArg(q,0)= flowblock;\ + q= newFcnCall(mb,languageRef,dataflowRef); + q->barrier= BARRIERsymbol; + getArg(q,0)= flowblock; + actions++; } //copyblock for( j=start ; jretc; k++) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Merge with Dec2016 branch.
Changeset: f054b7310be7 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f054b7310be7 Added Files: monetdb5/modules/atoms/Tests/strappend.malC monetdb5/modules/atoms/Tests/strappend.stable.err monetdb5/modules/atoms/Tests/strappend.stable.out Modified Files: gdk/ChangeLog.Dec2016 gdk/gdk.h gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_join.c monetdb5/mal/mal_client.c monetdb5/modules/atoms/Tests/All monetdb5/optimizer/opt_pipes.c sql/backends/monet5/sql_statistics.c Branch: default Log Message: Merge with Dec2016 branch. NOTE: YOUR DATABASES THAT YOU CREATED IN THE DEFAULT BRANCH CANNOT BE USED. In the Dec2016 branch the BBP version number was increased in order to fix a bug where the empty string might occur with different offsets in a duplicate-eliminated string heap. This conflicts with a version number increase that had been done in the default branch. This means that your default databases are now incompatible with the current code. Dump before you build, and restore after (or simply throw them away). diffs (truncated from 627 to 300 lines): diff --git a/gdk/ChangeLog.Dec2016 b/gdk/ChangeLog.Dec2016 --- a/gdk/ChangeLog.Dec2016 +++ b/gdk/ChangeLog.Dec2016 @@ -1,3 +1,12 @@ # ChangeLog file for MonetDB # This file is updated with Maddlog +* Tue Feb 28 2017 Sjoerd Mullender+- Fixed a bug when appending string bats that are fully duplicate + eliminated. It could happend that the to-be-appended bat had an empty + string at an offset and at that same offset in the to-be-appended-to bat + there happened to be a (sequence of) NULL(s). Then this offset would be + used, even though it might nog be the right offset for the empty string + in the to-be-appended-to bat. This would result in multiple offsets for + the empty string, breaking the promise of being duplicate eliminated. + diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -854,8 +854,9 @@ typedef struct { #define GDKLIBRARY_INSERTED061032 /* inserted and deleted in BBP.dir */ #define GDKLIBRARY_HEADED 061033 /* head properties are stored */ #define GDKLIBRARY_NOKEY 061034 /* nokey values can't be trusted */ -#define GDKLIBRARY_TALIGN 061035 /* talign field in BBP.dir */ -#define GDKLIBRARY 061036 +#define GDKLIBRARY_BADEMPTY061035 /* possibility of duplicate empty str */ +#define GDKLIBRARY_TALIGN 061036 /* talign field in BBP.dir */ +#define GDKLIBRARY 061037 typedef struct BAT { /* static bat properties */ diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -301,10 +301,13 @@ insert_string_bat(BAT *b, BAT *n, BAT *s } bunfastapp(b, tp); } - } else if (b->tvheap->free < n->tvheap->free / 2) { + } else if (b->tvheap->free < n->tvheap->free / 2 || + GDK_ELIMDOUBLES(b->tvheap)) { /* if b's string heap is much smaller than n's string * heap, don't bother checking whether n's string -* values occur in b's string heap */ +* values occur in b's string heap; also, if b is +* (still) fully double eliminated, we must continue +* to use the double elimination mechanism */ r = BUNlast(b); if (cand) { oid hseq = n->hseqbase; diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -629,6 +629,287 @@ fixwkbheap(void) } #endif +#ifdef GDKLIBRARY_BADEMPTY +/* There was a bug (fixed in changeset 1f5498568a24) which could + * result in empty strings not being double-eliminated. This code + * fixes the affected bats. + * Note that we only fix BATs whose string heap is still fully double + * eliminated. */ +static inline int +offsearch(const int *restrict offsets, int noffsets, int val) +{ + /* binary search on offsets for val, return whether present */ + int lo = 0, hi = noffsets - 1, mid; + + while (hi > lo) { + mid = (lo + hi) / 2; + if (offsets[mid] == val) + return 1; + if (offsets[mid] < val) + lo = mid + 1; + else + hi = mid - 1; + } + return offsets[lo] == val; +} + +static void +fixstroffheap(BAT *b, int *restrict offsets) +{ + long_str filename; + Heap h1;/* old offset heap */ + Heap h2;/* new string heap */ + Heap h3;/* new offset heap */ + Heap *h;/* string heap */ + int noffsets = 0; + const size_t extralen = b->tvheap->hashash ? EXTRALEN : 0; + size_t pos; + var_t emptyoff = 0; + const char *nme, *bnme; + char *srcdir; +
MonetDB: Dec2016 - Better check to see whether join result is em...
Changeset: 310b7a4f966d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=310b7a4f966d Modified Files: gdk/gdk_join.c Branch: Dec2016 Log Message: Better check to see whether join result is empty. This fixes bug 6237. diffs (94 lines): diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c --- a/gdk/gdk_join.c +++ b/gdk/gdk_join.c @@ -117,10 +117,17 @@ joininitresults(BAT **r1p, BAT **r2p, BU *r1p = NULL; if (r2p) *r2p = NULL; - if (rkey | semi | only_misses) { + if (lcnt == 0) { + /* there is nothing to match */ + maxsize = 0; + } else if (!only_misses && !nil_on_miss && rcnt == 0) { + /* if right is empty, we have no hits, so if we don't +* want misses, the result is empty */ + maxsize = 0; + } else if (rkey | semi | only_misses) { /* each entry left matches at most one on right, in * case nil_on_miss is also set, each entry matches -* exactly one */ +* exactly one (see below) */ maxsize = lcnt; } else if (lkey) { /* each entry on right is matched at most once */ @@ -149,6 +156,25 @@ joininitresults(BAT **r1p, BAT **r2p, BU size = maxsize; } + if (maxsize == 0) { + r1 = COLnew(0, TYPE_void, 0, TRANSIENT); + if (r1 == NULL) { + return BUN_NONE; + } + BATtseqbase(r1, 0); + if (r2p) { + r2 = COLnew(0, TYPE_void, 0, TRANSIENT); + if (r2 == NULL) { + BBPreclaim(r1); + return BUN_NONE; + } + BATtseqbase(r2, 0); + *r2p = r2; + } + *r1p = r1; + return 0; + } + r1 = COLnew(0, TYPE_oid, size, TRANSIENT); if (r1 == NULL) { return BUN_NONE; @@ -3797,6 +3823,8 @@ subleftjoin(BAT **r1p, BAT **r2p, BAT *l *r1p = r1; if (r2p) *r2p = r2; + if (maxsize == 0) + return GDK_SUCCEED; if (BATtdense(r) && (sr == NULL || BATtdense(sr)) && lcount > 0 && rcount > 0) { /* use special implementation for dense right-hand side */ return mergejoin_void(r1, r2, l, r, sl, sr, @@ -3907,6 +3935,8 @@ BATthetajoin(BAT **r1p, BAT **r2p, BAT * return GDK_FAIL; *r1p = r1; *r2p = r2; + if (maxsize == 0) + return GDK_SUCCEED; return thetajoin(r1, r2, l, r, sl, sr, opcode, maxsize, t0); } @@ -3957,6 +3987,8 @@ BATjoin(BAT **r1p, BAT **r2p, BAT *l, BA return GDK_FAIL; *r1p = r1; *r2p = r2; + if (maxsize == 0) + return GDK_SUCCEED; swap = 0; /* some statistics to help us decide */ @@ -4085,6 +4117,8 @@ BATbandjoin(BAT **r1p, BAT **r2p, BAT *l return GDK_FAIL; *r1p = r1; *r2p = r2; + if (maxsize == 0) + return GDK_SUCCEED; return bandjoin(r1, r2, l, r, sl, sr, c1, c2, li, hi, maxsize, t0); } @@ -4104,6 +4138,8 @@ BATrangejoin(BAT **r1p, BAT **r2p, BAT * return GDK_FAIL; *r1p = r1; *r2p = r2; + if (maxsize == 0) + return GDK_SUCCEED; /* note, the rangejoin implementation is in gdk_select.c since * it uses the imprints code there */ ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Improved fix for fixing duplicate empty strings.
Changeset: 69603b650998 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=69603b650998 Modified Files: gdk/gdk_bbp.c Branch: Dec2016 Log Message: Improved fix for fixing duplicate empty strings. See changesets 1f5498568a24 and 0c475cc54c60. diffs (221 lines): diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -897,13 +897,18 @@ fixstroffheap(BAT *b, int *restrict offs { long_str filename; Heap h1;/* old offset heap */ + Heap h2;/* new string heap */ + Heap h3;/* new offset heap */ Heap *h;/* string heap */ int noffsets = 0; const size_t extralen = b->tvheap->hashash ? EXTRALEN : 0; - size_t pos, emptyoff = 0; + size_t pos; + var_t emptyoff = 0; const char *nme, *bnme; char *srcdir; BUN i; + int width; + int nofix = 1; assert(GDK_ELIMDOUBLES(b->tvheap)); @@ -930,31 +935,64 @@ fixstroffheap(BAT *b, int *restrict offs pos += pad + extralen; s = h->base + pos; if (*s == '\0') - emptyoff = pos; + emptyoff = (var_t) pos; offsets[noffsets++] = (int) pos; /* < 65536, i.e. fits */ pos += GDK_STRLEN(s); } HEAPfree(b->tvheap, 0); - if (emptyoff == 0) { - /* no empty string encountered in the heap, so the bug -* is extremely unlikely to have occurred (we need an -* area of zero bytes that is aligned on a var_t -* boundary and two var_t's long -- in the absence of -* an empty string, this can only happen if hashash is -* set and a string hashes to zero) */ - GDKfree(srcdir); - return; - } if ((bnme = strrchr(nme, DIR_SEP)) != NULL) bnme++; else bnme = nme; sprintf(filename, "BACKUP%c%s", DIR_SEP, bnme); + + width = b->twidth; + h2.dirty = 0; + if (emptyoff == 0) { + /* no legitimate empty string in the string heap; we +* now make a backup of the old string heap and create +* a new one to which we add an empty string */ + h2 = *b->tvheap; + if (GDKmove(h2.farmid, srcdir, bnme, "theap", BAKDIR, bnme, "theap") != GDK_SUCCEED) + GDKfatal("fixstroffheap: cannot make backup of %s.theap\n", nme); + h2.filename = GDKfilepath(NOFARM, NULL, nme, "theap"); + if (h2.filename == NULL) + GDKfatal("fixstroffheap: GDKmalloc failed\n"); + h2.base = NULL; + if (HEAPalloc(, h2.size, 1) != GDK_SUCCEED) + GDKfatal("fixstroffheap: allocating new string heap " +"for BAT %d failed\n", b->batCacheid); + h2.cleanhash = b->tvheap->cleanhash; + h2.hashash = b->tvheap->hashash; + h2.free = b->tvheap->free; + /* load old offset heap and copy contents to new heap */ + h1 = *b->tvheap; + h1.filename = NULL; + h1.base = NULL; + h1.dirty = 0; + if (HEAPload(, filename, "theap", 0) != GDK_SUCCEED) + GDKfatal("fixstroffheap: loading old tail heap " +"for BAT %d failed\n", b->batCacheid); + memcpy(h2.base, h1.base, h2.free); + HEAPfree(, 0); + h2.dirty = 1; + if ((*BATatoms[TYPE_str].atomPut)(, , "") == 0) + GDKfatal("fixstroffheap: cannot insert empty string " +"in BAT %d failed\n", b->batCacheid); + /* if the offset of the new empty string doesn't fit +* in the offset heap (too many bits for the current +* width), we will also make the new offset heap +* wider */ + if ((width <= 2 ? emptyoff - GDK_VAROFFSET : emptyoff) >= (var_t) (1 << (width * 8))) { + width <<= 1; + assert((width <= 2 ? emptyoff - GDK_VAROFFSET : emptyoff) < (var_t) (1 << (width * 8))); + } + } + + /* make backup of offset heap */ if (GDKmove(b->theap.farmid, srcdir, bnme, "tail", BAKDIR, bnme, "tail") != GDK_SUCCEED) GDKfatal("fixstroffheap: cannot make backup of %s.tail\n", nme); - GDKfree(srcdir); - /* load old offset heap */ h1 = b->theap; h1.filename = NULL; @@ -965,58 +1003,98 @@ fixstroffheap(BAT *b, int *restrict offs "for BAT %d failed\n", b->batCacheid); /* create new offset heap */ - b->theap.filename = GDKfilepath(NOFARM, NULL, nme,
MonetDB: wlcr - Remove everything related to the master-replica ...
Changeset: bb12643ccbf0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb12643ccbf0 Modified Files: tools/merovingian/client/monetdb.c tools/merovingian/daemon/controlrunner.c tools/merovingian/utils/database.c tools/merovingian/utils/database.h Branch: wlcr Log Message: Remove everything related to the master-replica commands The snapshots will be handled by the server itself. diffs (201 lines): diff --git a/tools/merovingian/client/monetdb.c b/tools/merovingian/client/monetdb.c --- a/tools/merovingian/client/monetdb.c +++ b/tools/merovingian/client/monetdb.c @@ -1620,63 +1620,6 @@ command_release(int argc, char *argv[]) simple_command(argc, argv, "release", "taken database out of maintenance mode", 1); } -static void -command_master(int argc, char *argv[]) -{ - char *e = NULL; - sabdb *orig = NULL; - sabdb *stats = NULL; - char *target_path = NULL; - - - if (argc != 2 && argc != 3) { - /* print help message for this command */ - command_help(argc + 1, [-1]); - exit(1); - } - - if (argc == 3) { - target_path = strdup(argv[2]); - argv[2] = NULL; - } - - if ((e = MEROgetStatus(, NULL)) != NULL) { - fprintf(stderr, "master: %s\n", e); - free(e); - exit(2); - } - stats = globMatchDBS(argc, argv, , "master"); - msab_freeStatus(); - orig = stats; - - if (target_path != NULL) { - size_t len = strlen("master path=") + strlen(target_path) + 1; - char *cmd = (char *)malloc(len); - snprintf(cmd, len, "master path=%s", target_path); - simple_argv_cmd(argv[0], orig, cmd, - "set database as master", NULL); - free(target_path); - free(cmd); - } else { - simple_argv_cmd(argv[0], orig, "master", - "set database as master", NULL); - } -} - -static void -command_replica(int argc, char *argv[]) -{ - int i; - if (argc != 3) { - /* print help message for this command */ - command_help(argc + 1, [-1]); - exit(1); - } - fprintf(stderr, "Called replica with %d args:\n", argc); - for (i = 0; i < argc; i++) - fprintf(stderr, " %s\n", argv[i]); -} - int main(int argc, char *argv[]) { @@ -1870,10 +1813,6 @@ main(int argc, char *argv[]) command_set(argc - i, [i], INHERIT); } else if (strcmp(argv[i], "discover") == 0) { command_discover(argc - i, [i]); - } else if (strcmp(argv[i], "master") == 0) { - command_master(argc - i, [i]); - } else if (strcmp(argv[i], "replica") == 0) { - command_replica(argc - i, [i]); } else { fprintf(stderr, "monetdb: unknown command: %s\n", argv[i]); command_help(0, NULL); diff --git a/tools/merovingian/daemon/controlrunner.c b/tools/merovingian/daemon/controlrunner.c --- a/tools/merovingian/daemon/controlrunner.c +++ b/tools/merovingian/daemon/controlrunner.c @@ -618,21 +618,6 @@ static void ctl_handle_client( len = snprintf(buf2, sizeof(buf2), "OK\n"); send_client("="); } - } else if (strcmp(p, "master") == 0 || - strncmp(p, "master path=", strlen("master path=")) == 0) { - char *e = db_master(q); - if (e != NULL) { - Mfprintf(_mero_ctlerr, "%s: got command master %s: %s\n", origin, q, getErrMsg(e)); - len = snprintf(buf2, sizeof(buf2), - "%s\n", getErrMsg(e)); - send_client("!"); - free(e); - } else { - Mfprintf(_mero_ctlout, "%s: set database '%s' to master mode\n", -origin, q); - len = snprintf(buf2, sizeof(buf2), "OK\n"); - send_client("="); - } } else if (strncmp(p, "name=", strlen("name=")) == 0) { char *e; diff --git a/tools/merovingian/utils/database.c b/tools/merovingian/utils/database.c --- a/tools/merovingian/utils/database.c +++ b/tools/merovingian/utils/database.c @@ -373,84 +373,3 @@ char *db_release(char *dbname) { return(NULL); } - -/* - * The following functions require rsync
MonetDB: Dec2016 - Fix string BAT that contain non-double-elimin...
Changeset: 0c475cc54c60 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c475cc54c60 Modified Files: gdk/gdk.h gdk/gdk_bbp.c Branch: Dec2016 Log Message: Fix string BAT that contain non-double-eliminated empty strings. This bug was fixed in changeset 1f5498568a24, here we fix the old BATs that were affected. diffs (245 lines): diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -858,7 +858,8 @@ typedef struct { #define GDKLIBRARY_INSERTED061032 /* inserted and deleted in BBP.dir */ #define GDKLIBRARY_HEADED 061033 /* head properties are stored */ #define GDKLIBRARY_NOKEY 061034 /* nokey values can't be trusted */ -#define GDKLIBRARY 061035 +#define GDKLIBRARY_BADEMPTY061035 /* possibility of duplicate empty str */ +#define GDKLIBRARY 061036 typedef struct BAT { /* static bat properties */ diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -868,6 +868,209 @@ fixwkbheap(void) } #endif +#ifdef GDKLIBRARY_BADEMPTY +/* There was a bug (fixed in changeset 1f5498568a24) which could + * result in empty strings not being double-eliminated. This code + * fixes the affected bats. + * Note that we only fix BATs whose string heap is still fully double + * eliminated. */ +static inline int +offsearch(const int *restrict offsets, int noffsets, int val) +{ + /* binary search on offsets for val, return whether present */ + int lo = 0, hi = noffsets - 1, mid; + + while (hi > lo) { + mid = (lo + hi) / 2; + if (offsets[mid] == val) + return 1; + if (offsets[mid] < val) + lo = mid + 1; + else + hi = mid - 1; + } + return offsets[lo] == val; +} + +static void +fixstroffheap(BAT *b, int *restrict offsets) +{ + long_str filename; + Heap h1;/* old offset heap */ + Heap *h;/* string heap */ + int noffsets = 0; + const size_t extralen = b->tvheap->hashash ? EXTRALEN : 0; + size_t pos, emptyoff = 0; + const char *nme, *bnme; + char *srcdir; + BUN i; + + assert(GDK_ELIMDOUBLES(b->tvheap)); + + nme = BBP_physical(b->batCacheid); + srcdir = GDKfilepath(NOFARM, BATDIR, nme, NULL); + if (srcdir == NULL) + GDKfatal("fixstroffheap: GDKmalloc failed\n"); + *strrchr(srcdir, DIR_SEP) = 0; + + /* load string heap */ + if (HEAPload(b->tvheap, nme, "theap", 0) != GDK_SUCCEED) + GDKfatal("fixstroffheap: loading string (theap) heap " +"for BAT %d failed\n", b->batCacheid); + h = b->tvheap; /* abbreviation */ + /* collect valid offsets */ + pos = GDK_STRHASHSIZE; + while (pos < h->free) { + const char *s; + size_t pad; + + pad = GDK_VARALIGN - (pos & (GDK_VARALIGN - 1)); + if (pad < sizeof(stridx_t)) + pad += GDK_VARALIGN; + pos += pad + extralen; + s = h->base + pos; + if (*s == '\0') + emptyoff = pos; + offsets[noffsets++] = (int) pos; /* < 65536, i.e. fits */ + pos += GDK_STRLEN(s); + } + HEAPfree(b->tvheap, 0); + if (emptyoff == 0) { + /* no empty string encountered in the heap, so the bug +* is extremely unlikely to have occurred (we need an +* area of zero bytes that is aligned on a var_t +* boundary and two var_t's long -- in the absence of +* an empty string, this can only happen if hashash is +* set and a string hashes to zero) */ + GDKfree(srcdir); + return; + } + + if ((bnme = strrchr(nme, DIR_SEP)) != NULL) + bnme++; + else + bnme = nme; + sprintf(filename, "BACKUP%c%s", DIR_SEP, bnme); + if (GDKmove(b->theap.farmid, srcdir, bnme, "tail", BAKDIR, bnme, "tail") != GDK_SUCCEED) + GDKfatal("fixstroffheap: cannot make backup of %s.tail\n", nme); + GDKfree(srcdir); + + /* load old offset heap */ + h1 = b->theap; + h1.filename = NULL; + h1.base = NULL; + h1.dirty = 0; + if (HEAPload(, filename, "tail", 0) != GDK_SUCCEED) + GDKfatal("fixstroffheap: loading old tail heap " +"for BAT %d failed\n", b->batCacheid); + + /* create new offset heap */ + b->theap.filename = GDKfilepath(NOFARM, NULL, nme, "tail"); + if (b->theap.filename == NULL) + GDKfatal("fixstroffheap: GDKmalloc failed\n"); + if (HEAPalloc(>theap, b->batCapacity, b->twidth) != GDK_SUCCEED) + GDKfatal("fixstroffheap: allocating new tail heap " +
MonetDB: default - Stop storing string hashes in string heaps.
Changeset: 7fa0ede6314b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7fa0ede6314b Modified Files: gdk/gdk_atoms.c Branch: default Log Message: Stop storing string hashes in string heaps. This change has no effect on existing string heaps. It only affects new ones. diffs (12 lines): diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c --- a/gdk/gdk_atoms.c +++ b/gdk/gdk_atoms.c @@ -1108,7 +1108,7 @@ strHeap(Heap *d, size_t cap) d->free = GDK_STRHASHTABLE * sizeof(stridx_t); d->dirty = 1; memset(d->base, 0, d->free); - d->hashash = 1; /* new string heaps get the hash value (and length) stored */ + d->hashash = 0; #ifndef NDEBUG /* fill should solve initialization problems within valgrind */ memset(d->base + d->free, 0, d->size - d->free); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list