Changeset: 4d1315f86145 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4d1315f86145
Added Files:
sql/test/BugTracker-2013/Tests/case_when.Bug-3395.sql
Modified Files:
clients/mapiclient/tomograph.c
gdk/gdk.h
gdk/gdk_aggr.c
gdk/gdk_atoms.c
gdk/gdk_bbp.c
gdk/gdk_calc.h
gdk/gdk_heap.c
gdk/gdk_utils.c
monetdb5/mal/mal_factory.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_instruction.h
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_readline.c
monetdb5/modules/kernel/aggr.c
monetdb5/modules/kernel/aggr.mal
monetdb5/modules/kernel/aggr.mal.sh
monetdb5/modules/mal/tablet.c
monetdb5/optimizer/opt_qep.c
sql/backends/monet5/gsl/Tests/gsl01.stable.err
sql/backends/monet5/gsl/Tests/gsl01.stable.out
sql/backends/monet5/gsl/gsl.c
sql/backends/monet5/sql.mx
sql/scripts/39_analytics.sql
sql/server/rel_dump.c
sql/test/BugTracker-2013/Tests/All
sql/test/BugTracker-2013/Tests/Cannot_Find_Column_Type.Bug-3394.sql
tools/merovingian/client/monetdb.c
Branch: default
Log Message:
Merger
diffs (truncated from 1476 to 300 lines):
diff --git a/clients/mapiclient/tomograph.c b/clients/mapiclient/tomograph.c
--- a/clients/mapiclient/tomograph.c
+++ b/clients/mapiclient/tomograph.c
@@ -2157,7 +2157,7 @@ doProfile(void *d)
for (i = 0; i < MAXTHREADS; i++)
threads[i] = topbox++;
- /* sent single query */
+ /* send single query */
if (sqlstatement) {
doRequest(dbhsql, sqlstatement);
}
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -2275,7 +2275,7 @@ gdk_export str GDKstrdup(const char *s);
void *_res = GDKmmap(_path, _mode, _len); \
ALLOCDEBUG \
fprintf(stderr, \
- "#GDKmmap(%s,0x%x," SZFMT ") -> " SZFMT \
+ "#GDKmmap(%s,0x%x," SZFMT ") -> " PTRFMT \
" %s[%s:%d]\n", \
_path ? _path : "NULL", _mode, _len, \
PTRFMTCAST _res, \
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -2183,10 +2183,16 @@ BATmax(BAT *b, void *aggr)
/* ---------------------------------------------------------------------- */
-/* median */
+/* quantiles/median */
BAT *
-BATgroupmedian(BAT *b, BAT *g, BAT *e, BAT *s, int tp, int skip_nils, int
abort_on_error)
+
+BATgroupmedian(BAT *b, BAT *g, BAT *e, BAT *s, int tp, int skip_nils, int
abort_on_error) {
+ return BATgroupquantile(b,g,e,s,tp,0.5,skip_nils,abort_on_error);
+}
+
+BAT *
+ BATgroupquantile(BAT *b, BAT *g, BAT *e, BAT *s, int tp, double quantile,int
skip_nils, int abort_on_error)
{
int freeb = 0, freeg = 0;
oid min, max;
@@ -2201,20 +2207,23 @@ BATgroupmedian(BAT *b, BAT *g, BAT *e, B
const void *nil;
int (*atomcmp)(const void *, const void *);
const char *err;
-
(void) abort_on_error;
if ((err = BATgroupaggrinit(b, g, e, s, &min, &max, &ngrp, &start, &end,
&cnt, &cand, &candend)) != NULL) {
- GDKerror("BATgroupmedian: %s\n", err);
+ GDKerror("BATgroupquantile: %s\n", err);
return NULL;
}
assert(tp == b->ttype);
if (!ATOMlinear(b->ttype)) {
- GDKerror("BATgroupmedian: cannot determine median on "
+ GDKerror("BATgroupquantile: cannot determine quantile on "
"non-linear type %s\n", ATOMname(b->ttype));
return NULL;
}
+ if (quantile < 0 || quantile > 1) {
+ GDKerror("BATgroupquantile: cannot determine quantile for p=%f
(p has to be in [0,1])\n",quantile);
+ return NULL;
+ }
if (BATcount(b) == 0 || ngrp == 0) {
/* trivial: no medians, so return bat aligned with e with
@@ -2305,7 +2314,8 @@ BATgroupmedian(BAT *b, BAT *g, BAT *e, B
nil, 0, Tsize(bn));
nils++;
} else {
- v = BUNtail(bi, BUNfirst(b) + (r + p -
1) / 2);
+ // actual selection of quantile value
for groups
+ v = BUNtail(bi, (oid)( BUNfirst(b) + (r
+ p - 1) * quantile));
bunfastins_nocheck(bn, BUNlast(bn), 0,
v, 0, Tsize(bn));
nils += (*atomcmp)(v, nil) == 0;
@@ -2321,7 +2331,8 @@ BATgroupmedian(BAT *b, BAT *g, BAT *e, B
}
BATseqbase(bn, min);
} else {
- v = BUNtail(bi, BUNfirst(b) + (BATcount(b) - 1) / 2);
+ // actual selection of quantile value
+ v = BUNtail(bi, (oid) (BUNfirst(b) + (BATcount(b) - 1) *
quantile));
BUNappend(bn, v, FALSE);
BATseqbase(bn, 0);
nils += (*atomcmp)(v, nil) == 0;
diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -963,6 +963,9 @@ strHash(const char *s)
return res;
}
+/* if at least (2*SIZEOF_BUN), also store length (heaps are then
+ * incompatible) */
+#define EXTRALEN ((SIZEOF_BUN + GDK_VARALIGN - 1) & ~(GDK_VARALIGN - 1))
void
strCleanHash(Heap *h, int rebuild)
@@ -971,6 +974,50 @@ strCleanHash(Heap *h, int rebuild)
if (!GDK_ELIMDOUBLES(h)) {
/* flush hash table for security */
memset(h->base, 0, GDK_STRHASHSIZE);
+ } else {
+ /* rebuild hash table for double elimination
+ *
+ * If appending strings to the BAT was aborted, if the
+ * heap was memory mapped, the hash in the string heap
+ * may well be incorrect. Therefore we don't trust it
+ * when we read in a string heap and we rebuild the
+ * complete table (it is small, so this won't take any
+ * time at all). */
+ size_t pad, pos;
+ const size_t extralen = h->hashash ? EXTRALEN : 0;
+ stridx_t *bucket;
+ BUN off, strhash;
+ const char *s;
+
+ memset(h->base, 0, GDK_STRHASHSIZE);
+ pos = GDK_STRHASHSIZE;
+ while (pos < h->free) {
+ pad = GDK_VARALIGN - (pos & (GDK_VARALIGN - 1));
+ if (pad < sizeof(stridx_t))
+ pad += GDK_VARALIGN;
+ pos += pad + extralen;
+ s = h->base + pos;
+ if (h->hashash)
+ strhash = ((const BUN *) s)[-1];
+ else
+ GDK_STRHASH(s, strhash);
+ off = strhash & GDK_STRHASHMASK;
+ bucket = ((stridx_t *) h->base) + off;
+ *bucket = pos - extralen - sizeof(stridx_t);
+ pos += GDK_STRLEN(s);
+ }
+#ifndef NDEBUG
+ pos = GDK_STRHASHSIZE;
+ while (pos < h->free) {
+ pad = GDK_VARALIGN - (pos & (GDK_VARALIGN - 1));
+ if (pad < sizeof(stridx_t))
+ pad += GDK_VARALIGN;
+ pos += pad + extralen;
+ s = h->base + pos;
+ assert(strLocate(h, s) != 0);
+ pos += GDK_STRLEN(s);
+ }
+#endif
}
}
@@ -979,15 +1026,11 @@ strCleanHash(Heap *h, int rebuild)
* the location of a string in the heap if it exists. Otherwise it
* returns zero.
*/
-/* if at least (2*SIZEOF_BUN), also store length (heaps are then
- * incompatible) */
-#define EXTRALEN ((SIZEOF_BUN + GDK_VARALIGN - 1) & ~(GDK_VARALIGN - 1))
-
var_t
strLocate(Heap *h, const char *v)
{
stridx_t *ref, *next;
- size_t extralen = h->hashash ? EXTRALEN : 0;
+ const size_t extralen = h->hashash ? EXTRALEN : 0;
/* search hash-table, if double-elimination is still in place */
BUN off;
@@ -1012,7 +1055,7 @@ strPut(Heap *h, var_t *dst, const char *
size_t elimbase = GDK_ELIMBASE(h->free);
size_t pad = GDK_VARALIGN - (h->free & (GDK_VARALIGN - 1));
size_t pos, len = GDK_STRLEN(v);
- size_t extralen = h->hashash ? EXTRALEN : 0;
+ const size_t extralen = h->hashash ? EXTRALEN : 0;
stridx_t *bucket, *ref, *next;
BUN off, strhash;
@@ -1021,24 +1064,30 @@ strPut(Heap *h, var_t *dst, const char *
off &= GDK_STRHASHMASK;
bucket = ((stridx_t *) h->base) + off;
- /* search hash-table, if double-elimination is still in place */
- if (elimbase == 0) { /* small string heap (<64KB) -- fully double
eliminated */
- for (ref = bucket; *ref; ref = next) { /* search the linked
list */
+ /* if double-elimination is still in place, search hash-table */
+ if (elimbase == 0) {
+ /* small string heap (<64KB) -- fully double eliminated */
+ for (ref = bucket; *ref; ref = next) {
+ /* search the linked list */
next = (stridx_t *) (h->base + *ref);
- if (GDK_STRCMP(v, (str) (next + 1) + extralen) == 0) {
/* found */
+ if (GDK_STRCMP(v, (str) (next + 1) + extralen) == 0) {
+ /* found */
pos = sizeof(stridx_t) + *ref + extralen;
return *dst = (var_t) (pos >> GDK_VARSHIFT);
}
}
/* is there room for the next pointer in the padding space? */
- if (pad < sizeof(stridx_t))
- pad += GDK_VARALIGN; /* if not, pad more */
+ if (pad < sizeof(stridx_t)) {
+ /* if not, pad more */
+ pad += GDK_VARALIGN;
+ }
} else if (*bucket) {
/* large string heap (>=64KB) --
* opportunistic/probabilistic double elimination */
pos = elimbase + *bucket + extralen;
if (GDK_STRCMP(v, h->base + pos) == 0) {
- return *dst = (var_t) (pos >> GDK_VARSHIFT); /*
already in heap; do not insert! */
+ /* already in heap; do not insert! */
+ return *dst = (var_t) (pos >> GDK_VARSHIFT);
}
#if SIZEOF_VAR_T >= SIZEOF_VOID_P /* in fact SIZEOF_VAR_T == SIZEOF_VOID_P */
if (extralen == 0)
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -3165,8 +3165,14 @@ do_backup(const char *srcdir, const char
int ret = 0;
/* direct mmap is unprotected (readonly usage, or has WAL
- * protection) */
- if (h->storage != STORE_MMAP) {
+ * protection); however, if we're backing up for subcommit
+ * and a backup already exists in the main backup directory
+ * (see GDKupgradevarheap), move the file */
+ if (subcommit && file_exists(BAKDIR, nme, extbase)) {
+ assert(h->storage == STORE_MMAP);
+ if (file_move(BAKDIR, SUBDIR, nme, extbase))
+ return -1;
+ } else if (h->storage != STORE_MMAP) {
/* STORE_PRIV saves into X.new files. Two cases could
* happen. The first is when a valid X.new exists
* because of an access change or a previous
diff --git a/gdk/gdk_calc.h b/gdk/gdk_calc.h
--- a/gdk/gdk_calc.h
+++ b/gdk/gdk_calc.h
@@ -126,6 +126,8 @@ gdk_export BAT *BATgroupsize(BAT *b, BAT
gdk_export BAT *BATgroupmin(BAT *b, BAT *g, BAT *e, BAT *s, int tp, int
skip_nils, int abort_on_error);
gdk_export BAT *BATgroupmax(BAT *b, BAT *g, BAT *e, BAT *s, int tp, int
skip_nils, int abort_on_error);
gdk_export BAT *BATgroupmedian(BAT *b, BAT *g, BAT *e, BAT *s, int tp, int
skip_nils, int abort_on_error);
+gdk_export BAT *BATgroupquantile(BAT *b, BAT *g, BAT *e, BAT *s, int tp,
double quantile, int skip_nils, int abort_on_error);
+
/* helper function for grouped aggregates */
gdk_export const char *BATgroupaggrinit(
const BAT *b, const BAT *g, const BAT *e, const BAT *s,
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -390,6 +390,20 @@ HEAPshrink(Heap *h, size_t size)
return -1;
}
+/* returns 1 if the file exists */
+static int
+file_exists(const char *dir, const char *name, const char *ext)
+{
+ long_str path;
+ struct stat st;
+ int ret;
+
+ GDKfilepath(path, dir, name, ext);
+ ret = stat(path, &st);
+ IODEBUG THRprintf(GDKstdout, "#stat(%s) = %d\n", path, ret);
+ return (ret == 0);
+}
+
int
GDKupgradevarheap(COLrec *c, var_t v, int copyall)
{
@@ -403,6 +417,7 @@ GDKupgradevarheap(COLrec *c, var_t v, in
#endif
size_t i, n;
size_t savefree;
+ const char *filename;
assert(c->heap.parentid == 0);
assert(width != 0);
@@ -414,11 +429,65 @@ GDKupgradevarheap(COLrec *c, var_t v, in
}
assert(c->width < width);
assert(c->shift < shift);
+
/* if copyall is set, we need to convert the whole heap, since
* we may be in the middle of an insert loop that adjusts the
* free value at the end; otherwise only copy the area
* indicated by the "free" pointer */
n = (copyall ? c->heap.size : c->heap.free) >> c->shift;
+
+ /* for memory mapped files, create a backup copy before widening
+ *
+ * this solves a problem that we don't control what's in the
+ * actual file until the next commit happens, so a crash might
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
