Changeset: a50b28040bbf for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/a50b28040bbf
Modified Files:
gdk/gdk.h
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_logger.c
gdk/gdk_private.h
gdk/gdk_storage.c
sql/storage/bat/bat_logger.c
sql/test/BugTracker-2017/Tests/crash_after_oidx_on_sys_statistics.Bug-6251.test
Branch: default
Log Message:
Use different .tail files for different width string offsets.
This solves the problem why we had only 8 byte wide .tail files when
they were memory mapped. If the tail file needs to grow in width and
there are views on it, the views would get totally confused if the
heap file was memory mapped. With multiple files that problem doesn't
happen.
Also, improve BATappend speed, especially when appending to an empty
string bat.
diffs (truncated from 1206 to 300 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -742,8 +742,9 @@ typedef struct {
#define assert_shift_width(shift,width) assert(((shift) == 0 && (width) == 0)
|| ((unsigned)1<<(shift)) == (unsigned)(width))
#define GDKLIBRARY_MINMAX_POS 061042U /* first in Nov2019: no min/max
position; no BBPinfo value */
+#define GDKLIBRARY_TAILN 061043U /* first after Oct2020: str offset
heaps names don't take width into account */
/* if the version number is updated, also fix snapshot_bats() in bat_logger.c
*/
-#define GDKLIBRARY 061043U /* first after Oct2020 */
+#define GDKLIBRARY 061044U /* first after Oct2020 */
typedef struct BAT {
/* static bat properties */
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -329,7 +329,7 @@ VIEWreset(BAT *b)
.farmid = BBPselectfarm(b->batRole, b->ttype, offheap),
};
ATOMIC_INIT(&tail->refs, 1);
- strconcat_len(tail->filename, sizeof(tail->filename), nme, ".tail",
NULL);
+ settailname(tail, nme, b->ttype, b->twidth);
if (b->ttype && HEAPalloc(tail, cnt, Tsize(b), Tsize(b)) !=
GDK_SUCCEED) {
GDKfree(tail);
BBPunfix(v->batCacheid);
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -179,6 +179,55 @@ BATsetdims(BAT *b)
b->tvarsized = b->ttype == TYPE_void || BATatoms[b->ttype].atomPut !=
NULL;
}
+const char *
+gettailname(const BAT *b)
+{
+ if (b->ttype != TYPE_str)
+ return "tail";
+ switch (b->twidth) {
+ case 1:
+ return "tail1";
+ case 2:
+ return "tail2";
+#if SIZEOF_VAR_T == 8
+ case 4:
+ return "tail4";
+#endif
+ default:
+ return "tail";
+ }
+}
+
+void
+settailname(Heap *restrict tail, const char *restrict physnme, int tt, int
width)
+{
+ strconcat_len(tail->filename, sizeof(tail->filename), physnme,
+ ".tail", NULL);
+ if (tt == TYPE_str) {
+ switch (width) {
+ case 1:
+ strconcat_len(tail->filename,
+ sizeof(tail->filename), physnme,
+ ".tail1", NULL);
+ break;
+ case 2:
+ strconcat_len(tail->filename,
+ sizeof(tail->filename), physnme,
+ ".tail2", NULL);
+ break;
+#if SIZEOF_VAR_T == 8
+ case 4:
+ strconcat_len(tail->filename,
+ sizeof(tail->filename), physnme,
+ ".tail4", NULL);
+ break;
+#endif
+ default:
+ break;
+ }
+ }
+}
+
/*
* @- BAT allocation
* Allocate BUN heap and variable-size atomheaps (see e.g. strHeap).
@@ -223,15 +272,13 @@ COLnew(oid hseq, int tt, BUN cap, role_t
if (ATOMstorage(tt) == TYPE_msk)
cap /= 8; /* 8 values per byte */
+ else if (tt == TYPE_str)
+ settailname(bn->theap, BBP_physical(bn->batCacheid), tt,
bn->twidth);
/* alloc the main heaps */
if (tt && HEAPalloc(bn->theap, cap, bn->twidth, ATOMsize(bn->ttype)) !=
GDK_SUCCEED) {
goto bailout;
}
- if (bn->theap->storage == STORE_MMAP) {
- bn->twidth = ATOMsize(bn->ttype);
- bn->tshift = ATOMelmshift(Tsize(bn));
- }
if (bn->tvheap && ATOMheap(tt, bn->tvheap, cap) != GDK_SUCCEED) {
goto bailout;
@@ -680,8 +727,9 @@ BATdestroy(BAT *b)
static void
heapmove(Heap *dst, Heap *src)
{
- HEAPfree(dst, false);
- /* copy all fields of src except filename and refs */
+ HEAPfree(dst, strcmp(dst->filename, src->filename) != 0);
+ /* copy all fields of src except refs */
+ strcpy_len(dst->filename, src->filename, sizeof(dst->filename));
dst->free = src->free;
dst->size = src->size;
dst->base = src->base;
@@ -808,9 +856,8 @@ COLcopy(BAT *b, int tt, bool writable, r
.farmid = BBPselectfarm(role, b->ttype,
varheap),
.parentid = bn->batCacheid,
};
- strconcat_len(bthp.filename, sizeof(bthp.filename),
- BBP_physical(bn->batCacheid),
- ".tail", NULL);
+ settailname(&bthp, BBP_physical(bn->batCacheid),
+ bn->ttype, bn->twidth);
strconcat_len(thp.filename, sizeof(thp.filename),
BBP_physical(bn->batCacheid),
".theap", NULL);
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -66,7 +66,7 @@ insert_string_bat(BAT *b, BAT *n, struct
#if SIZEOF_VAR_T == 8
unsigned int tiv; /* tail value-as-int */
#endif
- var_t v; /* value */
+ var_t v = GDK_VAROFFSET; /* value */
size_t off; /* offset within n's string heap */
BUN cnt = ci->ncand;
BUN oldcnt = BATcount(b);
@@ -80,9 +80,9 @@ insert_string_bat(BAT *b, BAT *n, struct
return GDK_SUCCEED;
ni = bat_iterator(n);
tp = NULL;
- if ((!GDK_ELIMDOUBLES(b->tvheap) || oldcnt == 0) &&
- !GDK_ELIMDOUBLES(n->tvheap) &&
- b->tvheap->hashash == n->tvheap->hashash) {
+ if (oldcnt == 0 || (!GDK_ELIMDOUBLES(b->tvheap) &&
+ !GDK_ELIMDOUBLES(n->tvheap) &&
+ b->tvheap->hashash == n->tvheap->hashash)) {
if (b->batRole == TRANSIENT || b->tvheap == n->tvheap) {
/* If b is in the transient farm (i.e. b will
* never become persistent), we try some
@@ -128,6 +128,12 @@ insert_string_bat(BAT *b, BAT *n, struct
MT_lock_unset(&n->theaplock);
b->batDirtydesc = true;
toff = 0;
+ v = n->twidth == 1 ? GDK_VAROFFSET + 1 :
+ n->twidth == 2 ? GDK_VAROFFSET + (1 <<
9) :
+#if SIZEOF_VAR_T == 8
+ n->twidth != 4 ? (var_t) 1 << 33 :
+#endif
+ (var_t) 1 << 17;
} else if (b->tvheap->parentid == n->tvheap->parentid &&
ci->tpe == cand_dense) {
toff = 0;
@@ -135,6 +141,25 @@ insert_string_bat(BAT *b, BAT *n, struct
unshare_varsized_heap(b) != GDK_SUCCEED) {
return GDK_FAIL;
}
+ } else if (oldcnt == 0) {
+ v = n->twidth == 1 ? GDK_VAROFFSET + 1 :
+ n->twidth == 2 ? GDK_VAROFFSET + (1 << 9) :
+#if SIZEOF_VAR_T == 8
+ n->twidth != 4 ? (var_t) 1 << 33 :
+#endif
+ (var_t) 1 << 17;
+ if (b->tvheap->size < n->tvheap->free) {
+ Heap *h = HEAPgrow(b->tvheap, n->tvheap->free);
+ if (h == NULL)
+ return GDK_FAIL;
+ MT_lock_set(&b->theaplock);
+ HEAPdecref(b->tvheap, false);
+ b->tvheap = h;
+ MT_lock_unset(&b->theaplock);
+ }
+ memcpy(b->tvheap->base, n->tvheap->base,
n->tvheap->free);
+ b->tvheap->free = n->tvheap->free;
+ toff = 0;
}
if (toff == ~(size_t) 0 && cnt > 1024 && b->tvheap->free >=
n->tvheap->free) {
/* If b and n aren't sharing their string
@@ -196,9 +221,8 @@ insert_string_bat(BAT *b, BAT *n, struct
return GDK_FAIL;
/* make sure there is (vertical) space in the offset heap, we
- * may have to widen the heap later */
- if (GDKupgradevarheap(b, (var_t) b->tvheap->size, BATcount(b) + cnt,
- false) != GDK_SUCCEED)
+ * may also widen if v was set to some limit above */
+ if (GDKupgradevarheap(b, v, oldcnt + cnt < b->batCapacity ?
b->batCapacity : oldcnt + cnt, false) != GDK_SUCCEED)
return GDK_FAIL;
if (toff == 0 && n->twidth == b->twidth && ci->tpe == cand_dense) {
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -494,8 +494,7 @@ heapinit(BAT *b, const char *buf, int *h
b->theap->free = (size_t) free;
b->theap->size = (size_t) size;
b->theap->base = NULL;
- strconcat_len(b->theap->filename, sizeof(b->theap->filename),
- filename, ".tail", NULL);
+ settailname(b->theap, filename, t, width);
b->theap->storage = (storage_t) storage;
b->theap->newstorage = (storage_t) storage;
b->theap->farmid = BBPselectfarm(PERSISTENT, b->ttype, offheap);
@@ -729,8 +728,9 @@ BBPreadEntries(FILE *fp, unsigned bbpver
/* check that the necessary files for all BATs exist and are large
* enough */
static gdk_return
-BBPcheckbats(void)
+BBPcheckbats(unsigned bbpversion)
{
+ (void) bbpversion;
for (bat bid = 1; bid < (bat) ATOMIC_GET(&BBPsize); bid++) {
struct stat statb;
BAT *b;
@@ -744,11 +744,34 @@ BBPcheckbats(void)
/* no files needed */
continue;
}
- path = GDKfilepath(0, BATDIR, BBP_physical(b->batCacheid),
"tail");
+ path = GDKfilepath(0, BATDIR, b->theap->filename, NULL);
if (path == NULL)
return GDK_FAIL;
- if (MT_stat(path, &statb) < 0) {
- GDKsyserror("BBPcheckbats: cannot stat file %s
(expected size %zu)\n",
+#ifdef GDKLIBRARY_TAILN
+ /* if bbpversion > GDKLIBRARY_TAILN, the offset heap can
+ * exist with either name .tail1 (etc) or .tail, if <=
+ * GDKLIBRARY_TAILN, only with .tail */
+ char tailsave = 0;
+ size_t taillen = 0;
+ if (b->ttype == TYPE_str &&
+ b->twidth < SIZEOF_VAR_T) {
+ /* old version: .tail, not .tail1, .tail2, .tail4 */
+ taillen = strlen(path) - 1;
+ tailsave = path[taillen];
+ path[taillen] = 0;
+ }
+#endif
+ if (MT_stat(path, &statb) < 0
+#ifdef GDKLIBRARY_TAILN
+ && bbpversion > GDKLIBRARY_TAILN
+ && b->ttype == TYPE_str
+ && b->twidth < SIZEOF_VAR_T
+ && (path[taillen] = tailsave) != 0
+ && MT_stat(path, &statb) < 0
+#endif
+ ) {
+
+ GDKsyserror("cannot stat file %s (expected size %zu)\n",
path, b->theap->free);
GDKfree(path);
return GDK_FAIL;
@@ -764,7 +787,7 @@ BBPcheckbats(void)
if (path == NULL)
return GDK_FAIL;
if (MT_stat(path, &statb) < 0) {
- GDKsyserror("BBPcheckbats: cannot stat file
%s\n",
+ GDKsyserror("cannot stat file %s\n",
path);
GDKfree(path);
return GDK_FAIL;
@@ -805,6 +828,7 @@ BBPheader(FILE *fp, int *lineno)
return 0;
}
if (bbpversion != GDKLIBRARY &&
+ bbpversion != GDKLIBRARY_TAILN &&
bbpversion != GDKLIBRARY_MINMAX_POS) {
TRC_CRITICAL(GDK, "incompatible BBP version: expected 0%o, got
0%o. "
"This database was probably created by a %s
version of MonetDB.",
@@ -958,6 +982,30 @@ BBPaddfarm(const char *dirname, uint32_t
return GDK_FAIL;
}
+#ifdef GDKLIBRARY_TAILN
+static gdk_return
+movestrbats(void)
+{
+ for (bat bid = 1, nbat = (bat) ATOMIC_GET(&BBPsize); bid < nbat; bid++)
{
+ BAT *b = BBP_desc(bid);
+ if (b == NULL) {
+ /* not a valid BAT */
+ continue;
+ }
+ if (b->ttype != TYPE_str || b->twidth == SIZEOF_VAR_T)
+ continue;
+ char *oldpath = GDKfilepath(0, BATDIR,
BBP_physical(b->batCacheid), "tail");
+ char *newpath = GDKfilepath(0, BATDIR, b->theap->filename,
NULL);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
