Changeset: 4118258f2b52 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/4118258f2b52
Modified Files:
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_join.c
gdk/gdk_private.h
gdk/gdk_select.c
gdk/gdk_string.c
Branch: default
Log Message:
During upgrade, convert all string and blob bats so that offsets for nil are
always 0.
Also adapt code to make use of the fact that all nils in string bats now
have offset 0. (blob bats don't get this because var-sized bats in
general can store nils in the vheap at offsets other than 0.)
diffs (truncated from 698 to 300 lines):
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -160,25 +160,19 @@ insert_string_bat(BAT *b, BATiter *ni, s
toff = 0;
b->tvkey |= ni->vkey;
MT_thread_setalgorithm("shared vheap", __func__);
- } else if (ni->vhfree <= GDK_STRHASHSIZE &&
- (!GDK_ELIMDOUBLES(b->tvheap) ||
- strLocate(b->tvheap, str_nil) == 0)) {
+ } else if (ni->vhfree <= GDK_STRHASHSIZE) {
/* the incoming bat's vheap is empty, meaning the bat
- * contains only nil values, if the destination bat's
- * nil is also at offset 0, we can just copy zeros */
+ * contains only nil values: we can just copy zeros */
toff = 0;
MT_thread_setalgorithm("empty source vheap (just nils)",
__func__);
} else if (b->ustr) {
MT_thread_setalgorithm("individual inserts into ustr",
__func__);
} else if (mayshare && b->batRole == TRANSIENT &&
- (oldcnt == 0 || (b->tvheap->free <= GDK_STRHASHSIZE &&
- (!GDK_ELIMDOUBLES(ni->vh) ||
- strLocate(ni->vh, str_nil) == 0)))) {
+ (oldcnt == 0 || b->tvheap->free <= GDK_STRHASHSIZE)) {
/* we can share the vheaps, so we then only need to
* append the offsets; the condition is: b is empty, or
- * its vheap is empty (i.e. there are only nils) and the
- * incoming heap also stores them with offset 0 */
+ * its vheap is empty (i.e. there are only nils) */
MT_lock_set(&b->theaplock);
bat bid = b->tvheap->parentid;
HEAPdecref(b->tvheap, bid == b->batCacheid);
@@ -3453,70 +3447,23 @@ BATcount_no_nil(BAT *b, BAT *s)
cnt += !is_inet6_nil(((const inet6 *)
p)[canditer_next(&ci) - hseq]);
break;
case TYPE_str:
- if (bi.vkey) {
- if (GDK_ELIMDOUBLES(bi.vh)) {
- off = strLocate(bi.vh, str_nil);
- if (off == (var_t) -2) {
- cnt = ci.ncand;
- break;
- }
- } else {
- off = 0;
- }
- switch (bi.width) {
- case 1:
- if (off != 0)
- off -= GDK_VAROFFSET;
- CAND_LOOP(&ci)
- cnt += (var_t) ((const uint8_t *)
p)[canditer_next(&ci) - hseq] != off;
- break;
- case 2:
- if (off != 0)
- off -= GDK_VAROFFSET;
- CAND_LOOP(&ci)
- cnt += (var_t) ((const uint16_t *)
p)[canditer_next(&ci) - hseq] != off;
- break;
- case 4:
- CAND_LOOP(&ci)
- cnt += (var_t) ((const uint32_t *)
p)[canditer_next(&ci) - hseq] != off;
- break;
-#if SIZEOF_VAR_T == 8
- case 8:
- CAND_LOOP(&ci)
- cnt += (var_t) ((const uint64_t *)
p)[canditer_next(&ci) - hseq] != off;
- break;
-#endif
- default:
- MT_UNREACHABLE();
- }
- break;
- }
- base = bi.vh->base;
switch (bi.width) {
case 1:
- CAND_LOOP(&ci) {
- off = (var_t) ((const uint8_t *)
p)[canditer_next(&ci) - hseq];
- cnt += off != 0 && base[off + GDK_VAROFFSET] !=
'\200';
- }
+ CAND_LOOP(&ci)
+ cnt += (var_t) ((const uint8_t *)
p)[canditer_next(&ci) - hseq] != 0;
break;
case 2:
- CAND_LOOP(&ci) {
- off = (var_t) ((const uint16_t *)
p)[canditer_next(&ci) - hseq];
- cnt += off != 0 && base[off + GDK_VAROFFSET] !=
'\200';
- }
+ CAND_LOOP(&ci)
+ cnt += (var_t) ((const uint16_t *)
p)[canditer_next(&ci) - hseq] != 0;
break;
case 4:
- CAND_LOOP(&ci) {
- off = (var_t) ((const uint32_t *)
p)[canditer_next(&ci) - hseq];
- cnt += off != 0 && base[off] != '\200';
- }
+ CAND_LOOP(&ci)
+ cnt += (var_t) ((const uint32_t *)
p)[canditer_next(&ci) - hseq] != 0;
break;
#if SIZEOF_VAR_T == 8
case 8:
- CAND_LOOP(&ci) {
- off = (var_t) ((const uint64_t *)
p)[canditer_next(&ci) - hseq];
- cnt += off != 0 && base[off] != '\200';
- }
+ CAND_LOOP(&ci)
+ cnt += (var_t) ((const uint64_t *)
p)[canditer_next(&ci) - hseq] != 0;
break;
#endif
default:
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1649,6 +1649,291 @@ BBPjson_upgrade(json_storage_conversion
}
#endif
+#ifdef GDKLIBRARY_USTR
+static bool file_exists(int farmid, const char *dir, const char *name, const
char *ext);
+
+static gdk_return
+fixstrnilbat(BAT *b)
+{
+ const char *nme = BBP_physical(b->batCacheid);
+ char srcdir[MAXPATH];
+ var_t niloff = 0;
+
+ if (GDKfilepath(srcdir, sizeof(srcdir), NOFARM, BATDIR, nme, NULL) !=
GDK_SUCCEED) {
+ return GDK_FAIL;
+ }
+ char *s;
+ if ((s = strrchr(srcdir, DIR_SEP)) != NULL)
+ *s = 0;
+ const char *bnme;
+ if ((bnme = strrchr(nme, DIR_SEP)) != NULL)
+ bnme++;
+ else
+ bnme = nme;
+ long_str filename;
+ strtconcat(filename, sizeof(filename), "BACKUP", DIR_SEP_STR, bnme,
NULL);
+ const char *t = BATtailname(b);
+
+ if (file_exists(0, BAKDIR, bnme, t)) {
+ /* already done by another upgrade */
+ return GDK_SUCCEED;
+ }
+
+ if (HEAPload(b->tvheap, nme, "theap", false) != GDK_SUCCEED) {
+ TRC_CRITICAL(GDK, "loading string heap "
+ "for BAT %d failed\n", b->batCacheid);
+ return GDK_FAIL;
+ }
+ strCleanHash(b->tvheap, false);
+ b->tvheap->cleanhash = true;
+ b->tvheap->dirty = false;
+ if (GDK_ELIMDOUBLES(b->tvheap)) {
+ niloff = oldstrnilLocate(b->tvheap);
+ if (niloff == (var_t) -2) {
+ /* fully double eliminated, and str_nil does not
+ * occur: nothing to do */
+ HEAPfree(b->tvheap, false);
+ return GDK_SUCCEED;
+ }
+ }
+
+ if (GDKmove(b->theap->farmid, srcdir, bnme, t,
+ BAKDIR, bnme, t, false) != GDK_SUCCEED) {
+ HEAPfree(b->tvheap, false);
+ TRC_CRITICAL(GDK, "cannot make backup of %s.%s\n", nme, t);
+ return GDK_FAIL;
+ }
+
+ Heap h1 = *b->theap; /* old heap */
+ h1.base = NULL;
+ h1.dirty = false;
+ strtconcat(h1.filename, sizeof(h1.filename), filename, ".", t, NULL);
+ if (HEAPload(&h1, filename, t, false) != GDK_SUCCEED) {
+ HEAPfree(b->tvheap, false);
+ TRC_CRITICAL(GDK, "loading old tail heap "
+ "for BAT %d failed\n", b->batCacheid);
+ return GDK_FAIL;
+ }
+
+ /* create new heap */
+ Heap *h2 = GDKmalloc(sizeof(Heap));
+ if (h2 == NULL) {
+ HEAPfree(&h1, false);
+ HEAPfree(b->tvheap, false);
+ TRC_CRITICAL(GDK, "allocating new heaps "
+ "for BAT %d failed\n", b->batCacheid);
+ return GDK_FAIL;
+ }
+ *h2 = *b->theap;
+ h2->base = NULL;
+ if (HEAPalloc(h2, b->batCapacity, b->twidth) != GDK_SUCCEED) {
+ GDKfree(h2);
+ HEAPfree(&h1, false);
+ HEAPfree(b->tvheap, false);
+ TRC_CRITICAL(GDK, "allocating new tail heap "
+ "for BAT %d failed\n", b->batCacheid);
+ return GDK_FAIL;
+ }
+ h2->dirty = true;
+ h2->free = h1.free;
+ ATOMIC_INIT(&h2->refs, 1);
+
+ const char *vbase = b->tvheap->base;
+ size_t vfree = b->tvheap->free;
+ switch (b->twidth) {
+ case 1: {
+ const uint8_t *p1 = (const uint8_t *) h1.base;
+ uint8_t *p2 = (uint8_t *) h2->base;
+ vfree -= 8192;
+ niloff -= 8192;
+ for (BUN i = 0; i < b->batCount; i++) {
+ uint8_t v = p1[i];
+ if (v == niloff)
+ p2[i] = 0;
+ else if (v >= vfree || vbase[v + 8192] == '\200')
+ p2[i] = 0;
+ else
+ p2[i] = v;
+ }
+ break;
+ }
+ case 2: {
+ const uint16_t *p1 = (const uint16_t *) h1.base;
+ uint16_t *p2 = (uint16_t *) h2->base;
+ vfree -= 8192;
+ niloff -= 8192;
+ for (BUN i = 0; i < b->batCount; i++) {
+ uint16_t v = p1[i];
+ if (v == niloff)
+ p2[i] = 0;
+ else if (v >= vfree || vbase[v + 8192] == '\200')
+ p2[i] = 0;
+ else
+ p2[i] = v;
+ }
+ break;
+ }
+ case 4: {
+ const uint32_t *p1 = (const uint32_t *) h1.base;
+ uint32_t *p2 = (uint32_t *) h2->base;
+ for (BUN i = 0; i < b->batCount; i++) {
+ uint32_t v = p1[i];
+ if (v >= vfree || vbase[v] == '\200')
+ p2[i] = 0;
+ else
+ p2[i] = v;
+ }
+ break;
+ }
+#if SIZEOF_VAR_T == 8
+ case 8: {
+ const uint64_t *p1 = (const uint64_t *) h1.base;
+ uint64_t *p2 = (uint64_t *) h2->base;
+ for (BUN i = 0; i < b->batCount; i++) {
+ uint64_t v = p1[i];
+ if (v >= vfree || vbase[v] == '\200')
+ p2[i] = 0;
+ else
+ p2[i] = v;
+ }
+ break;
+ }
+#endif
+ default:
+ MT_UNREACHABLE();
+ }
+ HEAPfree(&h1, false);
+ HEAPfree(b->tvheap, false);
+ if (HEAPsave(h2, nme, t, true, h2->free, NULL) != GDK_SUCCEED) {
+ HEAPdecref(h2, true);
+ TRC_CRITICAL(GDK, "saving heap failed\n");
+ return GDK_FAIL;
+ }
+ HEAPdecref(b->theap, false);
+ b->theap = h2;
+ HEAPfree(h2, false);
+
+ return GDK_SUCCEED;
+}
+
+static gdk_return
+fixblobnilbat(BAT *b)
+{
+ const char *nme = BBP_physical(b->batCacheid);
+ char srcdir[MAXPATH];
+
+ if (GDKfilepath(srcdir, sizeof(srcdir), NOFARM, BATDIR, nme, NULL) !=
GDK_SUCCEED) {
+ return GDK_FAIL;
+ }
+ char *s;
+ if ((s = strrchr(srcdir, DIR_SEP)) != NULL)
+ *s = 0;
+ const char *bnme;
+ if ((bnme = strrchr(nme, DIR_SEP)) != NULL)
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
