Changeset: 17fa6642c874 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/17fa6642c874
Modified Files:
gdk/gdk_align.c
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_heap.c
gdk/gdk_imprints.c
gdk/gdk_private.h
gdk/gdk_tm.c
Branch: Jul2021
Log Message:
Backported changeset d0abd3472eda, 1dac463d64c8, 0743319615a9, 2e3d1d0cbc4a,
9dd7ea5944d3, e901f1e4d8cf.
These changesets fix various problems leading to missing tail files.
The backport is done in such a way that there are no ABI changes, so
they have been edited by hand.
diffs (truncated from 488 to 300 lines):
diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c
--- a/gdk/gdk_align.c
+++ b/gdk/gdk_align.c
@@ -166,6 +166,7 @@ BATmaterialize(BAT *b)
{
BUN cnt;
Heap *tail;
+ Heap *h, *vh = NULL;
BUN p, q;
oid t, *x;
@@ -211,7 +212,7 @@ BATmaterialize(BAT *b)
ATOMIC_INIT(&tail->refs, 1);
/* point of no return */
MT_lock_set(&b->theaplock);
- assert(ATOMIC_GET(&b->theap->refs) > 0);
+ assert((ATOMIC_GET(&b->theap->refs) & HEAPREFS) > 0);
/* can only look at tvheap when lock is held */
if (complex_cand(b)) {
assert(b->batRole == TRANSIENT);
@@ -246,18 +247,21 @@ BATmaterialize(BAT *b)
}
assert(n == q);
}
- HEAPdecref(b->tvheap, true);
+ vh = b->tvheap;
b->tvheap = NULL;
}
- HEAPdecref(b->theap, false);
+ h = b->theap;
b->theap = tail;
b->tbaseoff = 0;
BATsetprop_nolock(b, GDK_NUNIQUE, TYPE_oid, &(oid){is_oid_nil(t) ? 1 :
b->batCount});
BATsetprop_nolock(b, GDK_UNIQUE_ESTIMATE, TYPE_dbl,
&(dbl){is_oid_nil(t) ? 1.0 : (dbl)b->batCount});
+ MT_lock_unset(&b->theaplock);
b->ttype = TYPE_oid;
BATsetdims(b);
BATsetcount(b, b->batCount);
- MT_lock_unset(&b->theaplock);
+ HEAPdecref(h, false);
+ if (vh)
+ HEAPdecref(vh, true);
return GDK_SUCCEED;
}
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -684,12 +684,12 @@ BATfree(BAT *b)
BATsetprop_nolock(b, GDK_HASH_BUCKETS, TYPE_oid,
&(oid){nbucket});
}
if (b->theap) {
- assert(ATOMIC_GET(&b->theap->refs) == 1);
+ assert((ATOMIC_GET(&b->theap->refs) & HEAPREFS) == 1);
assert(b->theap->parentid == b->batCacheid);
HEAPfree(b->theap, false);
}
if (b->tvheap) {
- assert(ATOMIC_GET(&b->tvheap->refs) == 1);
+ assert((ATOMIC_GET(&b->tvheap->refs) & HEAPREFS) == 1);
assert(b->tvheap->parentid == b->batCacheid);
HEAPfree(b->tvheap, false);
}
@@ -707,6 +707,14 @@ BATdestroy(BAT *b)
ATOMIC_DESTROY(&b->tvheap->refs);
GDKfree(b->tvheap);
}
+ ValPtr p = BATgetprop_nolock(b, (enum prop_t) 21);
+ if (p != NULL) {
+ Heap *h = p->val.pval;
+ ATOMIC_AND(&h->refs, ~DELAYEDREMOVE);
+ /* the bat has not been committed, so we cannot remove
+ * the old tail file */
+ HEAPdecref(h, false);
+ }
PROPdestroy_nolock(b);
MT_lock_destroy(&b->theaplock);
MT_lock_destroy(&b->batIdxLock);
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -41,9 +41,10 @@ unshare_varsized_heap(BAT *b)
ATOMIC_INIT(&h->refs, 1);
MT_lock_set(&b->theaplock);
int parent = b->tvheap->parentid;
- HEAPdecref(b->tvheap, false);
+ Heap *oh = b->tvheap;
b->tvheap = h;
MT_lock_unset(&b->theaplock);
+ HEAPdecref(oh, false);
BBPunshare(parent);
BBPunfix(parent);
}
@@ -446,11 +447,12 @@ append_varsized_bat(BAT *b, BAT *n, stru
}
bat parid = b->tvheap->parentid;
BBPunshare(parid);
+ ATOMIC_INIT(&h->refs, 1);
MT_lock_set(&b->theaplock);
- HEAPdecref(b->tvheap, false);
- ATOMIC_INIT(&h->refs, 1);
+ Heap *oh = b->tvheap;
b->tvheap = h;
MT_lock_unset(&b->theaplock);
+ HEAPdecref(oh, false);
BBPunfix(parid);
}
/* copy data from n to b */
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -3012,6 +3012,13 @@ BBPdestroy(BAT *b)
}
if (tp || vtp)
VIEWunlink(b);
+ ValPtr p = BATgetprop(b, (enum prop_t) 21);
+ if (p != NULL) {
+ Heap *h = p->val.pval;
+ BATrmprop(b, (enum prop_t) 21);
+ ATOMIC_AND(&h->refs, ~DELAYEDREMOVE);
+ HEAPdecref(h, true);
+ }
BATdelete(b);
BBPclear(b->batCacheid, true); /* if destroyed; de-register from BBP */
@@ -3746,7 +3753,19 @@ BBPsync(int cnt, bat *restrict subcommit
if (size > bi.count) /* includes sizes==NULL */
size = bi.count;
bi.b->batInserted = size;
- if (size == 0) {
+ if (bi.b->ttype >= 0 && ATOMvarsized(bi.b->ttype)) {
+ /* see epilogue() for other part of this */
+ /* remember the tail we're saving */
+ if (BATsetprop_nolock(bi.b, (enum prop_t) 20,
TYPE_ptr, &bi.h) == NULL) {
+ GDKerror("setprop failed\n");
+ ret = GDK_FAIL;
+ } else {
+ if (BATgetprop_nolock(bi.b, (enum
prop_t) 21) == NULL)
+ BATsetprop_nolock(bi.b, (enum
prop_t) 21, TYPE_ptr, &(void *){(Heap *) 1});
+ HEAPincref(bi.h);
+ }
+ }
+ if (size == 0 || ret != GDK_SUCCEED) {
/* no need to save anything */
MT_lock_unset(&bi.b->theaplock);
} else {
@@ -3834,7 +3853,25 @@ BBPsync(int cnt, bat *restrict subcommit
TRC_DEBUG(PERF, "%s (ready time %d)\n",
ret == GDK_SUCCEED ? "" : " failed",
(t0 = GDKms()) - t1);
+
bailout:
+ if (ret != GDK_SUCCEED) {
+ /* clean up extra refs we created */
+ for (int idx = 1; idx < cnt; idx++) {
+ bat i = subcommit ? subcommit[idx] : idx;
+ BAT *b = BBP_desc(i);
+ if (b && ATOMvarsized(b->ttype)) {
+ MT_lock_set(&b->theaplock);
+ ValPtr p = BATgetprop_nolock(b, (enum prop_t)
20);
+ if (p != NULL) {
+ HEAPdecref(p->val.pval, false);
+ BATrmprop_nolock(b, (enum prop_t) 20);
+ }
+ MT_lock_unset(&b->theaplock);
+ }
+ }
+ }
+
/* turn off the BBPSYNCING bits for all bats, even when things
* didn't go according to plan (i.e., don't check for ret ==
* GDK_SUCCEED) */
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -92,7 +92,8 @@ HEAPgrow(MT_Lock *lock, Heap **hp, size_
Heap *new;
MT_lock_set(lock);
- if (ATOMIC_GET(&(*hp)->refs) == 1) {
+ ATOMIC_BASE_TYPE refs = ATOMIC_GET(&(*hp)->refs);
+ if ((refs & HEAPREFS) == 1) {
gdk_return rc = HEAPextend((*hp), size, mayshare);
MT_lock_unset(lock);
return rc;
@@ -104,13 +105,12 @@ HEAPgrow(MT_Lock *lock, Heap **hp, size_
.farmid = old->farmid,
.hashash = old->hashash,
.dirty = true,
- .remove = old->remove,
.parentid = old->parentid,
.wasempty = old->wasempty,
};
memcpy(new->filename, old->filename, sizeof(new->filename));
if (HEAPalloc(new, size, 1, 1) == GDK_SUCCEED) {
- ATOMIC_INIT(&new->refs, 1);
+ ATOMIC_INIT(&new->refs, 1 | (refs & HEAPREMOVE));
new->free = old->free;
new->cleanhash = old->cleanhash;
if (old->free > 0 &&
@@ -375,23 +375,6 @@ HEAPshrink(Heap *h, size_t size)
return GDK_FAIL;
}
-/* returns 1 if the file exists */
-static int
-file_exists(int farmid, const char *dir, const char *name, const char *ext)
-{
- char *path;
- struct stat st;
- int ret;
-
- path = GDKfilepath(farmid, dir, name, ext);
- if (path == NULL)
- return -1;
- ret = MT_stat(path, &st);
- TRC_DEBUG(IO_, "stat(%s) = %d\n", path, ret);
- GDKfree(path);
- return (ret == 0);
-}
-
/* grow the string offset heap so that the value v fits (i.e. wide
* enough to fit the value), and it has space for at least cap elements;
* copy ncopy BUNs, or up to the heap size, whichever is smaller */
@@ -408,7 +391,6 @@ GDKupgradevarheap(BAT *b, var_t v, BUN c
#endif
size_t i, n;
size_t newsize;
- const char *filename;
bat bid = b->batCacheid;
Heap *old, *new;
@@ -441,104 +423,7 @@ GDKupgradevarheap(BAT *b, var_t v, BUN c
n = MIN(ncopy, old->size >> b->tshift);
- if (width > b->twidth)
- MT_thread_setalgorithm(n ? "widen offset heap" : "widen empty
offset heap");
- /* Create a backup copy before widening.
- *
- * If the file is memory-mapped, this solves a problem that we
- * don't control what's in the actual file until the next
- * commit happens, so a crash might otherwise leave the file
- * (and the database) in an inconsistent state. If, on the
- * other hand, the heap is allocated, it may happen that later
- * on the heap is extended and converted into a memory-mapped
- * file. Then the same problem arises.
- *
- * also see do_backup in gdk_bbp.c */
- filename = strrchr(old->filename, DIR_SEP);
- if (filename == NULL)
- filename = old->filename;
- else
- filename++;
- int exists = 0;
- if (BBP_status(bid) & (BBPEXISTING|BBPDELETED) && width > b->twidth) {
- char fname[sizeof(old->filename)];
- char *p = strrchr(old->filename, DIR_SEP);
- strcpy_len(fname, p ? p + 1 : old->filename, sizeof(fname));
- p = fname + strlen(fname) - 1;
- if (*p == 'l') {
- p++;
- p[1] = 0;
- }
- MT_lock_set(&GDKtmLock);
- for (;;) {
- exists = file_exists(old->farmid, BAKDIR, fname, NULL);
- if (exists == -1) {
- MT_lock_unset(&GDKtmLock);
- return GDK_FAIL;
- }
- if (exists == 1)
- break;
- if (*p == '1')
- break;
- if (*p == '2')
- *p = '1';
-#if SIZEOF_VAR_T == 8
- else if (*p != '4')
- *p = '4';
-#endif
- else
- *p = '2';
- }
- if (exists == 0 &&
- (old->storage != STORE_MEM ||
- GDKmove(old->farmid, BATDIR, old->filename, NULL,
- BAKDIR, filename, NULL, false) != GDK_SUCCEED)) {
- int fd;
- ssize_t ret = 0;
- size_t size = n << b->tshift;
- const char *base = old->base;
-
- /* first save heap in file with extra .tmp extension */
- if ((fd = GDKfdlocate(old->farmid, old->filename, "wb",
"tmp")) < 0) {
- MT_lock_unset(&GDKtmLock);
- return GDK_FAIL;
- }
- while (size > 0) {
- ret = write(fd, base, (unsigned) MIN(1 << 30,
size));
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
