Changeset: fbbfc61a01a8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/fbbfc61a01a8
Modified Files:
gdk/gdk_bat.c
gdk/gdk_bbp.c
monetdb5/modules/mal/clients.c
sql/server/rel_optimizer.c
Branch: default
Log Message:
Merged with Jul2021
diffs (truncated from 488 to 300 lines):
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -94,7 +94,7 @@ BATcreatedesc(oid hseq, int tt, bool hea
.batRole = role,
.batTransient = true,
};
- if (heapnames && (bn->theap = GDKzalloc(sizeof(Heap))) == NULL) {
+ if (heapnames && (bn->theap = GDKmalloc(sizeof(Heap))) == NULL) {
GDKfree(bn);
return NULL;
}
@@ -115,16 +115,23 @@ BATcreatedesc(oid hseq, int tt, bool hea
if (heapnames) {
assert(bn->theap != NULL);
- bn->theap->parentid = bn->batCacheid;
- bn->theap->farmid = BBPselectfarm(role, bn->ttype, offheap);
+ *bn->theap = (Heap) {
+ .parentid = bn->batCacheid,
+ .farmid = BBPselectfarm(role, bn->ttype, offheap),
+ };
const char *nme = BBP_physical(bn->batCacheid);
strconcat_len(bn->theap->filename, sizeof(bn->theap->filename),
nme, ".tail", NULL);
if (ATOMneedheap(tt)) {
- if ((bn->tvheap = GDKmalloc(sizeof(Heap))) == NULL)
- goto bailout;
+ if ((bn->tvheap = GDKmalloc(sizeof(Heap))) == NULL) {
+ BBPclear(bn->batCacheid, true);
+ HEAPfree(bn->theap, true);
+ GDKfree(bn->theap);
+ GDKfree(bn);
+ return NULL;
+ }
*bn->tvheap = (Heap) {
.parentid = bn->batCacheid,
.farmid = BBPselectfarm(role, bn->ttype,
varheap),
@@ -147,14 +154,6 @@ BATcreatedesc(oid hseq, int tt, bool hea
MT_rwlock_init(&bn->thashlock, name);
bn->batDirtydesc = true;
return bn;
- bailout:
- BBPclear(bn->batCacheid, true);
- if (bn->theap)
- HEAPdecref(bn->theap, true);
- if (bn->tvheap)
- HEAPdecref(bn->tvheap, true);
- GDKfree(bn);
- return NULL;
}
uint8_t
@@ -1780,7 +1779,7 @@ BATsetcount(BAT *b, BUN cnt)
MT_lock_set(&b->theaplock);
b->batCount = cnt;
b->batDirtydesc = true;
- b->theap->dirty |= b->ttype != TYPE_void && b->theap->parentid ==
b->batCacheid;
+ b->theap->dirty |= b->ttype != TYPE_void && b->theap->parentid ==
b->batCacheid && cnt > 0;
if (b->theap->parentid == b->batCacheid)
b->theap->free = tailsize(b, cnt);
if (b->ttype == TYPE_void)
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1285,6 +1285,39 @@ movestrbats(void)
#endif
static void
+BBPtrim(bool aggressive)
+{
+ int n = 0;
+ unsigned flag = BBPUNLOADING | BBPSYNCING | BBPSAVING;
+ if (!aggressive)
+ flag |= BBPHOT;
+ for (bat bid = 1; bid < (bat) ATOMIC_GET(&BBPsize); bid++) {
+ MT_lock_set(&GDKswapLock(bid));
+ BAT *b = NULL;
+ bool swap = false;
+ if (BBP_refs(bid) == 0 &&
+ BBP_lrefs(bid) != 0 &&
+ (b = BBP_cache(bid)) != NULL &&
+ b->batSharecnt == 0 &&
+ (!BATdirty(b) || (aggressive && b->theap->storage ==
STORE_MMAP && (b->tvheap == NULL || b->tvheap->storage == STORE_MMAP))) &&
+ !(BBP_status(bid) & flag) /*&&
+ (BBP_status(bid) & BBPPERSISTENT ||
+ (b->batRole == PERSISTENT && BBP_lrefs(bid) == 1)) */) {
+ BBP_status_on(bid, BBPUNLOADING);
+ swap = true;
+ }
+ MT_lock_unset(&GDKswapLock(bid));
+ if (swap) {
+ TRC_DEBUG(BAT_, "unload and free bat %d\n", bid);
+ if (BBPfree(b) != GDK_SUCCEED)
+ GDKerror("unload failed for bat %d", bid);
+ n++;
+ }
+ }
+ TRC_DEBUG(BAT_, "unloaded %d bats%s\n", n, aggressive ? " (also hot)" :
"");
+}
+
+static void
BBPmanager(void *dummy)
{
(void) dummy;
@@ -1294,41 +1327,19 @@ BBPmanager(void *dummy)
for (bat bid = 1; bid < (bat) ATOMIC_GET(&BBPsize); bid++) {
MT_lock_set(&GDKswapLock(bid));
if (BBP_refs(bid) == 0 && BBP_lrefs(bid) != 0) {
+ n += (BBP_status(bid) & BBPHOT) != 0;
BBP_status_off(bid, BBPHOT);
- n++;
}
MT_lock_unset(&GDKswapLock(bid));
}
TRC_DEBUG(BAT_, "cleared HOT bit from %d bats\n", n);
- for (int i = 0; i < 100; i++) {
+ size_t cur = GDKvm_cursize();
+ for (int i = 0, n = cur > GDK_vm_maxsize / 2 ? 1 : cur >
GDK_vm_maxsize / 4 ? 10 : 100; i < n; i++) {
MT_sleep_ms(100);
if (GDKexiting())
return;
}
- n = 0;
- for (bat bid = 1; bid < (bat) ATOMIC_GET(&BBPsize); bid++) {
- MT_lock_set(&GDKswapLock(bid));
- BAT *b = NULL;
- bool swap = false;
- if (BBP_refs(bid) == 0 &&
- BBP_lrefs(bid) != 0 &&
- (b = BBP_cache(bid)) != NULL &&
- b->batSharecnt == 0 &&
- !BATdirty(b) &&
- !(BBP_status(bid) & (BBPHOT | BBPUNLOADING |
BBPSYNCING)) &&
- (BBP_status(bid) & BBPPERSISTENT)) {
- BBP_status_on(bid, BBPUNLOADING);
- swap = true;
- }
- MT_lock_unset(&GDKswapLock(bid));
- if (swap) {
- TRC_DEBUG(BAT_, "unload and free bat %d\n",
bid);
- if (BBPfree(b) != GDK_SUCCEED)
- GDKerror("unload failed for bat %d",
bid);
- n++;
- }
- }
- TRC_DEBUG(BAT_, "unloaded %d bats\n", n);
+ BBPtrim(false);
if (GDKexiting())
return;
}
@@ -1943,10 +1954,11 @@ BBPdump(void)
fprintf(stderr, " Theap -> %d",
b->theap->parentid);
} else {
fprintf(stderr,
- " Theap=[%zu,%zu,f=%d]%s",
- HEAPmemsize(b->theap),
- HEAPvmsize(b->theap),
+ " Theap=[%zu,%zu,f=%d]%s%s",
+ b->theap->free,
+ b->theap->size,
b->theap->farmid,
+ b->theap->base == NULL ? "X" :
b->theap->storage == STORE_MMAP ? "M" : "",
status & BBPSWAPPED ? "(Swapped)" :
b->theap->dirty ? "(Dirty)" : "");
if (BBP_logical(i) && BBP_logical(i)[0] == '.')
{
cmem += HEAPmemsize(b->theap);
@@ -1966,10 +1978,11 @@ BBPdump(void)
b->tvheap->parentid);
} else {
fprintf(stderr,
- " Tvheap=[%zu,%zu,f=%d]%s",
- HEAPmemsize(b->tvheap),
- HEAPvmsize(b->tvheap),
+ " Tvheap=[%zu,%zu,f=%d]%s%s",
+ b->tvheap->free,
+ b->tvheap->size,
b->tvheap->farmid,
+ b->tvheap->base == NULL ? "X" :
b->tvheap->storage == STORE_MMAP ? "M" : "",
b->tvheap->dirty ? "(Dirty)" : "");
if (BBP_logical(i) && BBP_logical(i)[0] == '.')
{
cmem += HEAPmemsize(b->tvheap);
@@ -2724,11 +2737,15 @@ decref(bat i, bool logical, bool release
/* we destroy transients asap and unload persistent bats only
* if they have been made cold or are not dirty */
+ unsigned chkflag = BBPSYNCING;
+ if (GDKvm_cursize() < GDK_vm_maxsize &&
+ ((b && b->theap ? b->theap->size : 0) + (b && b->tvheap ?
b->tvheap->size : 0)) < (GDK_vm_maxsize - GDKvm_cursize()) / 32)
+ chkflag |= BBPHOT;
if (BBP_refs(i) > 0 ||
(BBP_lrefs(i) > 0 &&
(b == NULL ||
BATdirty(b) ||
- (BBP_status(i) & (BBPHOT | BBPSYNCING)) ||
+ (BBP_status(i) & chkflag) ||
!(BBP_status(i) & BBPPERSISTENT) ||
GDKinmemory(farmid)))) {
/* bat cannot be swapped out */
@@ -3580,24 +3597,41 @@ BBPsync(int cnt, bat *restrict subcommit
while (++idx < cnt) {
bat i = subcommit ? subcommit[idx] : idx;
/* BBP_desc(i) may be NULL */
- BATiter bi = bat_iterator(BBP_desc(i));
BUN size = sizes ? sizes[idx] : BUN_NONE;
- if (size > bi.count)
- size = bi.count;
if (BBP_status(i) & BBPPERSISTENT) {
BAT *b = dirty_bat(&i, subcommit != NULL);
if (i <= 0) {
- bat_iterator_end(&bi);
break;
}
- if (b)
+ if (b) {
+ /* wait for BBPSAVING so that we
+ * can set it, wait for
+ * BBPUNLOADING before
+ * attempting to save */
+ for (;;) {
+ if (lock)
+
MT_lock_set(&GDKswapLock(i));
+ if (!(BBP_status(i) &
(BBPSAVING|BBPUNLOADING)))
+ break;
+ if (lock)
+
MT_lock_unset(&GDKswapLock(i));
+ BBPspin(i, __func__,
BBPSAVING|BBPUNLOADING);
+ }
+ BBP_status_on(i, BBPSAVING);
+ if (lock)
+ MT_lock_unset(&GDKswapLock(i));
+ BATiter bi = bat_iterator(b);
+ if (size > bi.count)
+ size = bi.count;
ret = BATsave_locked(b, &bi, size);
+ bat_iterator_end(&bi);
+ BBP_status_off(i, BBPSAVING);
+ }
}
if (ret == GDK_SUCCEED) {
n = BBPdir_step(i, size, n, buf, sizeof(buf),
&obbpf, nbbpf);
}
- bat_iterator_end(&bi);
if (n == -2)
break;
/* we once again have a saved heap */
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -877,6 +877,8 @@ GDKembedded(void)
return Mbedded;
}
+static MT_Id mainpid;
+
gdk_return
GDKinit(opt *set, int setlen, bool embedded)
{
@@ -888,6 +890,8 @@ GDKinit(opt *set, int setlen, bool embed
int i, nlen = 0;
char buf[16];
+ mainpid = MT_getpid();
+
if (GDKinmemory(0)) {
dbpath = dbtrace = NULL;
} else {
@@ -1188,12 +1192,13 @@ GDKexiting(void)
void
GDKprepareExit(void)
{
- if (ATOMIC_ADD(&GDKstopped, 1) > 0)
- return;
+ ATOMIC_ADD(&GDKstopped, 1);
- TRC_DEBUG_IF(THRD)
- dump_threads();
- join_detached_threads();
+ if (MT_getpid() == mainpid) {
+ TRC_DEBUG_IF(THRD)
+ dump_threads();
+ join_detached_threads();
+ }
}
void
diff --git a/monetdb5/modules/mal/clients.c b/monetdb5/modules/mal/clients.c
--- a/monetdb5/modules/mal/clients.c
+++ b/monetdb5/modules/mal/clients.c
@@ -844,7 +844,7 @@ CLTshutdown(Client cntxt, MalBlkPtr mb,
snprintf(buf, 1024,"%d client sessions still running",leftover);
*ret = GDKstrdup(buf);
if ( force)
- mal_exit(0);
+ GDKprepareExit();
if (*ret == NULL)
throw(MAL, "mal.shutdown", SQLSTATE(HY013) MAL_MALLOC_FAIL);
return MAL_SUCCEED;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
