Changeset: 286231e79265 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/286231e79265 Branch: manti Log Message:
merged from default diffs (truncated from 964 to 300 lines): diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,16 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Wed Oct 21 2026 Sjoerd Mullender <[email protected]> +- There is a new type of database farm, like the --dbextra farm, for + temporary heaps. Temporary heaps are heaps which are created and + destroyed in the same function, so they never need to be saved to disk. + By using the new --dbtemp option in mserver5, it is possible to choose + a separate directory in case the heap needs to be memory mapped, or by + specifying `in-memory` it is possible to direct the server to never + use a file-backed memory map (in this case, there should be enough + memory for these heaps). + * Tue Oct 6 2026 Sjoerd Mullender <[email protected]> - The function sys.local_timezone() has been renamed to sys.system_timezone() since it returns the time zone of the server. diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -235,6 +235,7 @@ void BBPrelinquishbats(void); int BBPrename(BAT *b, const char *nme); int BBPretain(bat b); gdk_return BBPsave(BAT *b); +int BBPselectfarm(role_t role, int type, enum heaptype hptype); void BBPtmlock(void); void BBPtmunlock(void); int BBPunfix(bat b); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -1443,6 +1443,7 @@ typedef enum { PERSISTENT = 0, TRANSIENT, SYSTRANS, + TEMPFARM, } role_t; /* Heap storage modes */ @@ -1464,7 +1465,8 @@ typedef struct { #endif ATOMIC_TYPE refs; /* reference count for this heap */ - bte farmid; /* id of farm where heap is located */ + int8_t farmid; /* id of farm where heap is located */ + role_t role; /* role of this heap */ bool cleanhash; /* string heaps must clean hash */ bool dirty; /* specific heap dirty marker */ bool remove; /* remove storage file when freeing */ @@ -3892,6 +3894,15 @@ gdk_export int BBPreadBBPline(FILE *fp, /* global calls */ gdk_export gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror); +enum heaptype { + offheap, + varheap, + hashheap, + orderidxheap, + strimpheap, + dataheap +}; +gdk_export int BBPselectfarm(role_t role, int type, enum heaptype hptype); /* update interface */ gdk_export gdk_return BBPsave(BAT *b); diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c --- a/gdk/gdk_align.c +++ b/gdk/gdk_align.c @@ -247,6 +247,7 @@ BATmaterialize(BAT *b, BUN cap) *tail = (Heap) { .farmid = BBPselectfarm(b->batRole, TYPE_oid, offheap), + .role = b->batRole, .parentid = b->batCacheid, .dirty = true, .refs = ATOMIC_VAR_INIT(1), diff --git a/gdk/gdk_analytic_func.c b/gdk/gdk_analytic_func.c --- a/gdk/gdk_analytic_func.c +++ b/gdk/gdk_analytic_func.c @@ -22,6 +22,7 @@ GDKinitialize_segment_tree(void) return NULL; *h = (Heap) { .farmid = BBPselectfarm(TRANSIENT, TYPE_bte, dataheap), + .role = TRANSIENT, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -131,6 +131,7 @@ BATcreatedesc(oid hseq, int tt, bool hea } *h = (Heap) { .farmid = BBPselectfarm(role, tt, offheap), + .role = role, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; @@ -142,6 +143,7 @@ BATcreatedesc(oid hseq, int tt, bool hea } *vh = (Heap) { .farmid = BBPselectfarm(role, tt, varheap), + .role = role, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; @@ -506,6 +508,7 @@ BATclear(BAT *b, bool force) } *th = (Heap) { .farmid = b->tvheap->farmid, + .role = b->tvheap->role, .parentid = b->tvheap->parentid, .dirty = true, .hasfile = b->tvheap->hasfile, @@ -2465,9 +2468,13 @@ BATassertProps(BAT *b) assert(isview1 || b->ttype == TYPE_void || BBPfarms[b->theap->farmid].roles & (1 << b->batRole)); + assert(b->ttype == TYPE_void || + BBPfarms[b->theap->farmid].roles & (1 << b->theap->role)); assert(isview2 || b->tvheap == NULL || (BBPfarms[b->tvheap->farmid].roles & (1 << b->batRole))); + assert(b->tvheap == NULL || + (BBPfarms[b->tvheap->farmid].roles & (1 << b->tvheap->role))); cmpf = ATOMcompare(b->ttype); eqf = ATOMequal(b->ttype); @@ -2714,37 +2721,34 @@ BATassertProps(BAT *b) /* we need to check for uniqueness the hard * way (i.e. using a hash table) */ const char *nme = BBP_physical(b->batCacheid); - Hash *hs = NULL; + Hash hsh; BUN mask; - if ((hs = GDKzalloc(sizeof(Hash))) == NULL) { - TRC_WARNING(BAT, "Cannot allocate hash table\n"); - goto abort_check; - } - if (snprintf(hs->heaplink.filename, sizeof(hs->heaplink.filename), "%s.thshprpl%x", nme, (unsigned) MT_getpid()) >= (int) sizeof(hs->heaplink.filename) || - snprintf(hs->heapbckt.filename, sizeof(hs->heapbckt.filename), "%s.thshprpb%x", nme, (unsigned) MT_getpid()) >= (int) sizeof(hs->heapbckt.filename)) { - /* cannot happen, see comment in gdk.h - * about sizes near definition of - * BBPINIT */ - GDKfree(hs); - TRC_CRITICAL(BAT, "Heap filename is too large\n"); - goto abort_check; - } + hsh = (Hash) { + .heapbckt.parentid = b->batCacheid, + .heaplink.parentid = b->batCacheid, + .heaplink.farmid = BBPselectfarm( + TEMPFARM, b->ttype, hashheap), + .heaplink.role = TEMPFARM, + .heapbckt.farmid = BBPselectfarm( + TEMPFARM, b->ttype, hashheap), + .heapbckt.role = TEMPFARM, + }; if (ATOMsize(b->ttype) == 1) mask = (BUN) 1 << 8; else if (ATOMsize(b->ttype) == 2) mask = (BUN) 1 << 16; else mask = HASHmask(b->batCount); - hs->heapbckt.parentid = b->batCacheid; - hs->heaplink.parentid = b->batCacheid; - if ((hs->heaplink.farmid = BBPselectfarm( - TRANSIENT, b->ttype, hashheap)) < 0 || - (hs->heapbckt.farmid = BBPselectfarm( - TRANSIENT, b->ttype, hashheap)) < 0 || - HASHnew(hs, b->ttype, BATcount(b), + if (hsh.heaplink.farmid < 0 || + hsh.heapbckt.farmid < 0 || + snprintf(hsh.heaplink.filename, sizeof(hsh.heaplink.filename), "%s.thshprpl%x", nme, (unsigned) MT_getpid()) >= (int) sizeof(hsh.heaplink.filename) || + snprintf(hsh.heapbckt.filename, sizeof(hsh.heapbckt.filename), "%s.thshprpb%x", nme, (unsigned) MT_getpid()) >= (int) sizeof(hsh.heapbckt.filename) || + HASHnew(&hsh, b->ttype, BATcount(b), mask, BUN_NONE, false) != GDK_SUCCEED) { - GDKfree(hs); + /* snprintf failing cannot happen, see + * comment in gdk.h about sizes near + * definition of BBPINIT */ TRC_WARNING(BAT, "Cannot allocate hash table\n"); goto abort_check; } @@ -2776,20 +2780,19 @@ BATassertProps(BAT *b) assert(cmp <= 0); seenmin |= cmp == 0; } - prb = HASHprobe(hs, valp); - for (hb = HASHget(hs, prb); + prb = HASHprobe(&hsh, valp); + for (hb = HASHget(&hsh, prb); hb != BUN_NONE; - hb = HASHgetlink(hs, hb)) + hb = HASHgetlink(&hsh, hb)) if (eqf(valp, BUNtail(&bi, hb))) assert(!b->tkey); - HASHputlink(hs, p, HASHget(hs, prb)); - HASHput(hs, prb, p); + HASHputlink(&hsh, p, HASHget(&hsh, prb)); + HASHput(&hsh, prb, p); assert(!b->tnonil || !isnil); seennil |= isnil; } - HEAPfree(&hs->heaplink, true); - HEAPfree(&hs->heapbckt, true); - GDKfree(hs); + HEAPfree(&hsh.heaplink, true); + HEAPfree(&hsh.heapbckt, true); } abort_check: GDKclrerr(); diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -32,6 +32,7 @@ unshare_varsized_heap(BAT *b) *h = (Heap) { .parentid = b->batCacheid, .farmid = BBPselectfarm(b->batRole, b->ttype, varheap), + .role = b->batRole, .refs = ATOMIC_VAR_INIT(1), }; strtconcat(h->filename, sizeof(h->filename), @@ -550,6 +551,7 @@ append_varsized_bat(BAT *b, BATiter *ni, *h = (Heap) { .parentid = b->batCacheid, .farmid = BBPselectfarm(b->batRole, b->ttype, varheap), + .role = b->batRole, .refs = ATOMIC_VAR_INIT(1), }; strtconcat(h->filename, sizeof(h->filename), diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c --- a/gdk/gdk_bbp.c +++ b/gdk/gdk_bbp.c @@ -448,6 +448,7 @@ vheapinit(BAT *b, const char *buf, unsig .dirty = false, .parentid = b->batCacheid, .farmid = BBPselectfarm(PERSISTENT, b->ttype, varheap), + .role = PERSISTENT, .hasfile = free > 0, }; strtconcat(b->tvheap->filename, sizeof(b->tvheap->filename), @@ -587,6 +588,7 @@ heapinit(BAT *b, const char *buf, settailname(b->theap, filename, t, width); b->theap->storage = STORE_INVALID; b->theap->farmid = BBPselectfarm(PERSISTENT, b->ttype, offheap); + b->theap->role = PERSISTENT, b->theap->dirty = false; b->theap->parentid = b->batCacheid; if (minpos < b->batCount) @@ -1102,6 +1104,9 @@ BBPaddfarm(const char *dirname, uint32_t GDKerror("bad rolemask\n"); return GDK_FAIL; } + /* transient and systrans must be same farm */ + if (rolemask & (1 << TRANSIENT)) + rolemask |= 1 << SYSTRANS; if (strcmp(dirname, "in-memory") == 0 || /* backward compatibility: */ strcmp(dirname, ":memory:") == 0) { dirname = NULL; @@ -1158,6 +1163,10 @@ BBPaddfarm(const char *dirname, uint32_t } return GDK_SUCCEED; } + if (BBPfarms[i].roles & rolemask) { + GDKerror("role added multiple times"); + return GDK_FAIL; + } } if (logerror) GDKerror("too many farms\n"); @@ -1168,6 +1177,7 @@ gdk_return BBPchkfarms(void) { const char *dir = NULL; + const char *tdir = NULL; uint32_t rolemask = 0; if ((BBPfarms[0].roles & 1) == 0) { GDKerror("Must call BBPaddfarms at least once for persistent data\n"); @@ -1177,17 +1187,30 @@ BBPchkfarms(void) if (BBPfarms[i].roles != 0) { dir = BBPfarms[i].dirname; rolemask |= BBPfarms[i].roles; + if (BBPfarms[i].roles & (1U << TRANSIENT)) + tdir = BBPfarms[i].dirname; } } if (dir == NULL) dir = "in-memory"; + if (tdir == NULL) + tdir = "in-memory"; if ((rolemask & (1U << TRANSIENT)) == 0) { gdk_return rc = BBPaddfarm(dir, 1U << TRANSIENT, true); if (rc != GDK_SUCCEED) return rc; + tdir = dir; _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
