Changeset: 7f16d65d1d17 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/7f16d65d1d17 Modified Files: testing/Mtest.py.in Branch: samloader Log Message:
merge with default branch diffs (truncated from 1240 to 300 lines): diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,16 @@ # ChangeLog file for devel # This file is updated with Maddlog +* Wed Oct 21 2026 Sjoerd Mullender <[email protected]> +- There is a new type of database farm, like the --dbextra farm, for + temporary heaps. Temporary heaps are heaps which are created and + destroyed in the same function, so they never need to be saved to disk. + By using the new --dbtemp option in mserver5, it is possible to choose + a separate directory in case the heap needs to be memory mapped, or by + specifying `in-memory` it is possible to direct the server to never + use a file-backed memory map (in this case, there should be enough + memory for these heaps). + * Tue Oct 6 2026 Sjoerd Mullender <[email protected]> - The function sys.local_timezone() has been renamed to sys.system_timezone() since it returns the time zone of the server. diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -235,6 +235,7 @@ void BBPrelinquishbats(void); int BBPrename(BAT *b, const char *nme); int BBPretain(bat b); gdk_return BBPsave(BAT *b); +int BBPselectfarm(role_t role, int type, enum heaptype hptype); void BBPtmlock(void); void BBPtmunlock(void); int BBPunfix(bat b); diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -173,16 +173,66 @@ typedef enum gdk_return { GDK_FAIL, GDK_SUCCEED } gdk_return; /* basic GDK types */ -typedef bool msk; -typedef int8_t bit; -typedef int8_t bte; -typedef int16_t sht; -typedef int64_t lng; -typedef uint64_t ulng; +typedef bool msk; /* one bit Boolean */ +typedef int8_t bit; /* three-value Boolean (8 bits) */ +typedef int8_t bte; /* 8 bit signed integer */ +typedef int16_t sht; /* 16 bit signed integer */ +/* standard type int is used for 32 bit signed integer */ +typedef int64_t lng; /* 64 bit signed integer */ +typedef uint64_t ulng; /* 64 bit unsigned integer */ #ifdef HAVE_HGE -typedef int128_t hge; -typedef uint128_t uhge; +typedef int128_t hge; /* 128 bit signed integer */ +typedef uint128_t uhge; /* 128 bit unsigned integer */ #endif +typedef int bat; /* Index into BBP */ +typedef void *ptr; /* generic pointer */ +typedef float flt; /* 32 bit floating point */ +typedef double dbl; /* 64 bit floating point */ +typedef char *str; /* null terminated UTF-8 string */ +typedef size_t oid; /* object identifier */ +typedef oid var_t; /* type used for heap index of var-sized BAT */ +typedef oid BUN; /* BUN position */ + +/* The types `oid`, `BUN` and `var_t` are all defined as `size_t`, but + * they serve different purposes. `oid` is the *logical* row number of + * a BAT; `BUN` is the *physical* row number of a BAT and is therefore + * also used for the total number of rows and the potential capacity of + * a BAT; `var_t` is an *offset* into the var-size heap (vheap). The + * difference between the *logical* and *physical* row number is the + * value of the head sequence base (`hseqbase`) of the BAT. */ + +typedef union { + uint32_t align; /* force alignment, only used for equality */ + uint8_t quad[4] __attribute__((__nonstring__)); +} inet4; /* IPv4 internet address */ + +typedef union { +#ifdef HAVE_HGE + uint128_t align; /* force alignment, only used for equality */ +#else + uint64_t align[2]; /* force alignment, not otherwise used */ +#endif + uint8_t hex[16] __attribute__((__nonstring__)); +} inet6; /* IPv6 internet address */ + +#define UUID_SIZE 16 /* size of a UUID */ +#define UUID_STRLEN 36 /* length of string representation */ + +typedef union { +#ifdef HAVE_HGE + uint128_t h; /* force alignment, only used for equality */ +#else + uint64_t l[2]; /* force alignment, not otherwise used */ +#endif + uint8_t u[UUID_SIZE] __attribute__((__nonstring__)); +} uuid; + +typedef struct { + size_t nitems; + uint8_t data[] __attribute__((__nonstring__)) + __attribute__((__counted_by__(nitems))); +} blob; +gdk_export size_t blobsize(size_t nitems) __attribute__((__const__)); #define SIZEOF_LNG 8 #define LL_CONSTANT(val) INT64_C(val) @@ -202,61 +252,12 @@ typedef uint128_t uhge; #endif #define SIZEOF_OID SIZEOF_SIZE_T -typedef size_t oid; /* macro in the style of <inttypes.h> for formatting oid */ #define PRIuOID "zu" #define OIDFMT "%" PRIuOID -typedef int bat; /* Index into BBP */ -typedef void *ptr; /* Internal coding of types */ - #define SIZEOF_PTR SIZEOF_VOID_P -typedef float flt; -typedef double dbl; -typedef char *str; - -typedef union { - uint32_t align; /* force alignment, only used for equality */ - uint8_t quad[4] __attribute__((__nonstring__)); -} inet4; - -typedef union { -#ifdef HAVE_HGE - uint128_t align; /* force alignment, only used for equality */ -#else - uint64_t align[2]; /* force alignment, not otherwise used */ -#endif - uint8_t hex[16] __attribute__((__nonstring__)); -} inet6; - -#define UUID_SIZE 16 /* size of a UUID */ -#define UUID_STRLEN 36 /* length of string representation */ - -typedef union { -#ifdef HAVE_HGE - uint128_t h; /* force alignment, only used for equality */ -#else - uint64_t l[2]; /* force alignment, not otherwise used */ -#endif - uint8_t u[UUID_SIZE] __attribute__((__nonstring__)); -} uuid; - -typedef struct { - size_t nitems; - uint8_t data[] __attribute__((__nonstring__)) - __attribute__((__counted_by__(nitems))); -} blob; -gdk_export size_t blobsize(size_t nitems) __attribute__((__const__)); - -/* The types `oid`, `BUN` and `var_t` are all defined as `size_t`, but - * they serve different purposes. `oid` is the *logical* row number of - * a BAT; `BUN` is the *physical* row number of a BAT and is therefore - * also used for the total number of rows and the potential capacity of - * a BAT; `var_t` is an *offset* into the var-size heap (vheap). The - * difference between the *logical* and *physical* row number is the - * value of the head sequence base (`hseqbase`) of the BAT. */ - -typedef oid var_t; /* type used for heap index of var-sized BAT */ + #define SIZEOF_VAR_T SIZEOF_OID #define VARFMT OIDFMT @@ -266,7 +267,6 @@ typedef oid var_t; /* type used for hea #define VAR_MAX ((var_t) INT64_MAX) #endif -typedef oid BUN; /* BUN position */ #define SIZEOF_BUN SIZEOF_OID #define PRIuBUN PRIuOID #define BUNFMT OIDFMT @@ -1443,6 +1443,7 @@ typedef enum { PERSISTENT = 0, TRANSIENT, SYSTRANS, + TEMPFARM, } role_t; /* Heap storage modes */ @@ -1464,7 +1465,8 @@ typedef struct { #endif ATOMIC_TYPE refs; /* reference count for this heap */ - bte farmid; /* id of farm where heap is located */ + int8_t farmid; /* id of farm where heap is located */ + role_t role; /* role of this heap */ bool cleanhash; /* string heaps must clean hash */ bool dirty; /* specific heap dirty marker */ bool remove; /* remove storage file when freeing */ @@ -3892,6 +3894,15 @@ gdk_export int BBPreadBBPline(FILE *fp, /* global calls */ gdk_export gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror); +enum heaptype { + offheap, + varheap, + hashheap, + orderidxheap, + strimpheap, + dataheap +}; +gdk_export int BBPselectfarm(role_t role, int type, enum heaptype hptype); /* update interface */ gdk_export gdk_return BBPsave(BAT *b); diff --git a/gdk/gdk_align.c b/gdk/gdk_align.c --- a/gdk/gdk_align.c +++ b/gdk/gdk_align.c @@ -247,6 +247,7 @@ BATmaterialize(BAT *b, BUN cap) *tail = (Heap) { .farmid = BBPselectfarm(b->batRole, TYPE_oid, offheap), + .role = b->batRole, .parentid = b->batCacheid, .dirty = true, .refs = ATOMIC_VAR_INIT(1), diff --git a/gdk/gdk_analytic_func.c b/gdk/gdk_analytic_func.c --- a/gdk/gdk_analytic_func.c +++ b/gdk/gdk_analytic_func.c @@ -22,6 +22,7 @@ GDKinitialize_segment_tree(void) return NULL; *h = (Heap) { .farmid = BBPselectfarm(TRANSIENT, TYPE_bte, dataheap), + .role = TRANSIENT, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c --- a/gdk/gdk_bat.c +++ b/gdk/gdk_bat.c @@ -131,6 +131,7 @@ BATcreatedesc(oid hseq, int tt, bool hea } *h = (Heap) { .farmid = BBPselectfarm(role, tt, offheap), + .role = role, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; @@ -142,6 +143,7 @@ BATcreatedesc(oid hseq, int tt, bool hea } *vh = (Heap) { .farmid = BBPselectfarm(role, tt, varheap), + .role = role, .dirty = true, .refs = ATOMIC_VAR_INIT(1), }; @@ -506,6 +508,7 @@ BATclear(BAT *b, bool force) } *th = (Heap) { .farmid = b->tvheap->farmid, + .role = b->tvheap->role, .parentid = b->tvheap->parentid, .dirty = true, .hasfile = b->tvheap->hasfile, @@ -2465,9 +2468,13 @@ BATassertProps(BAT *b) assert(isview1 || b->ttype == TYPE_void || BBPfarms[b->theap->farmid].roles & (1 << b->batRole)); + assert(b->ttype == TYPE_void || + BBPfarms[b->theap->farmid].roles & (1 << b->theap->role)); assert(isview2 || b->tvheap == NULL || (BBPfarms[b->tvheap->farmid].roles & (1 << b->batRole))); + assert(b->tvheap == NULL || + (BBPfarms[b->tvheap->farmid].roles & (1 << b->tvheap->role))); cmpf = ATOMcompare(b->ttype); eqf = ATOMequal(b->ttype); @@ -2714,37 +2721,34 @@ BATassertProps(BAT *b) /* we need to check for uniqueness the hard * way (i.e. using a hash table) */ const char *nme = BBP_physical(b->batCacheid); - Hash *hs = NULL; + Hash hsh; BUN mask; - if ((hs = GDKzalloc(sizeof(Hash))) == NULL) { - TRC_WARNING(BAT, "Cannot allocate hash table\n"); - goto abort_check; - } - if (snprintf(hs->heaplink.filename, sizeof(hs->heaplink.filename), "%s.thshprpl%x", nme, (unsigned) MT_getpid()) >= (int) sizeof(hs->heaplink.filename) || - snprintf(hs->heapbckt.filename, sizeof(hs->heapbckt.filename), "%s.thshprpb%x", nme, (unsigned) MT_getpid()) >= (int) sizeof(hs->heapbckt.filename)) { - /* cannot happen, see comment in gdk.h - * about sizes near definition of - * BBPINIT */ - GDKfree(hs); - TRC_CRITICAL(BAT, "Heap filename is too large\n"); - goto abort_check; - } + hsh = (Hash) { + .heapbckt.parentid = b->batCacheid, _______________________________________________ checkin-list mailing list -- [email protected] To unsubscribe send an email to [email protected]
