Changeset: 8fc883b7fdef for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8fc883b7fdef
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_bbp.mx
monetdb5/modules/kernel/status.mx
Branch: default
Log Message:
Never reallocate BBP. Instead we have a new scheme to extend the BBP.
BBP is now an array of pointers, each of which may point to an array
of BBPrec entries of a predetermined size. In this way we can
allocate lots of space for BATs without allocating rediculous amounts
when we don't need much. The drawback is, there is a maximum:
currently 819200 BATs on a 64-bit system and 204800 on a 32-bit
system. This can be extended relatively easily by increasing the
value of N_BBPINIT or BBPINITLOG in gdk.h.
diffs (183 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -178,7 +178,7 @@ int BATtopN(BAT *b, BUN topN);
BAT *BATundo(BAT *b);
BAT *BATuselect(BAT *b, const void *tl, const void *th);
BAT *BATuselect_(BAT *b, const void *tl, const void *th, bit li, bit hi);
-BBPrec *BBP;
+BBPrec *BBP[N_BBPINIT];
int BBP_curstamp;
void BBPclear(bat bid);
void BBPcold(bat b);
@@ -195,7 +195,6 @@ void BBPkeepref(bat i);
bat BBPlimit;
void BBPlock(const char *s);
str BBPlogical(bat b, str buf);
-bat BBPmaxsize;
int BBPout;
str BBPphysical(bat b, str buf);
BAT *BBPquickdesc(bat b, int delaccess);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1840,23 +1840,30 @@ typedef struct {
/* MT_Id pid; non-zero thread-id if this BAT is private */
} BBPrec;
-gdk_export bat BBPmaxsize;
gdk_export bat BBPlimit;
-gdk_export BBPrec *BBP;
+#define N_BBPINIT 100
+#if SIZEOF_VOID_P == 4
+#define BBPINITLOG 11
+#else
+#define BBPINITLOG 13
+#endif
+#define BBPINIT (1 << BBPINITLOG)
+/* absolute maximum number of BATs is N_BBPINIT * BBPINIT */
+gdk_export BBPrec *BBP[N_BBPINIT];
/* fast defines without checks; internal use only */
-#define BBP_cache(i) BBP[ABS(i)].cache[(i)<0]
-#define BBP_logical(i) BBP[ABS(i)].logical[(i)<0]
-#define BBP_bak(i) BBP[ABS(i)].bak[(i)<0]
-#define BBP_next(i) BBP[ABS(i)].next[(i)<0]
-#define BBP_physical(i) BBP[ABS(i)].physical
-#define BBP_options(i) BBP[ABS(i)].options
-#define BBP_desc(i) BBP[ABS(i)].desc
-#define BBP_refs(i) BBP[ABS(i)].refs
-#define BBP_lrefs(i) BBP[ABS(i)].lrefs
-#define BBP_lastused(i) BBP[ABS(i)].lastused
-#define BBP_status(i) BBP[ABS(i)].status
-#define BBP_pid(i) BBP[ABS(i)].pid
+#define BBP_cache(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].cache[(i)<0]
+#define BBP_logical(i)
BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].logical[(i)<0]
+#define BBP_bak(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].bak[(i)<0]
+#define BBP_next(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].next[(i)<0]
+#define BBP_physical(i)
BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].physical
+#define BBP_options(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].options
+#define BBP_desc(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].desc
+#define BBP_refs(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].refs
+#define BBP_lrefs(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].lrefs
+#define BBP_lastused(i)
BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].lastused
+#define BBP_status(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].status
+#define BBP_pid(i) BBP[ABS(i)>>BBPINITLOG][ABS(i)&(BBPINIT-1)].pid
/* macros that nicely check parameters */
#define BBPcacheid(b) ((b)->batCacheid)
diff --git a/gdk/gdk_bbp.mx b/gdk/gdk_bbp.mx
--- a/gdk/gdk_bbp.mx
+++ b/gdk/gdk_bbp.mx
@@ -88,8 +88,6 @@ All Rights Reserved.
#ifndef _GDK_BBP_H_
#define _GDK_BBP_H_
-#define BBPINIT 2048
-
#define BBPLOADED 1 /* set if bat in memory */
#define BBPSWAPPED 2 /* set if dirty bat is not in memory */
#define BBPTMP 4 /* set if non-persistent bat has image on disk
*/
@@ -157,8 +155,7 @@ gdk_export void BBPshare(bat b);
* are found in O(1) by keeping a freelist that uses the 'next' field
* in the BBPrec records.
*/
-BBPrec *BBP = NULL; /* fixed base VM address of BBP array */
-bat BBPmaxsize; /* size of non-committed VM BBP array */
+BBPrec *BBP[N_BBPINIT]; /* fixed base VM address of BBP array */
bat BBPlimit = 0; /* current committed VM BBP array */
bat BBPsize = 0; /* current used size of BBP array */
@@ -408,23 +405,22 @@ BBPinithash(void)
* BBPtrim, causing deadlock.
*/
static void
-BBPextend(dbl factor, int buildhash)
+BBPextend(int buildhash)
{
- int newsize;
+ BBP_notrim = BBP_getpid();
+
+ if (BBPsize >= N_BBPINIT * BBPINIT)
+ GDKfatal("BBPextend: trying to extend BAT pool beyond the "
+ "limit (%d)\n", N_BBPINIT * BBPINIT);
/* make sure the new size is at least BBPsize large */
- newsize = (int) (BBPlimit * factor);
- while (newsize < BBPsize)
- newsize = (int) (newsize * factor);
-
- BBP_notrim = BBP_getpid();
- BBP = (BBPrec *) GDKrealloc(BBP, newsize * sizeof(BBPrec));
- if (BBP == NULL)
- GDKfatal("BBPextend: failed to extend BAT pool\n");
-
- memset(BBP + BBPlimit, 0, (newsize - BBPlimit) * sizeof(BBPrec));
- BBPlimit = newsize;
- BBPmaxsize = BBPlimit;
+ while (BBPlimit < BBPsize) {
+ assert(BBP[BBPlimit >> BBPINITLOG] == NULL);
+ BBP[BBPlimit >> BBPINITLOG] = GDKzalloc(BBPINIT *
sizeof(BBPrec));
+ if (BBP[BBPlimit >> BBPINITLOG] == NULL)
+ GDKfatal("BBPextend: failed to extend BAT pool\n");
+ BBPlimit += BBPINIT;
+ }
if (buildhash) {
int i;
@@ -816,7 +812,7 @@ BBPreadEntries(FILE *fp, char *src, int
if ((bat) batid >= BBPsize) {
BBPsize = (bat) batid + 1;
if (BBPsize >= BBPlimit)
- BBPextend(BATMARGIN, FALSE);
+ BBPextend(FALSE);
}
if (src == 0 && BBP_desc(bid) != NULL)
GDKfatal("BBPinit: duplicate entry in BBP.dir.");
@@ -1299,11 +1295,11 @@ BBPinit(void)
/* allocate structures: try to reserve as much space as
possible */
- BBP = GDKzalloc(BBPlimit * sizeof(BBPrec));
- if (BBP == NULL)
+ memset(BBP, 0, sizeof(BBP));
+ BBP[0] = GDKzalloc(BBPlimit * sizeof(BBPrec));
+ if (BBP[0] == NULL)
GDKfatal("BBPinit: failed to allocate BBP");
- assert(BBP != NULL);
- BBPmaxsize = BBPlimit;
+ assert(BBP[0] != NULL);
(void) BBPreadEntries(fp, 0, &min_stamp, &max_stamp, oidsize,
bbpversion);
fclose(fp);
@@ -1361,9 +1357,6 @@ BBPexit(void)
bat i;
int skipped;
- if (!BBP)
- return; /* AARGH */
-
BBPrecycle_minsize(0); /* clear the bat cache */
BBPlock("BBPexit"); /* stop all threads ever touching more
descriptors */
@@ -1917,7 +1910,7 @@ BBPinsert(BATstore *bs)
* while we were waiting */
if (BBP_free(idx) <= 0) {
if (BBPsize++ >= BBPlimit) {
- BBPextend(BATMARGIN, TRUE);
+ BBPextend(TRUE);
/* it seems BBPextend could return and
* still leaving BBP_free(idx) == 0 */
if (BBP_free(idx) == 0)
diff --git a/monetdb5/modules/kernel/status.mx
b/monetdb5/modules/kernel/status.mx
--- a/monetdb5/modules/kernel/status.mx
+++ b/monetdb5/modules/kernel/status.mx
@@ -513,7 +513,7 @@ SYSvm_usage(int *ret, lng *minsize)
BUNins(bn, "_tot/tind", &tind, FALSE);
/* special area 1: BBP rec */
- sz = BBPmaxsize * sizeof(BBPrec);
+ sz = BBPlimit * sizeof(BBPrec);
BUNins(bn, "_tot/bbp", &sz, FALSE);
tot += sz;
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
