Changeset: 2bfd82708703 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2bfd82708703
Modified Files:
gdk/gdk_bbp.c
gdk/gdk_private.h
gdk/gdk_utils.c
gdk/gdk_utils.h
Branch: default
Log Message:
Removed trimming on malloc failure; removed trim thread.
diffs (truncated from 771 to 300 lines):
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -185,7 +185,6 @@ getBBPsize(void)
static MT_Lock BBP_curstampLock MT_LOCK_INITIALIZER("BBP_curstampLock");
#endif
static volatile ATOMIC_TYPE BBP_curstamp = 0; /* unique stamp for creation of
a bat */
-MT_Id BBP_notrim = ~((MT_Id) 0); /* avoids BBPtrim when we really do not
want it */
int BBP_dirty = 0; /* BBP structures modified? */
int BBPin = 0; /* bats loaded statistic */
int BBPout = 0; /* bats saved statistic */
@@ -284,12 +283,11 @@ BBPlock(void)
for (i = 0; i <= BBP_THREADMASK; i++)
MT_lock_set(&GDKtrimLock(i));
- BBP_notrim = MT_getpid();
for (i = 0; i <= BBP_THREADMASK; i++)
MT_lock_set(&GDKcacheLock(i));
for (i = 0; i <= BBP_BATMASK; i++)
MT_lock_set(&GDKswapLock(i));
- locked_by = BBP_notrim;
+ locked_by = MT_getpid();
MT_lock_unset(&GDKunloadLock);
}
@@ -303,7 +301,6 @@ BBPunlock(void)
MT_lock_unset(&GDKswapLock(i));
for (i = BBP_THREADMASK; i >= 0; i--)
MT_lock_unset(&GDKcacheLock(i));
- BBP_notrim = 0;
locked_by = 0;
for (i = BBP_THREADMASK; i >= 0; i--)
MT_lock_unset(&GDKtrimLock(i));
@@ -357,14 +354,11 @@ BBPselectfarm(int role, int type, enum h
/*
* BBPextend must take the trimlock, as it is called when other BBP
- * locks are held and it will allocate memory. This could trigger a
- * BBPtrim, causing deadlock.
+ * locks are held and it will allocate memory.
*/
static void
BBPextend(int idx, int buildhash)
{
- BBP_notrim = MT_getpid();
-
if ((bat) ATOMIC_GET(BBPsize, BBPsizeLock) >= N_BBPINIT * BBPINIT)
GDKfatal("BBPextend: trying to extend BAT pool beyond the "
"limit (%d)\n", N_BBPINIT * BBPINIT);
@@ -387,7 +381,6 @@ BBPextend(int idx, int buildhash)
BBP_free(i) = 0;
BBPinithash(idx);
}
- BBP_notrim = 0;
}
static inline str
@@ -1401,7 +1394,6 @@ BBPinit(void)
}
BBPinithash(0);
- BBP_notrim = 0;
OIDbase(BBPoid);
@@ -2089,8 +2081,7 @@ BBPinsert(BAT *bn)
MT_lock_unset(&GDKcacheLock(idx));
MT_lock_unset(&GDKtrimLock(idx));
}
- /* rest of the work outside the lock , as GDKstrdup/GDKmalloc
- * may trigger a BBPtrim */
+ /* rest of the work outside the lock */
/* fill in basic BBP fields for the new bat */
@@ -2294,7 +2285,6 @@ BBPrename(bat bid, const char *nme)
MT_lock_unset(&GDKtrimLock(idx));
return BBPRENAME_ALREADY;
}
- BBP_notrim = MT_getpid();
/* carry through the name change */
if (BBP_logical(bid) && BBPtmpcheck(BBP_logical(bid)) == 0) {
@@ -2318,7 +2308,6 @@ BBPrename(bat bid, const char *nme)
BBPdirty(1);
}
MT_lock_unset(&GDKnameLock);
- BBP_notrim = 0;
MT_lock_unset(&GDKtrimLock(idx));
return 0;
}
@@ -2832,418 +2821,19 @@ BBPfree(BAT *b, const char *calledFrom)
}
/*
- * @- Storage trimming
- * BBPtrim unloads the least recently used BATs to free memory
- * resources. It gets passed targets in bytes of physical memory and
- * logical virtual memory resources to free. Overhead costs are
- * reduced by making just one scan, analyzing the first BBPMAXTRIM
- * bats and keeping the result in a list for later use (the oldest bat
- * now is going to be the oldest bat in the future as well). This
- * list is sorted on last-used timestamp. BBPtrim keeps unloading BATs
- * till the targets are met or there are no more BATs to unload.
- *
- * In determining whether a BAT will be unloaded, first it has to be
- * BBPswappable, and second its resources occupied must be of the
- * requested type. The algorithm actually makes two passes, in the
- * first only clean bats are unloaded (in order of their stamp).
- *
- * In order to keep this under control with multiple threads all
- * running out of memory at the same time, we make sure that
- * @itemize
- * @item
- * just one thread does a BBPtrim at a time (by having a BBPtrimLock
- * set).
- * @item
- * while decisions are made as to which bats to unload (1) the BBP is
- * scanned, and (2) unload decisions are made. Due to these
- * properties, the search&decide phase of BBPtrim acquires both
- * GDKcacheLock (due to (1)) and all GDKswapLocks (due to (2)). They
- * must be released during the actual unloading. (as otherwise
- * deadlock occurs => unloading a bat may e.g. kill an accelerator
- * that is a BAT, which in turn requires BBP lock acquisition).
- * @item
- * to avoid further deadlock, the update functions in BBP that hold
- * either GDKcacheLock or a GDKswapLock may never cause a BBPtrim
- * (notice that BBPtrim could theoretically be set off just by
- * allocating a little piece of memory, e.g. GDKstrdup()). If these
- * routines must alloc memory, they must set the BBP_notrim variable,
- * acquiring the addition GDKtrimLock, in order to prevent such
- * deadlock.
- * @item
- * the BBPtrim is atomic; only releases its locks when all BAT unload
- * work is done. This ensures that if all memory requests that
- * triggered BBPtrim could possible be satisfied by unloading BATs,
- * this will succeed.
- * @end itemize
- *
- * The scan phase was optimized further in order to stop early when it
- * is a priori known that the targets are met (which is the case if
- * the BBPtrim is not due to memory shortage but due to the ndesc
- * quota). Note that scans may always stop before BBPsize as the
- * BBPMAXTRIM is a fixed number which may be smaller. As such, a
- * mechanism was added to resume a broken off scan at the point where
- * scanning was broken off rather than always starting at BBP[1] (this
- * does more justice to the lower numbered bats and will more quickly
- * find fresh unload candidates).
- *
- * We also refined the swap criterion. If the BBPtrim was initiated
- * due to:
- * - too much descriptors: small bats are unloaded first (from LRU
- * cold to hot)
- * - too little memory: big bats are unloaded first (from LRU cold to
- * hot).
- * Unloading-first is enforced by subtracting @math{2^31} from the
- * stamp in the field where the candidates are sorted on.
- *
- * BBPtrim is abandoned when the application has indicated that it
- * does not need it anymore.
- */
-#define BBPMAXTRIM 40000
-#define BBPSMALLBAT 1000
-
-typedef struct {
- bat bid; /* bat id */
- int next; /* next position in list */
- BUN cnt; /* bat count */
-#if SIZEOF_BUN == SIZEOF_INT
- BUN dummy; /* padding to power-of-two size */
-#endif
-} bbptrim_t;
-
-static unsigned lastused[BBPMAXTRIM]; /* bat lastused stamp; sort on this
field */
-static bbptrim_t bbptrim[BBPMAXTRIM];
-static int bbptrimfirst = BBPMAXTRIM, bbptrimlast = 0, bbpunloadtail,
bbpunload, bbptrimmax = BBPMAXTRIM, bbpscanstart = 1;
-
-static bat
-BBPtrim_scan(bat bbppos, bat bbplim)
-{
- bbptrimlast = 0;
- bbptrimmax = BBPMAXTRIM;
- MEMDEBUG fprintf(stderr, "#TRIMSCAN: start=%d, limit=%d\n", (int)
bbppos, (int) bbplim);
-
- if (bbppos < (bat) ATOMIC_GET(BBPsize, BBPsizeLock))
- do {
- if (BBPvalid(bbppos)) {
- BAT *b = BBP_cache(bbppos);
-
- if (BBPtrimmable(b)) {
- /* when unloading for memory,
- * treat small BATs with a
- * preference over big ones.
- * rationale: I/O penalty for
- * cache miss is relatively
- * higher for small bats
- */
- BUN cnt = BATcount(b);
- unsigned swap_first = (cnt >=
BBPSMALLBAT);
-
- /* however, when we are
- * looking to decrease the
- * number of descriptors, try
- * to put the small bats in
- * front of the load list
- * instead..
- */
-
- /* subtract 2-billion to make
- * sure the swap_first class
- * bats are unloaded first */
- lastused[bbptrimlast] = (unsigned)
BBPLASTUSED(BBP_lastused(bbppos)) | (swap_first << 31);
- bbptrim[bbptrimlast].bid = bbppos;
- bbptrim[bbptrimlast].cnt = cnt;
- if (++bbptrimlast == bbptrimmax)
- break;
- }
- }
- if (++bbppos == (bat) ATOMIC_GET(BBPsize, BBPsizeLock))
- bbppos = 1; /* treat BBP as a circular
buffer */
- } while (bbppos != bbplim);
-
- if (bbptrimlast > 0) {
- int i;
- /* sort lastused array as (signed) int */
- GDKqsort(lastused, bbptrim, NULL, bbptrimlast,
- sizeof(lastused[0]), sizeof(bbptrim[0]), TYPE_int);
- for (i = bbptrimfirst = 0; i < bbptrimlast; i++) {
- MEMDEBUG fprintf(stderr, "#TRIMSCAN: %11d%c %9d=%s\t(#"
BUNFMT ")\n", (int) BBPLASTUSED(lastused[i]), lastused[i] & ((unsigned) 1 <<
31) ? '*' : ' ', i, BBPname(bbptrim[i].bid), bbptrim[i].cnt);
-
- bbptrim[i].next = i + 1;
- }
- bbptrim[bbptrimlast - 1].next = BBPMAXTRIM;
- } else {
- bbptrimfirst = BBPMAXTRIM;
- }
- MEMDEBUG fprintf(stderr, "#TRIMSCAN: end at %d (size=%d)\n", bbppos,
(int) (bat) ATOMIC_GET(BBPsize, BBPsizeLock));
-
- return bbppos;
-}
-
-
-/* insert BATs to unload from bbptrim list into bbpunload list;
- * rebuild bbptrimlist only with the useful leftovers */
-static size_t
-BBPtrim_select(size_t target, int dirty)
-{
- int bbptrimtail = BBPMAXTRIM, next = bbptrimfirst;
-
- MEMDEBUG fprintf(stderr, "#TRIMSELECT: dirty = %d\n", dirty);
-
- /* make the bbptrim-list empty; we will insert the untouched
- * elements in it */
- bbptrimfirst = BBPMAXTRIM;
-
- while (next != BBPMAXTRIM) {
- int cur = next; /* cur is the entry in the old bbptrimlist we
are processing */
- int untouched = BBPLASTUSED(BBP_lastused(bbptrim[cur].bid)) <=
(int) BBPLASTUSED(lastused[cur]);
- BAT *b = BBP_cache(bbptrim[cur].bid);
-
- next = bbptrim[cur].next; /* do now, because we overwrite
bbptrim[cur].next below */
-
- MEMDEBUG if (b) {
- fprintf(stderr,
- "#TRIMSELECT: candidate=%s BAT*=" PTRFMT "\n",
- BBPname(bbptrim[cur].bid),
- PTRFMTCAST(void *)b);
-
- fprintf(stderr,
- "# (cnt=" BUNFMT ", mode=%d, "
- "refs=%d, wait=%d, parent=%d, "
- "lastused=%d,%d,%d)\n",
- bbptrim[cur].cnt,
- (int) b->batPersistence,
- BBP_refs(b->batCacheid),
- (BBP_status(b->batCacheid) & BBPWAITING) != 0,
- VIEWtparent(b),
- BBP_lastused(b->batCacheid),
- (int) BBPLASTUSED(lastused[cur]),
- (int) lastused[cur]);
- }
- /* recheck if conditions encountered by trimscan in
- * the past still hold */
- if (BBPtrimmable(b) && untouched) {
- size_t memdelta = BATmemsize(b, FALSE) + BATvmsize(b,
FALSE);
- size_t memdirty = BATmemsize(b, TRUE) + BATvmsize(b,
TRUE);
-
- if (((b->batPersistence == TRANSIENT &&
- BBP_lrefs(bbptrim[cur].bid) == 0) || /* needs not
be saved when unloaded, OR.. */
- memdirty <= sizeof(BAT) || /* the BAT is actually
clean, OR.. */
- dirty) /* we are allowed to cause I/O (second
run).. */
- && /* AND ... */
- target > 0 && memdelta > 0)
- /* there is some reward in terms of
- * memory requirements */
- {
- /* only then we unload! */
- MEMDEBUG {
- fprintf(stderr,
- "#TRIMSELECT: unload %s ["
SZFMT "] bytes [" SZFMT "] dirty\n",
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
