Changeset: 0323ba750942 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0323ba750942
Modified Files:
monetdb5/mal/mal.h
monetdb5/mal/mal_recycle.c
monetdb5/mal/mal_resource.c
Branch: default
Log Message:
Use more memory on large machines
For the top of the line 1 TB RAM machines, we now
use up to 8GB before trimming parallel processing.
diffs (107 lines):
diff --git a/monetdb5/mal/mal.h b/monetdb5/mal/mal.h
--- a/monetdb5/mal/mal.h
+++ b/monetdb5/mal/mal.h
@@ -38,7 +38,13 @@
* for additional system variable settings.
*/
#define MAXSCRIPT 64
-#define MEMORY_THRESHOLD 0.8
+
+/*
+ * MonetDB assumes it can use most of the machines memory,
+ * leaving a small portion for other programs.
+ */
+#define GB (((lng)1024)*1024*1024)
+#define MEMORY_THRESHOLD (0.2 * monet_memory > 8 * GB? monet_memory - 8 *
GB: 0.8 * monet_memory)
mal_export char monet_cwd[PATHLENGTH];
mal_export size_t monet_memory;
diff --git a/monetdb5/mal/mal_recycle.c b/monetdb5/mal/mal_recycle.c
--- a/monetdb5/mal/mal_recycle.c
+++ b/monetdb5/mal/mal_recycle.c
@@ -168,10 +168,10 @@ int chooseVictims(Client cntxt, int *lea
{
int i,j,l;
lng freed, sz;
- lng target = (lng) MT_getrss() - (lng) (MEMORY_THRESHOLD *
monet_memory);
+ lng target = (lng) MT_getrss() - (lng) (MEMORY_THRESHOLD );
#ifdef _DEBUG_CACHE_
- mnstr_printf(cntxt->fdout,"#chooseVictims %d mem "LLFMT, ltop, (lng)
(MEMORY_THRESHOLD * monet_memory));
+ mnstr_printf(cntxt->fdout,"#chooseVictims %d mem "LLFMT, ltop, (lng)
(MEMORY_THRESHOLD ));
mnstr_printf(cntxt->fdout," RSS "LLFMT, (lng) MT_getrss());
mnstr_printf(cntxt->fdout," " LLFMT" srch %5.2f\n",
target, ((double)recycleSearchTime)/recycleSearchCalls);
@@ -382,7 +382,7 @@ newpass:
GDKfree(dmask);
/* check if a new pass of cache cleaning is needed */
- if ( recyclerMemoryUsed > (lng) (MEMORY_THRESHOLD * monet_memory) )
+ if ( recyclerMemoryUsed > (lng) (MEMORY_THRESHOLD ) )
goto newpass;
}
@@ -399,7 +399,7 @@ RECYCLEkeep(Client cntxt, MalBlkPtr mb,
if ( recycleBlk->stop >= recycleCacheLimit)
return ; /* no more caching */
- if ( recyclerMemoryUsed + p->wbytes > (lng) (MEMORY_THRESHOLD *
monet_memory))
+ if ( recyclerMemoryUsed + p->wbytes > (lng) (MEMORY_THRESHOLD ))
return ; /* no more caching */
rd = 0;
@@ -859,7 +859,7 @@ RECYCLEexit(Client cntxt, MalBlkPtr mb,
assert(recyclerMemoryUsed >=0);
MT_lock_set(&recycleLock, "recycle");
clk = GDKusec();
- if ( recyclerMemoryUsed > (lng) (MEMORY_THRESHOLD * monet_memory) ||
recycleBlk->stop >= recycleCacheLimit)
+ if ( recyclerMemoryUsed > (lng) (MEMORY_THRESHOLD ) ||
recycleBlk->stop >= recycleCacheLimit)
RECYCLEcleanCache(cntxt);
/* infinite case, admit all new instructions */
diff --git a/monetdb5/mal/mal_resource.c b/monetdb5/mal/mal_resource.c
--- a/monetdb5/mal/mal_resource.c
+++ b/monetdb5/mal/mal_resource.c
@@ -78,7 +78,7 @@ getMemoryClaim(MalBlkPtr mb, MalStkPtr s
heapinfo(&b->T->heap); total += vol;
heapinfo(b->T->vheap); total += vol;
hashinfo(b->T->hash); total += vol;
- total = total > (lng)(MEMORY_THRESHOLD * monet_memory) ?
(lng)(MEMORY_THRESHOLD * monet_memory) : total;
+ total = total > (lng)(MEMORY_THRESHOLD ) ?
(lng)(MEMORY_THRESHOLD ) : total;
BBPunfix(b->batCacheid);
}
return total;
@@ -127,7 +127,7 @@ MALadmission(lng argclaim, lng hotclaim)
if (memoryclaims < 0)
memoryclaims = 0;
if (memorypool <= 0 && memoryclaims == 0)
- memorypool = (lng)(MEMORY_THRESHOLD * monet_memory);
+ memorypool = (lng)(MEMORY_THRESHOLD );
if (argclaim > 0) {
if (memoryclaims == 0 || memorypool > argclaim + hotclaim) {
@@ -182,7 +182,7 @@ MALresourceFairness(lng usec)
/* use GDKmem_cursize as MT_getrss() is too expensive */
rss = GDKmem_cursize();
/* ample of memory available*/
- if ( rss < MEMORY_THRESHOLD * monet_memory && usec <= TIMESLICE)
+ if ( rss < MEMORY_THRESHOLD && usec <= TIMESLICE)
return;
/* worker reporting time spent in usec! */
@@ -194,13 +194,13 @@ MALresourceFairness(lng usec)
/* always keep one running to avoid all waiting */
while (clk > 0 && running >= 2 && delayed < MAX_DELAYS) {
/* speed up wake up when we have memory */
- if (rss < MEMORY_THRESHOLD * monet_memory)
+ if (rss < MEMORY_THRESHOLD )
break;
threads = GDKnr_threads > 0 ? GDKnr_threads : 1;
delay = (unsigned int) ( ((double)DELAYUNIT * running)
/ threads);
if (delay) {
if ( delayed++ == 0){
- mnstr_printf(GDKstdout, "#delay
initial %u["LLFMT"] memory "SZFMT"[%f]\n", delay, clk, rss, MEMORY_THRESHOLD *
monet_memory);
+ mnstr_printf(GDKstdout, "#delay
initial %u["LLFMT"] memory "SZFMT"[%f]\n", delay, clk, rss, MEMORY_THRESHOLD );
mnstr_flush(GDKstdout);
}
MT_sleep_ms(delay);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
