Changeset: be9c6128a574 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=be9c6128a574
Modified Files:
monetdb5/mal/mal_interpreter.mx
monetdb5/optimizer/opt_mitosis.mx
Branch: Dec2011
Log Message:
Use large blocks and minimal threads
Performance is improved if we use large blocks in mitosis, because
it minimizes the thread/dataflow related overhead. However, large
blocks ask for a more agressive memory control. Therefore, the
admission control scheme has been re-enabled.
Most likely there are more improvements in mitosis and runtime
scheduling possible.
(transplanted from b3e0b80a379a368d84503d2e9a238ddd3cba001c)
diffs (68 lines):
diff --git a/monetdb5/mal/mal_interpreter.mx b/monetdb5/mal/mal_interpreter.mx
--- a/monetdb5/mal/mal_interpreter.mx
+++ b/monetdb5/mal/mal_interpreter.mx
@@ -80,10 +80,12 @@ static void displayVolume(Client cntxt,
#define MEMORY_THRESHOLD 0.8
+#define USE_DFLOW_ADMISSION
#ifdef USE_DFLOW_ADMISSION
/* does not seem to have a major impact */
static lng memorypool; /* memory claimed by concurrent threads */
static int memoryclaims = 0; /* number of threads active with expensive
operations */
+mal_export int DFLOWadmission(lng argclaim, lng hotclaim);
#endif
#define heapinfo(X) if ((X) && (X)->base) vol = (X)->free; else vol = 0;
@@ -706,6 +708,8 @@ getMemoryClaim(MalBlkPtr mb, MalStkPtr s
return 0;
}
@:calcFootprint@
+ if ( b->T->hash == 0 || b->H->hash ==0) /* assume one
hash claim */
+ total+= BATcount(b) * sizeof(lng);
total = total > (lng)(MEMORY_THRESHOLD * monet_memory) ?
(lng)(MEMORY_THRESHOLD * monet_memory) : total;
BBPunfix(b->batCacheid);
}
diff --git a/monetdb5/optimizer/opt_mitosis.mx
b/monetdb5/optimizer/opt_mitosis.mx
--- a/monetdb5/optimizer/opt_mitosis.mx
+++ b/monetdb5/optimizer/opt_mitosis.mx
@@ -100,7 +100,7 @@ OPTmitosisImplementation(Client cntxt, M
oid slice;
wrd r = 0, rowcnt=0; /* table should be sizeable to consider
parallel execution*/
InstrPtr q,*old, target= 0, matq;
- size_t typewidth= 3 *sizeof(lng); /* 2 arguments and a result */
+ size_t argsize= 3 *sizeof(lng); /* 2 arguments and a result */
int threads = GDKnr_threads ? GDKnr_threads:1;
ValRecord vr;
VarPtr loc,rows;
@@ -137,23 +137,22 @@ OPTmitosisImplementation(Client cntxt, M
if (target == 0)
return 0;
/*
- * @-
* The number of pieces should be based on the footprint of the
queryplan,
* such that preferrably it can be handled without swapping
intermediates.
- * For the time being we just go for small pieces compared to memory
detected upon start.
- * A fictive rowcount is derived based on TYPE_lng, such that all pieces
+ * For the time being we just go for pieces that fit into memory in
isolation.
+ * A fictive rowcount is derived based on argument types, such that all
pieces
* would fit into memory conveniently for processing. We attempt to use
- * at least all threads.
+ * not more threads then strictly needed.
* Experience shows that the pieces should not be too small.
- * To keep the threads working, we create more pieces than threads.
+ * If we should limit to |threads| is still an open issue.
*/
if ( (i = OPTlegAdviceInternal(mb,stk,p)) > 0 )
pieces = i;
else {
/* ensure that GDKnr_threads partitions fit into main memory */
- r = (BUN) (monet_memory / typewidth / threads);
- if (rowcnt > r)
- pieces = (int) MAX ( (r ? rowcnt/r :1) , (wrd) threads
);
+ r = (BUN) (monet_memory / argsize / 4);
+ if ( (r? (int) rowcnt/r:1) < (wrd) threads && r > MINPARTCNT)
+ pieces = (r? (int) rowcnt/r:1);
else
/* exploit parallelism, but ensure minimal partition size to
limit overhead */
if (rowcnt > MINPARTCNT)
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
