Changeset: 0c0aca45ad39 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c0aca45ad39
Modified Files:
MonetDB5/src/optimizer/opt_mitosis.mx
Branch: Oct2010
Log Message:
cleaned up strategy to determine number of mitosis partitions ("pieces"):
- If the BAT is larger than 1/GDKnr_threads of main memory,
split it into GDKnr_threads or more partitions such that
each partition is smaller than 1/GDKnr_threads of main memory.
- Otherwise, if the BAT has more than MINPARTCNT (= 100000) BUNs,
split it into at most GDKnr_threads partitions such that
each partition has at least MINPARTCNT BUNs.
- With --forcemito (aka. --debug=536870912), e.g., during testing,
split also small BATs into up to GDKnr_threads non-empty partitions.
- Limit number of partitions to max. MAXSLICES (=256)
to prevent plan explosion.
diffs (42 lines):
diff -r b36659c13471 -r 0c0aca45ad39 MonetDB5/src/optimizer/opt_mitosis.mx
--- a/MonetDB5/src/optimizer/opt_mitosis.mx Mon Nov 08 11:01:56 2010 +0100
+++ b/MonetDB5/src/optimizer/opt_mitosis.mx Mon Nov 08 11:12:17 2010 +0100
@@ -60,6 +60,7 @@
#include "opt_support.h"
#define MAXSLICES 256 /* to be refined */
+#define MINPARTCNT 100000 /* minimal record count per partition */
@:exportOptimizer(mitosis)@
#define OPTDEBUGmitosis if ( optDebug & ((lng)1 <<DEBUG_OPT_MITOSIS) )
#endif
@@ -143,16 +144,23 @@
if ( (i = OPTlegAdviceInternal(mb,stk,p)) > 0 )
pieces = i;
else {
- r= (BUN) (monet_memory /typewidth/GDKnr_threads); /* how much
fits */
- if (rowcnt > r )
- pieces = (int) (rowcnt /r+1);
+ /* ensure that GDKnr_threads partitions fit into main memory */
+ r = (BUN) (monet_memory / typewidth / GDKnr_threads);
+ if (rowcnt > r)
+ pieces = MAX ( (int)(rowcnt / r + 1) , GDKnr_threads );
+ else
+ /* exploit parallelism, but ensure minimal partition size to
limit overhead */
+ if (rowcnt > MINPARTCNT)
+ pieces = MIN ( (int)(rowcnt / MINPARTCNT) ,
GDKnr_threads );
+ /* when testing, split also small BATs, but avoid empty pieces
*/
FORCEMITODEBUG
- if (pieces < GDKnr_threads )
- pieces = GDKnr_threads; /* split up anyway when
testing */
+ if (pieces < GDKnr_threads)
+ pieces = MIN ( GDKnr_threads , (int)rowcnt );
+ /* prevent plan explosion */
if (pieces > MAXSLICES)
- pieces = MAXSLICES; /* cut off potential plan explosion
*/
+ pieces = MAXSLICES;
- if ( (size_t) rowcnt < (size_t) pieces || pieces <=1)
+ if (pieces <= 1)
return 0;
}
OPTDEBUGmitosis
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
