Changeset: 7330f93e50af for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7330f93e50af
Modified Files:
sql/storage/bat/bat_storage.c
Branch: default
Log Message:
Merge from Dec2011 branch.
diffs (120 lines):
diff --git a/gdk/gdk_utils.mx b/gdk/gdk_utils.mx
--- a/gdk/gdk_utils.mx
+++ b/gdk/gdk_utils.mx
@@ -383,7 +383,7 @@ BATSIGinit(void)
size_t GDK_mmap_minsize = GDK_VM_MAXSIZE;
size_t GDK_mem_maxsize_max = GDK_VM_MAXSIZE;
size_t GDK_mem_maxsize = GDK_VM_MAXSIZE;
-size_t GDK_mem_bigsize = 1 << 20;
+size_t GDK_mem_bigsize = 1 << 30;
size_t GDK_vm_maxsize = GDK_VM_MAXSIZE;
int GDK_vm_allocs = 0;
@@ -1674,8 +1674,13 @@ GDKinit(opt *set, int setlen)
GDKlockHome();
/* Mserver by default takes 80% of all memory as a default */
- GDK_mem_maxsize_max = (size_t) ((double) MT_npages() * (double)
MT_pagesize() * 0.815);
- GDK_mmap_minsize = GDK_mem_maxsize = GDK_mem_maxsize_max;
+ GDK_mem_maxsize = GDK_mem_maxsize_max = (size_t) ((double) MT_npages()
* (double) MT_pagesize() * 0.815);
+#ifdef NATIVE_WIN32
+ GDK_mmap_minsize = GDK_mem_maxsize_max;
+#else
+ GDK_mmap_minsize = GDK_mem_bigsize = MIN( 1<<30 , GDK_mem_maxsize_max/6
);
+ /* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^
*/
+#endif
GDKmemchk(TRUE, TRUE);
GDKremovedir(DELDIR);
BBPinit();
@@ -1746,10 +1751,12 @@ GDKinit(opt *set, int setlen)
if (GDKnr_threads == 0)
GDKnr_threads = MT_check_nr_cores();
#ifdef NATIVE_WIN32
- if (GDKnr_threads)
- GDK_mmap_minsize /= (GDKnr_threads? GDKnr_threads: 1);
+ GDK_mmap_minsize /= (GDKnr_threads ? GDKnr_threads : 1);
#else
- GDK_mmap_minsize = 256 * 1024 * 1024;
+ /* WARNING: This unconditionally overwrites above settings, */
+ /* incl. setting via MonetDB env. var. "gdk_mmap_minsize" ! */
+ GDK_mmap_minsize = GDK_mem_bigsize = MIN( 1<<30 ,
(GDK_mem_maxsize_max/6) / (GDKnr_threads ? GDKnr_threads : 1) );
+ /* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6
^ */
#endif
#ifdef HAVE_POSIX_FADVISE
diff --git a/monetdb5/optimizer/opt_mitosis.mx
b/monetdb5/optimizer/opt_mitosis.mx
--- a/monetdb5/optimizer/opt_mitosis.mx
+++ b/monetdb5/optimizer/opt_mitosis.mx
@@ -73,8 +73,6 @@ comment "Modify the plan to exploit para
#include "opt_octopus.h"
#include "mal_interpreter.h"
-#define PARTITION_THRESHOLD (wrd) threads /* should be increased in
production version */
-
static int eligible(MalBlkPtr mb )
{
InstrPtr p;
@@ -97,10 +95,11 @@ OPTmitosisImplementation(Client cntxt, M
int i, j, k, limit, estimate=0, tpe, pieces=1;
str schema=0, table=0;
VarRecord low,hgh;
- oid slice;
+ BUN slice;
wrd r = 0, rowcnt=0; /* table should be sizeable to consider
parallel execution*/
InstrPtr q,*old, target= 0, matq;
- size_t argsize= 3 *sizeof(lng); /* 2 arguments and a result */
+ size_t argsize = 6 * sizeof(lng);
+ /* per op: 6 = (2+1)*2 <= 2 args + 1 res, each with head &
tail */
int threads = GDKnr_threads ? GDKnr_threads:1;
ValRecord vr;
VarPtr loc,rows;
@@ -142,22 +141,30 @@ OPTmitosisImplementation(Client cntxt, M
* For the time being we just go for pieces that fit into memory in
isolation.
* A fictive rowcount is derived based on argument types, such that all
pieces
* would fit into memory conveniently for processing. We attempt to use
- * not more threads then strictly needed.
+ * not more threads than strictly needed.
* Experience shows that the pieces should not be too small.
* If we should limit to |threads| is still an open issue.
*/
if ( (i = OPTlegAdviceInternal(mb,stk,p)) > 0 )
pieces = i;
else {
- /* ensure that GDKnr_threads partitions fit into main memory */
- r = (BUN) (monet_memory / argsize / 4);
- if ( (r? (int) rowcnt/r:1) < (wrd) threads && r > MINPARTCNT)
- pieces = (r? (int) rowcnt/r:1);
+ r = (wrd) (monet_memory / argsize);
+ /* if data exceeds memory size,
+ * i.e., (rowcnt*argsize > monet_memory),
+ * i.e., (rowcnt > monet_memory/argsize = r) */
+ if (rowcnt > r && r/threads > 0)
+ /* create |pieces| > |threads| partitions such that
+ * |threads| partitions at a time fit in memory,
+ * i.e., (threads*(rowcnt/pieces) <= r),
+ * i.e., (rowcnt/pieces <= r/threads),
+ * i.e., (pieces => rowcnt/(r/threads))
+ * (assuming that (r > threads*MINPARTCNT)) */
+ pieces = (int) (rowcnt / (r/threads)) + 1;
else
/* exploit parallelism, but ensure minimal partition size to
limit overhead */
if (rowcnt > MINPARTCNT)
pieces = (int) MIN ( (rowcnt / MINPARTCNT) , (wrd)
threads);
- /* when testing, split also small BATs, but avoid empty pieces
*/
+ /* when testing, always aim for full parallelism, but avoid
empty pieces */
FORCEMITODEBUG
if (pieces < threads)
pieces = (int) MIN ( (wrd) threads , rowcnt );
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -251,7 +251,8 @@ delta_append_bat( sql_delta *bat, BAT *i
/* We simply use the to be inserted bat directly.
* Disabled this optimization: sometimes the bat is used later in the
* mal plan.
- * This should be solved by changing the input into a view (somehow)
+ * This should be solved by changing the input into a view (somehow).
+ * Alternatively, COPY INTO ... LOCKED can/should be used.
if (BATcount(b) == 0 && !isVIEW(i) && BBP_lrefs(i->batCacheid) <= 1 &&
i->htype == TYPE_void && i->ttype != TYPE_void && bat->ibase == i->H->seq){
temp_destroy(bat->ibid);
bat->ibid = temp_create(i);
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
