Changeset: 476aeed5f46d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=476aeed5f46d
Modified Files:
monetdb5/modules/mal/mal_init.mx
monetdb5/optimizer/opt_tarantula.mx
monetdb5/scheduler/run_tarantula.mx
Branch: sciql
Log Message:
Merge with default branch.
diffs (truncated from 2999 to 300 lines):
diff --git a/buildtools/conf/rules.mk b/buildtools/conf/rules.mk
--- a/buildtools/conf/rules.mk
+++ b/buildtools/conf/rules.mk
@@ -131,4 +131,4 @@
distdir: check_dist
check_dist:
- @if [ "$(SWIG)" = "no" ]; then $(ECHO) "Cannot create distribution
because one of the necessary programs or libraries is missing"; echo "swig =
$(SWIG)"; exit 1; fi
+ @if [ "$(SWIG)" = "no" ]; then echo "Cannot create distribution because
one of the necessary programs or libraries is missing"; echo "swig =
$(SWIG)"; exit 1; fi
diff --git a/monetdb5/mal/mal_debugger.mx b/monetdb5/mal/mal_debugger.mx
--- a/monetdb5/mal/mal_debugger.mx
+++ b/monetdb5/mal/mal_debugger.mx
@@ -750,7 +750,7 @@
p= getInstrPtr(mb,pc);
msg = instruction2str(mb, stk, p, flags);
- mnstr_printf(out, "#%s\n", msg);
+ mnstr_printf(out, "#%s%s\n", (mb->errors?"!":""), msg);
GDKfree(msg);
}
diff --git a/monetdb5/modules/mal/mal_init.mx b/monetdb5/modules/mal/mal_init.mx
--- a/monetdb5/modules/mal/mal_init.mx
+++ b/monetdb5/modules/mal/mal_init.mx
@@ -150,11 +150,11 @@
include opt_reduce;
include opt_remap;
include opt_recycler;
+include opt_slicing;
include opt_singleton;
include opt_strengthReduction;
include opt_statistics;
include opt_trace;
-include opt_tarantula;
include opt_mapreduce;
include opt_remoteQueries;
diff --git a/monetdb5/modules/mal/mat.mx b/monetdb5/modules/mal/mat.mx
--- a/monetdb5/modules/mal/mat.mx
+++ b/monetdb5/modules/mal/mat.mx
@@ -49,6 +49,10 @@
address MATpackValues
comment "Materialize the MAT (of values) into a BAT";
+pattern pack(:any_2...):bat[:oid,:any_2]
+address MATpackValues
+comment "Materialize the MAT (of values) into a BAT";
+
pattern pack(b:bat[:any_1,:any_2]...):bat[:any_1,:any_2]
address MATpack
comment "Materialize the MAT into a BAT";
diff --git a/monetdb5/optimizer/Makefile.ag b/monetdb5/optimizer/Makefile.ag
--- a/monetdb5/optimizer/Makefile.ag
+++ b/monetdb5/optimizer/Makefile.ag
@@ -77,7 +77,7 @@
opt_statistics.mx \
opt_strengthReduction.mx \
opt_support.mx \
- opt_tarantula.mx \
+ opt_slicing.mx \
opt_trace.mx
}
@@ -97,7 +97,7 @@
opt_accessmode.mx opt_joinpath.mx opt_heuristics.mx
opt_remap.mx \
opt_statistics.mx opt_trace.mx opt_recycler.mx opt_dataflow.mx
\
opt_cluster.mx opt_dictionary.mx opt_mapreduce.mx opt_origin.mx
\
- opt_mitosis.mx opt_octopus.mx opt_tarantula.mx opt_history.mx
opt_selcrack.mx opt_sidcrack.mx
+ opt_mitosis.mx opt_octopus.mx opt_slicing.mx opt_history.mx
opt_selcrack.mx opt_sidcrack.mx
}
EXTRA_DIST = optimizerWrapper.mx
diff --git a/monetdb5/optimizer/opt_dataflow.mx
b/monetdb5/optimizer/opt_dataflow.mx
--- a/monetdb5/optimizer/opt_dataflow.mx
+++ b/monetdb5/optimizer/opt_dataflow.mx
@@ -38,6 +38,10 @@
Dataflow blocks may not be nested. Therefore, any dataflow block produced for
inlined code is removed first.
+A dataflow block should only be created if there is potential parallelism.
+This means that either we have multiple starters for a thread, or a
+result produced in the block is used concurrently multiple times.
+
Initial experiments on e.g. the RDF benchmark showed a speed up of
20\% on average, with a peak of a factor 2 for individual queries.
The main reason for this limited gain stems from the little
diff --git a/monetdb5/optimizer/opt_deadcode.mx
b/monetdb5/optimizer/opt_deadcode.mx
--- a/monetdb5/optimizer/opt_deadcode.mx
+++ b/monetdb5/optimizer/opt_deadcode.mx
@@ -128,6 +128,10 @@
break;
}
+ if ( p->token == NOOPsymbol){
+ freeInstruction(p);
+ actions++;
+ } else
if( getModuleId(p)== sqlRef && getFunctionId(p)== assertRef &&
isVarConstant(mb,getArg(p,1)) &&
getVarConstant(mb,getArg(p,1)).val.ival==0){
freeInstruction(p);
diff --git a/monetdb5/optimizer/opt_mitosis.mx
b/monetdb5/optimizer/opt_mitosis.mx
--- a/monetdb5/optimizer/opt_mitosis.mx
+++ b/monetdb5/optimizer/opt_mitosis.mx
@@ -68,7 +68,6 @@
#include "monetdb_config.h"
#include "opt_mitosis.h"
#include "opt_octopus.h"
-#include "opt_tarantula.h"
#include "mal_interpreter.h"
#define PARTITION_THRESHOLD (wrd) threads /* should be increased in
production version */
@@ -143,14 +142,9 @@
Experience shows that the pieces should not be too small.
To keep the threads working, we create more pieces than threads.
-The pieces to be used may be overruled by the octopus and
-tarantula optimizer.
@c
if ( (i = OPTlegAdviceInternal(mb,stk,p)) > 0 )
pieces = i;
- else
- if ( (i = OPTtarantulaAdviceInternal(mb,stk,p)) > 0 )
- pieces = i;
else {
/* ensure that GDKnr_threads partitions fit into main memory */
r = (BUN) (monet_memory / typewidth / threads);
@@ -243,7 +237,7 @@
setVarUDFtype(mb,k);
setVarUsed(mb,k);
@-
-The target variable should inherit file location and row count for the
tarantula.
+The target variable should inherit file location and row count
@c
loc = varGetProp(mb, getArg(p,0), fileProp);
if ( loc ){
diff --git a/monetdb5/optimizer/opt_origin.mx b/monetdb5/optimizer/opt_origin.mx
--- a/monetdb5/optimizer/opt_origin.mx
+++ b/monetdb5/optimizer/opt_origin.mx
@@ -20,7 +20,7 @@
@f opt_origin
@- OID origin tracking.
This simple module extends the MAL plan with properties that
-identify the source of OIDs in the underlying SQL persistent tables.
+identify the source of values in the underlying SQL persistent tables.
It stops when it can not be unique pinpointed anymore.
The information is kept with the variables as horigin and torigin.
[still incomplete]
@@ -42,9 +42,9 @@
@:exportOptimizer(origin)@
+/* #define _DEBUG_OPT_ORIGIN_ */
#undef DEBUG_OPT_ORIGIN
#define DEBUG_OPT_ORIGIN 1
-#define _DEBUG_OPT_ORIGIN_
#define OPTDEBUGorigin if ( optDebug & (1 <<DEBUG_OPT_ORIGIN) )
#endif
@@ -81,31 +81,16 @@
#endif
}
if ( getModuleId(p) == algebraRef) {
- if ( getFunctionId(p) == kunionRef) {
- h= varGetProp(mb, getArg(p,1), horiginProp);
- t= varGetProp(mb, getArg(p,2), horiginProp);
- if ( strcmp(h->value.val.sval,
t->value.val.sval) == 0) {
- VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
- varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
- }
- }
- if ( getFunctionId(p) == kdifferenceRef) {
+ if ( getFunctionId(p) == kdifferenceRef ||
+ getFunctionId(p) == kunionRef ||
+ getFunctionId(p) == thetauselectRef
+ ) {
h= varGetProp(mb, getArg(p,1), horiginProp);
if ( h ) {
VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
}
}
- if ( getFunctionId(p) == thetauselectRef) {
- h= varGetProp(mb, getArg(p,1), horiginProp);
- t= varGetProp(mb, getArg(p,1), toriginProp);
- if ( h ) {
- VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
- varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
- }
- if ( t)
- {}
- }
if ( getFunctionId(p) == markTRef) {
h= varGetProp(mb, getArg(p,1), horiginProp);
if ( h ){
@@ -113,8 +98,42 @@
varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
}
}
+ if ( getFunctionId(p) == leftjoinRef ||
+ getFunctionId(p) == joinRef) {
+ h= varGetProp(mb, getArg(p,1), horiginProp);
+ t= varGetProp(mb, getArg(p,2), toriginProp);
+ if ( h ){
+ VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
+ varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
+ }
+ if ( t ){
+ VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
+ varSetProp(mb, getArg(p,0),
toriginProp, op_eq, &val);
+ }
+ }
+ }
+ if ( getModuleId(p) == groupRef) {
+ if ( getFunctionId(p) == newRef ||
+ getFunctionId(p) == doneRef ) {
+ h= varGetProp(mb, getArg(p,1), horiginProp);
+ if ( h ){
+ VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
+ varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
+ VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
+ varSetProp(mb, getArg(p,1),
horiginProp, op_eq, &val);
+ }
+ }
}
if ( getModuleId(p) == batRef) {
+ if ( getFunctionId(p) == mirrorRef) {
+ h= varGetProp(mb, getArg(p,1), horiginProp);
+ if ( h ){
+ VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
+ varSetProp(mb, getArg(p,0),
horiginProp, op_eq, &val);
+ VALset(&val, TYPE_str,
GDKstrdup(h->value.val.sval));
+ varSetProp(mb, getArg(p,0),
toriginProp, op_eq, &val);
+ }
+ }
if ( getFunctionId(p) == reverseRef) {
h= varGetProp(mb, getArg(p,1), horiginProp);
t= varGetProp(mb, getArg(p,1), toriginProp);
diff --git a/monetdb5/optimizer/opt_prelude.mx
b/monetdb5/optimizer/opt_prelude.mx
--- a/monetdb5/optimizer/opt_prelude.mx
+++ b/monetdb5/optimizer/opt_prelude.mx
@@ -198,7 +198,6 @@
opt_export str utopn_minRef;
opt_export str utopn_maxRef;
opt_export str tuniqueRef;
-opt_export str tarantulaRef;
opt_export str not_uniqueRef;
opt_export str unionRef;
opt_export str unpackRef;
@@ -413,7 +412,6 @@
str sumRef;
str sunionRef;
str takeRef;
-str tarantulaRef;
str topn_minRef;
str topn_maxRef;
str utopn_minRef;
@@ -629,7 +627,6 @@
sumRef = putName("sum",3);
sunionRef= putName("sunion",6);
takeRef= putName("take",5);
- tarantulaRef = putName("tarantula",9);
topn_minRef= putName("topn_min",8);
topn_maxRef= putName("topn_max",8);
utopn_minRef= putName("utopn_min",9);
diff --git a/monetdb5/optimizer/opt_slicing.mx
b/monetdb5/optimizer/opt_slicing.mx
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_slicing.mx
@@ -0,0 +1,809 @@
+@/
+The contents of this file are subject to the MonetDB Public License
+Version 1.1 (the "License"); you may not use this file except in
+compliance with the License. You may obtain a copy of the License at
+http://monetdb.cwi.nl/Legal/MonetDBLicense-1.1.html
+
+Software distributed under the License is distributed on an "AS IS"
+basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+License for the specific language governing rights and limitations
+under the License.
+
+The Original Code is the MonetDB Database System.
+
+The Initial Developer of the Original Code is CWI.
+Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+Copyright August 2008-2011 MonetDB B.V.
+All Rights Reserved.
+@
+Use a single gauge structure to split the largest table.
+@f opt_slicing
+@a M. Kersten
+@- Slicing
+@{
+@mal
+pattern optimizer.slicing():str
+address OPTslicing;
+pattern optimizer.slicing(mod:str, fcn:str):str
+address OPTslicing
+comment "Modify the plan to exploit parallel processing on multiple cores";
+
+pattern sql.gauges(b:bat[:oid,:any_1]) :any_1...
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
