Changeset: 25dd9f357d94 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=25dd9f357d94
Modified Files:
sql/backends/monet5/Makefile.ag
sql/server/sql_parser.h
sql/server/sql_parser.y
Branch: sciql
Log Message:
Merge with default branch.
diffs (truncated from 8895 to 300 lines):
diff --git a/de-bootstrap b/de-bootstrap
--- a/de-bootstrap
+++ b/de-bootstrap
@@ -28,7 +28,7 @@
exit
fi
-function RM () {
+RM() {
if [ "x$1" = 'x-f' ]; then
shift
fi
diff --git a/monetdb5/mal/mal_interpreter.mx b/monetdb5/mal/mal_interpreter.mx
--- a/monetdb5/mal/mal_interpreter.mx
+++ b/monetdb5/mal/mal_interpreter.mx
@@ -1114,7 +1114,7 @@
static void
DFLOWinit(DataFlow flow, Client cntxt, MalBlkPtr mb, MalStkPtr stk, int size)
{
- int pc, i, j, k, n, etop = 0;
+ int pc, i, j, k, l, n, etop = 0;
int *assign;
InstrPtr p;
@@ -1135,6 +1135,7 @@
/* administer flow dependencies */
for (j = p->retc; j < p->argc; j++) {
+ /* list of instructions that wake n-th instruction up */
if (!isVarConstant(mb, getArg(p, j)) && (k =
assign[getArg(p, j)])) {
/* add edge to the target instruction for
wakeup call */
k -= flow->start;
@@ -1155,28 +1156,31 @@
flow->status[n].blocks++;
}
- /* be careful, watch out for garbage collection
interference */
- /* those should be scheduled after all its other uses */
- k = getEndOfLife(mb, getArg(p, j));
- if (k != pc && k < flow->stop && k > flow->start) {
- /* add edge to the target instruction for
wakeup call */
- PARDEBUG mnstr_printf(GDKstdout, "forward %d ->
%d\n", n + flow->start, k);
- k -= flow->start;
- if (flow->nodes[n]) {
- /* add wakeup to tail of list */
- for (i = n; flow->edges[i] > 0; i =
flow->edges[i])
- ;
- flow->nodes[etop] = k;
- flow->edges[etop] = -1;
- flow->edges[i] = etop;
- etop++;
- (void)size;
- assert(etop < size);
- } else {
- flow->nodes[n] = k;
- flow->edges[n] = -1;
+
+ /* list of instructions to be woken up explicitly */
+ if ( !isVarConstant(mb, getArg(p, j)) ) {
+ /* be careful, watch out for garbage collection
interference */
+ /* those should be scheduled after all its
other uses */
+ l = getEndOfLife(mb, getArg(p, j));
+ if (l != pc && l < flow->stop && l >
flow->start) {
+ /* add edge to the target instruction
for wakeup call */
+ PARDEBUG mnstr_printf(GDKstdout,
"endoflife for %s is %d -> %d\n", getVarName(mb, getArg(p,j)), n + flow->start,
l);
+ l -= flow->start;
+ if (flow->nodes[n]) {
+ /* add wakeup to tail of list */
+ for (i = n; flow->edges[i] > 0;
i = flow->edges[i])
+ ;
+ flow->nodes[etop] = l;
+ flow->edges[etop] = -1;
+ flow->edges[i] = etop;
+ etop++;
+ assert(etop < size);
+ } else {
+ flow->nodes[n] = l;
+ flow->edges[n] = -1;
+ }
+ flow->status[l].blocks++;
}
- flow->status[k].blocks++;
}
}
@@ -1184,10 +1188,11 @@
assign[getArg(p, j)] = pc; /* ensure recognition of
dependency on first instruction and constant */
}
GDKfree(assign);
- PARDEBUG for (n = 0; n < flow->stop - flow->start; n++) {
+ PARDEBUG
+ for (n = 0; n < flow->stop - flow->start; n++) {
mnstr_printf(GDKstdout, "#[%d] %d: ", flow->start + n, n);
printInstruction(GDKstdout, mb, 0, getInstrPtr(mb, n +
flow->start), LIST_MAL_STMT | LIST_MAPI);
- mnstr_printf(GDKstdout, "#[%d]Dependents blocks %d:",
flow->start + n, flow->status[n].blocks);
+ mnstr_printf(GDKstdout, "#[%d]Dependents block count %d
wakeup", flow->start + n, flow->status[n].blocks);
for (j = n; flow->edges[j]; j = flow->edges[j]) {
mnstr_printf(GDKstdout, "%d ", flow->start +
flow->nodes[j]);
if (flow->edges[j] == -1)
@@ -1206,15 +1211,31 @@
They take effect after we have ensured that the basic properties for
execution hold.
@c
+static void showFlowStatus(DataFlow flow, int pc)
+{
+ int i;
+ FlowStatus fs= flow->status;
+
+ mnstr_printf(GDKstdout, "#end of data flow %d done %d \n", pc,
flow->stop - flow->start);
+ for (i = 0; i < flow->stop - flow->start; i++)
+ if (fs[i].state != DFLOWwrapup && fs[i].pc >= 0) {
+ mnstr_printf(GDKstdout, "#missed pc %d status
%d %d blocks %d", fs[i].state, i, fs[i].pc, fs[i].blocks);
+ printInstruction(GDKstdout, fs[i].mb, 0,
getInstrPtr(fs[i].mb, fs[i].pc), LIST_MAL_STMT | LIST_MAPI);
+ }
+}
+
static str
DFLOWscheduler(DataFlow flow)
{
int queued = 0, oldq = 0, last;
- int pc = 0, i, j;
+ int i,pc = 0;
+#ifdef USE_DFLOW_ADMISSION
+ int j;
+ InstrPtr p;
+#endif
int todo = flow->stop - flow->start;
str ret = MAL_SUCCEED;
FlowStatus fs, f = 0;
- InstrPtr p;
if (todo == 0)
throw(MAL, "dataflow", "Empty dataflow block");
@@ -1233,9 +1254,11 @@
MT_lock_set(&flow->todo->l, "q_enqueue");
for (i = 0; i < todo; i++)
if (flow->status[i].blocks == 0) {
- p = getInstrPtr(fs[0].mb, i);
+#ifdef USE_DFLOW_ADMISSION
+ p = getInstrPtr(fs[0].mb, flow->start + i );
for (j = p->retc; j < p->argc; j++)
flow->status[i].argclaim +=
getMemoryClaim(flow->status[0].mb, flow->status[0].stk, p, j, FALSE);
+#endif
queued++;
flow->status[i].state = DFLOWrunning;
PARDEBUG mnstr_printf(GDKstdout, "#enqueue pc=%d
claim=%d queue %d\n", flow->status[i].pc, flow->status[i].argclaim, queued);
@@ -1247,10 +1270,11 @@
/* consume the remainder */
PARDEBUG mnstr_printf(GDKstdout, "#run %d instructions in dataflow
block\n", todo);
- while (queued) {
- PARDEBUG mnstr_printf(GDKstdout, "#waiting for results, queued
%d\n", queued);
+ while (queued || todo > 0 ) {
+ PARDEBUG mnstr_printf(GDKstdout, "#waiting for results, queued
%d todo %d\n", queued, todo);
f = q_dequeue(flow->done);
queued--;
+ todo = todo > 0 ? todo -1: 0;
if (f->pc < 0) {
PARDEBUG mnstr_printf(GDKstdout, "#errors encountered
%s ", f->error ? f->error : "unknown");
@@ -1268,6 +1292,8 @@
ret = z;
}
}
+ /* first error terminates the batch a.s.a.p. */
+ todo = 0;
}
/*
@@ -1285,6 +1311,7 @@
MT_lock_set(&flow->todo->l, "q_enqueue");
oldq = queued;
+ if ( f->pc > 0 )
for (; last >= 0 && (i = flow->nodes[last]) > 0; last =
flow->edges[last])
if (flow->status[i].state == DFLOWpending) {
flow->status[i].argclaim += f->hotclaim;
@@ -1313,14 +1340,7 @@
while (oldq++ < queued)
MT_sema_up(&flow->todo->s, "q_enqueue");
}
- PARDEBUG {
- mnstr_printf(GDKstdout, "#end of data flow %d todo %d \n", pc,
flow->stop - flow->start);
- for (i = 0; i < flow->stop - flow->start; i++)
- if (fs[i].state != DFLOWwrapup && fs[i].pc >= 0) {
- mnstr_printf(GDKstdout, "#missed %d %d %d ", i,
fs[i].state, fs[i].pc);
- printInstruction(GDKstdout, fs[i].mb, 0,
getInstrPtr(fs[i].mb, fs[i].pc), LIST_MAL_STMT | LIST_MAPI);
- }
- }
+ PARDEBUG showFlowStatus(flow,pc);
return ret;
}
diff --git a/monetdb5/optimizer/Makefile.ag b/monetdb5/optimizer/Makefile.ag
--- a/monetdb5/optimizer/Makefile.ag
+++ b/monetdb5/optimizer/Makefile.ag
@@ -38,7 +38,6 @@
opt_constants.mx \
opt_costModel.mx \
opt_crack.mx \
- opt_datacell.mx \
opt_datacyclotron.mx \
opt_dataflow.mx \
opt_deadcode.mx \
@@ -92,7 +91,7 @@
opt_singleton.mx opt_costModel.mx opt_reduce.mx opt_macro.mx \
opt_accumulators.mx opt_qep.mx opt_mergetable.mx \
opt_remoteQueries.mx opt_joinselect.mx opt_partitions.mx \
- opt_datacell.mx opt_reorder.mx opt_prejoin.mx
opt_compression.mx \
+ opt_reorder.mx opt_prejoin.mx opt_compression.mx \
opt_evaluate.mx opt_inline.mx opt_pushranges.mx
opt_derivepath.mx \
opt_accessmode.mx opt_joinpath.mx opt_heuristics.mx
opt_remap.mx \
opt_statistics.mx opt_trace.mx opt_recycler.mx opt_dataflow.mx
\
diff --git a/monetdb5/optimizer/opt_accumulators.mx
b/monetdb5/optimizer/opt_accumulators.mx
--- a/monetdb5/optimizer/opt_accumulators.mx
+++ b/monetdb5/optimizer/opt_accumulators.mx
@@ -33,7 +33,7 @@
If variable t2 is a temporary variable and not used any further in
the program block, we can re-use its storage space.
@example
- t3:= batcalc.*(t2,64,t2);
+ t3:= batcalc.*(t2,64,true,false);
t4:= batcalc.+(t1,t3);
@end example
The implementation is straight forward. It only deals with the
@@ -45,8 +45,13 @@
or represent a view over a persistent BAT.
[NOTE the accumulator optimizer is known to produce
-problems due to concurrent access to BATs.
-It has been disabled]
+problems due to concurrent access to the BATs.
+However, the last instruction for a BAT is now scheduled only
+when all other uses have finished.
+
+The accumulator can be installed just before garbage collector,
+because the other modules do not recognize batcalc operations with
+more arguments.]
@{
@mal
pattern optimizer.accumulators():str
diff --git a/monetdb5/optimizer/opt_commonTerms.mx
b/monetdb5/optimizer/opt_commonTerms.mx
--- a/monetdb5/optimizer/opt_commonTerms.mx
+++ b/monetdb5/optimizer/opt_commonTerms.mx
@@ -175,7 +175,7 @@
Like all optimizer decisions, it is safe to stop.
@c
barrier |= getFunctionId(p) == assertRef;
- if (p->token == NOOPsymbol || p->token == ASSIGNsymbol ||
barrier || p->retc == p->argc) {
+ if (p->token == NOOPsymbol || p->token == ASSIGNsymbol ||
barrier /* || p->retc == p->argc */) {
#ifdef DEBUG_OPT_COMMONTERMS_MORE
mnstr_printf(cntxt->fdout, "COMMON SKIPPED[%d]
%d %d\n",i, barrier, p->retc == p->argc);
#endif
@@ -184,7 +184,7 @@
/* from here we have a candidate to look for a match */
#ifdef DEBUG_OPT_COMMONTERMS_MORE
- mnstr_printf(cntxt->fdout,"#CANDIDATE[%d] last= %d ",i,last);
+ mnstr_printf(cntxt->fdout,"#CANDIDATE[%d] ",i);
printInstruction(cntxt->fdout, mb, 0, p, LIST_MAL_ALL);
#endif
prop = hasSideEffects(p,TRUE) || isUpdateInstruction(p);
@@ -192,9 +192,9 @@
for (j = candidate; j ; j = list[j])
if ( (q=getInstrPtr(mb,j))->fcn == p->fcn &&
!isUnsafeFunction(q)){
#ifdef DEBUG_OPT_COMMONTERMS_MORE
- mnstr_printf(cntxt->fdout,"#CANDIDATE %d, %d %d %d
lookback %d ", i, j,
+ mnstr_printf(cntxt->fdout,"#CANDIDATE %d, %d %d %d ",
i, j,
hasSameSignature(mb, p, q, p->retc),
- hasSameArguments(mb, p, q), last);
+ hasSameArguments(mb, p, q));
printInstruction(cntxt->fdout, mb, 0, q,
LIST_MAL_ALL);
mnstr_printf(cntxt->fdout," :%d %d %d=%d %d %d
%d %d %d\n",
q->token != ASSIGNsymbol ,
diff --git a/monetdb5/optimizer/opt_costModel.mx
b/monetdb5/optimizer/opt_costModel.mx
--- a/monetdb5/optimizer/opt_costModel.mx
+++ b/monetdb5/optimizer/opt_costModel.mx
@@ -177,14 +177,18 @@
@:newRows2(c1+c2)@
} else if (getFunctionId(p)== kdifferenceRef) {
@:newRows2(c1==0?0:c2==0?c1: c1 - c2 < 0 ? 1 :
c1 - c2+1)@
- } else if (getFunctionId(p) == joinRef ) {
+ } else if (getFunctionId(p) == joinRef ||
+ getFunctionId(p) == leftjoinRef ||
+ getFunctionId(p) == leftjoinPathRef ) {
/* assume 1-1 joins */
@:newRows2(c1 < c2 ? c1 : c2)@
} else if (getFunctionId(p) == semijoinRef ) {
/* assume 1-1 semijoins */
@:newRows2(c1 < c2? c1 : c2)@
} else if (getFunctionId(p) == selectRef ||
- getFunctionId(p) == uselectRef) {
+ getFunctionId(p) == uselectRef ||
+ getFunctionId(p) == thetaselectRef ||
+ getFunctionId(p) == thetauselectRef) {
@:newRows1(1, c1 > 100 ? c1 / 2 +1: c1)@
} else if (getFunctionId(p) == crossRef) {
@:newRows2((log((double) c1) + log((double) c2)
> log(INT_MAX) ? INT_MAX : c1 * c2 +1))@
@@ -192,10 +196,17 @@
@:newRows1(1, c1 < 50 ? c1 : c1 / 10+1)@
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
