Changeset: d1cecc2ef7bb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d1cecc2ef7bb
Added Files:
sql/server/rel_partition.c
sql/server/rel_partition.h
sql/test/BugTracker-2016/Tests/create_table_from_complex_query.Bug-3921.stable.err
sql/test/BugTracker-2016/Tests/create_table_from_complex_query.Bug-3921.stable.out
Modified Files:
monetdb5/mal/Tests/dynamicload.stable.err
monetdb5/optimizer/Tests/JPexample.stable.out
monetdb5/optimizer/Tests/projectionchain.stable.out
monetdb5/optimizer/opt_mitosis.c
monetdb5/optimizer/opt_pushselect.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_optimizer.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statement.h
sql/benchmarks/tpch/Tests/02-plan.stable.out
sql/benchmarks/tpch/Tests/17-plan.stable.out
sql/benchmarks/tpch/Tests/17-plan.stable.out.int128
sql/benchmarks/tpch/Tests/20-plan.stable.out
sql/benchmarks/tpch/Tests/20-plan.stable.out.int128
sql/benchmarks/tpch/Tests/21-plan.stable.out
sql/benchmarks/tpch/fileleak/Tests/leaks.stable.out
sql/include/sql_relation.h
sql/server/Makefile.ag
sql/server/rel_exp.c
sql/server/rel_optimizer.c
sql/server/rel_rel.c
sql/test/BugTracker-2013/Tests/correlated_select.Bug-3250.stable.out
sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out
sql/test/BugTracker-2015/Tests/ambiguous.Bug-3803.stable.out.int128
sql/test/BugTracker-2015/Tests/apply_merge_distinct.Bug-3760.stable.out
sql/test/BugTracker-2015/Tests/crash.Bug-3736.stable.out
sql/test/BugTracker-2016/Tests/DISTINCT_with_correlated_scalar_subquery_crashes_mserver.Bug-3920.stable.out
sql/test/BugTracker-2016/Tests/projectionpath-prefix.Bug.stable.out
sql/test/mergetables/Tests/mergequery.stable.out
sql/test/remote/Tests/partition_elim.stable.out
Branch: default
Log Message:
moved the selection of the table to split in mitosis to the sql layer. In the
code
generation we only add row counts for the selected table.
This solves mitosis problems with self joins
Also approved many tests most of them because of these changes, some of
older changes...
diffs (truncated from 2698 to 300 lines):
diff --git a/monetdb5/mal/Tests/dynamicload.stable.err
b/monetdb5/mal/Tests/dynamicload.stable.err
--- a/monetdb5/mal/Tests/dynamicload.stable.err
+++ b/monetdb5/mal/Tests/dynamicload.stable.err
@@ -80,7 +80,7 @@ stderr of test 'dynamicload` in director
# 13:21:38 > "mclient" "-lmal" "-ftest" "-Eutf-8"
"--host=/var/tmp/mtest-18020" "--port=37961"
# 13:21:38 >
-MAPI = (monetdb) /var/tmp/mtest-16561/.s.monetdb.38261
+MAPI = (monetdb) /var/tmp/mtest-30911/.s.monetdb.34033
QUERY = # examples of non-bindable instructions
#load unknown library
@@ -93,7 +93,7 @@ QUERY = # examples of non-bindable instr
wrong();
ERROR = !TypeException:user.main[3]:object code for command user.wrong missing
!TypeException:user.main[3]:'user.wrong' undefined in:
user.wrong();
-
+
# 13:26:24 >
# 13:26:24 > "Done."
# 13:26:24 >
diff --git a/monetdb5/optimizer/Tests/JPexample.stable.out
b/monetdb5/optimizer/Tests/JPexample.stable.out
--- a/monetdb5/optimizer/Tests/JPexample.stable.out
+++ b/monetdb5/optimizer/Tests/JPexample.stable.out
@@ -31,17 +31,7 @@ Ready.
#mdb.List("user","foo");
function user.foo():void; #[0] 0
- a:bat[:oid] := bat.new(:oid,:oid); #[1] CMDBATnew 1 <- 2 2
- b:bat[:oid] := bat.new(:oid,:oid); #[2] CMDBATnew 3 <- 2 2
- c:bat[:oid] := bat.new(:oid,:oid); #[3] CMDBATnew 4 <- 2 2
- d:bat[:oid] := bat.new(:oid,:oid); #[4] CMDBATnew 5 <- 2 2
- j1:bat[:oid] := algebra.projection(a:bat[:oid],b:bat[:oid]); #[5]
ALGprojection 6 <- 1 3
- X_11:bat[:oid] :=
algebra.projectionpath(a:bat[:oid],b:bat[:oid],c:bat[:oid]); #[6]
ALGprojectionpath 11 <- 1 3 4
- j2:bat[:oid] := X_11:bat[:oid]; #[7] 7 <- 11
- j3:bat[:any] := algebra.projectionpath(X_11:bat[:oid],d:bat[:oid]);
#[8] ALGprojectionpath 8 <- 11 5
- j4:bat[:any] := X_11:bat[:oid]; #[9] 9 <- 11
- j5:bat[:oid] := algebra.projectionpath(X_11:bat[:oid],b:bat[:oid]);
#[10] ALGprojectionpath 10 <- 11 3
-end user.foo; #[11]
+end user.foo; #[1]
# 06:41:06 >
# 06:41:06 > "Done."
diff --git a/monetdb5/optimizer/Tests/projectionchain.stable.out
b/monetdb5/optimizer/Tests/projectionchain.stable.out
--- a/monetdb5/optimizer/Tests/projectionchain.stable.out
+++ b/monetdb5/optimizer/Tests/projectionchain.stable.out
@@ -63,29 +63,17 @@ function user.qry():void;
bat.append(k,7);
bat.append(k,8);
bat.append(k,9);
- e := algebra.projection(a,b);
- X_91:bat[:oid] := algebra.projectionpath(a,b);
- X_89:bat[:oid] := algebra.projectionpath(X_91,c);
+ X_89:bat[:oid] := algebra.projectionpath(a,b,c);
f:bat[:oid] := X_89;
g := algebra.projection(f,d);
l := algebra.projection(f,k);
io.print(g);
io.print(l);
-# recognize common heads
- e1 := algebra.projection(a,b);
- X_90:bat[:oid] := algebra.projectionpath(X_91,b);
- e2:bat[:oid] := X_90;
- f1 := algebra.projection(a,b);
- f2:bat[:oid] := X_90;
- g1:bat[:str] := algebra.projectionpath(X_90,d);
- l1:bat[:int] := algebra.projectionpath(X_90,k);
-#io.print(g1);
-#io.print(l1);
h := algebra.projectionpath(c,d);
io.print(h);
i := algebra.projectionpath(b,c,d);
io.print(i);
- j := algebra.projectionpath(X_89,d);
+ j := algebra.projection(X_89,d);
io.print(j);
io.print("empty sequences");
w := algebra.projectionpath(empty,c,d);
@@ -129,29 +117,17 @@ function user.qry():void;
bat.append(k,7);
bat.append(k,8);
bat.append(k,9);
- e := algebra.projection(a,b);
- X_91:bat[:oid] := algebra.projectionpath(a,b);
- X_89:bat[:oid] := algebra.projectionpath(X_91,c);
+ X_89:bat[:oid] := algebra.projectionpath(a,b,c);
f:bat[:oid] := X_89;
g := algebra.projection(f,d);
l := algebra.projection(f,k);
io.print(g);
io.print(l);
-# recognize common heads
- e1 := algebra.projection(a,b);
- X_90:bat[:oid] := algebra.projectionpath(X_91,b);
- e2:bat[:oid] := X_90;
- f1 := algebra.projection(a,b);
- f2:bat[:oid] := X_90;
- g1:bat[:str] := algebra.projectionpath(X_90,d);
- l1:bat[:int] := algebra.projectionpath(X_90,k);
-#io.print(g1);
-#io.print(l1);
h := algebra.projectionpath(c,d);
io.print(h);
i := algebra.projectionpath(b,c,d);
io.print(i);
- j := algebra.projectionpath(X_89,d);
+ j := algebra.projection(X_89,d);
io.print(j);
io.print("empty sequences");
w := algebra.projectionpath(empty,c,d);
diff --git a/monetdb5/optimizer/opt_mitosis.c b/monetdb5/optimizer/opt_mitosis.c
--- a/monetdb5/optimizer/opt_mitosis.c
+++ b/monetdb5/optimizer/opt_mitosis.c
@@ -196,6 +196,11 @@ OPTmitosisImplementation(Client cntxt, M
pushInstruction(mb, p);
continue;
}
+ r = getRowCnt(mb, getArg(p, 0));
+ if (r < rowcnt) {
+ pushInstruction(mb, p);
+ continue;
+ }
/* Don't split the (index) bat if we already have identified a
range */
/* This will happen if we inline separately optimized routines
*/
if (p->argc > 7) {
diff --git a/monetdb5/optimizer/opt_pushselect.c
b/monetdb5/optimizer/opt_pushselect.c
--- a/monetdb5/optimizer/opt_pushselect.c
+++ b/monetdb5/optimizer/opt_pushselect.c
@@ -177,7 +177,7 @@ OPTpushselectImplementation(Client cntxt
if (getModuleId(p) == sqlRef && getFunctionId(p) == deltaRef)
push_down_delta++;
- if (getModuleId(p) == sqlRef && getFunctionId(p) == tidRef) {
/* rewrite equal table ids */
+ if (0 && getModuleId(p) == sqlRef && getFunctionId(p) ==
tidRef) { /* rewrite equal table ids */
int sname = getArg(p, 2), tname = getArg(p, 3), s;
for (s = 0; s < subselects.nr; s++) {
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -770,6 +770,8 @@ stmt_col( mvc *sql, sql_column *c, stmt
{
stmt *sc = stmt_bat(sql->sa, c, RDONLY);
+ if (del)
+ sc->partition = del->partition;
if (isTable(c->t) && c->t->access != TABLE_READONLY &&
(c->base.flag != TR_NEW || c->t->base.flag != TR_NEW /* alter */) &&
(c->t->persistence == SQL_PERSIST || c->t->persistence ==
SQL_DECLARED_TABLE) && !c->t->commit_action) {
@@ -777,6 +779,8 @@ stmt_col( mvc *sql, sql_column *c, stmt
stmt *u = stmt_bat(sql->sa, c, RD_UPD_ID);
sc = stmt_project_delta(sql->sa, sc, u, i);
sc = stmt_project(sql->sa, del, sc);
+ if (del)
+ u->partition = del->partition;
} else if (del) { /* always handle the deletes */
sc = stmt_project(sql->sa, del, sc);
}
@@ -788,6 +792,8 @@ stmt_idx( mvc *sql, sql_idx *i, stmt *de
{
stmt *sc = stmt_idxbat(sql->sa, i, RDONLY);
+ if (del)
+ sc->partition = del->partition;
if (isTable(i->t) && i->t->access != TABLE_READONLY &&
(i->base.flag != TR_NEW || i->t->base.flag != TR_NEW /* alter */) &&
(i->t->persistence == SQL_PERSIST || i->t->persistence ==
SQL_DECLARED_TABLE) && !i->t->commit_action) {
@@ -795,6 +801,8 @@ stmt_idx( mvc *sql, sql_idx *i, stmt *de
stmt *u = stmt_idxbat(sql->sa, i, RD_UPD_ID);
sc = stmt_project_delta(sql->sa, sc, u, ic);
sc = stmt_project(sql->sa, del, sc);
+ if (del)
+ u->partition = del->partition;
} else if (del) { /* always handle the deletes */
sc = stmt_project(sql->sa, del, sc);
}
@@ -1184,6 +1192,8 @@ rel2bin_basetable( mvc *sql, sql_rel *re
if (!t && c)
t = c->t;
dels = stmt_tid(sql->sa, t);
+ if (rel->flag == REL_PARTITION)
+ dels->partition = 1;
/* add aliases */
assert(rel->exps);
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -24,6 +24,7 @@
#include <sql_optimizer.h>
#include <sql_datetime.h>
#include <rel_optimizer.h>
+#include <rel_partition.h>
#include <rel_distribute.h>
#include <rel_select.h>
#include <rel_rel.h>
@@ -117,6 +118,7 @@ sql_symbol2relation(mvc *c, symbol *sym)
if (r) {
r = rel_optimizer(c, r);
r = rel_distribute(c, r);
+ r = rel_partition(c, r);
if (rel_is_point_query(r) || rel_need_distinct_query(r))
c->point_query = 1;
}
diff --git a/sql/backends/monet5/sql_gencode.c
b/sql/backends/monet5/sql_gencode.c
--- a/sql/backends/monet5/sql_gencode.c
+++ b/sql/backends/monet5/sql_gencode.c
@@ -1079,11 +1079,19 @@ static int
if (q == NULL)
return -1;
s->nr = getDestVar(q);
+ if (t && (!isRemote(t) && !isMergeTable(t)) &&
s->partition) {
+ sql_trans *tr = sql->mvc->session->tr;
+ BUN rows = (BUN) store_funcs.count_col(tr,
t->columns.set->h->data, 1);
+ setRowCnt(mb,getArg(q,0),rows);
+ if (t->p && 0)
+ setMitosisPartition(q, t->p->base.id);
+ }
}
break;
case st_bat:{
int tt = s->op4.cval->type.type->localtype;
- sql_table *t = s->op4.cval->t;
+ sql_column *c = s->op4.cval;
+ sql_table *t = c->t;
q = newStmt2(mb, sqlRef, bindRef);
if (q == NULL)
@@ -1097,7 +1105,7 @@ static int
q = pushArgument(mb, q, sql->mvc_var);
q = pushSchema(mb, q, t);
q = pushArgument(mb, q,
getStrConstant(mb,t->base.name));
- q = pushArgument(mb, q,
getStrConstant(mb,s->op4.cval->base.name));
+ q = pushArgument(mb, q,
getStrConstant(mb,c->base.name));
q = pushArgument(mb, q, getIntConstant(mb,s->flag));
if (q == NULL)
return -1;
@@ -1107,11 +1115,22 @@ static int
/* rename second result */
renameVariable(mb, getArg(q, 1), "r1_%d",
s->nr);
}
+ if (s->flag != RD_INS && s->partition) {
+ sql_trans *tr = sql->mvc->session->tr;
+
+ if (c && (!isRemote(c->t) &&
!isMergeTable(c->t))) {
+ BUN rows = (BUN)
store_funcs.count_col(tr, c, 1);
+ setRowCnt(mb,getArg(q,0),rows);
+ if (t->p && 0)
+ setMitosisPartition(q,
t->p->base.id);
+ }
+ }
}
break;
case st_idxbat:{
int tt = tail_type(s)->type->localtype;
- sql_table *t = s->op4.idxval->t;
+ sql_idx *i = s->op4.idxval;
+ sql_table *t = i->t;
q = newStmt2(mb, sqlRef, bindidxRef);
if (q == NULL)
@@ -1126,7 +1145,7 @@ static int
q = pushArgument(mb, q, sql->mvc_var);
q = pushSchema(mb, q, t);
q = pushArgument(mb, q,
getStrConstant(mb,t->base.name));
- q = pushArgument(mb, q,
getStrConstant(mb,s->op4.idxval->base.name));
+ q = pushArgument(mb, q,
getStrConstant(mb,i->base.name));
q = pushArgument(mb, q, getIntConstant(mb,s->flag));
if (q == NULL)
return -1;
@@ -1136,6 +1155,16 @@ static int
/* rename second result */
renameVariable(mb, getArg(q, 1), "r1_%d",
s->nr);
}
+ if (s->flag != RD_INS && s->partition) {
+ sql_trans *tr = sql->mvc->session->tr;
+
+ if (i && (!isRemote(i->t) &&
!isMergeTable(i->t))) {
+ BUN rows = (BUN)
store_funcs.count_idx(tr, i, 1);
+ setRowCnt(mb,getArg(q,0),rows);
+ if (t->p && 0)
+ setMitosisPartition(q,
t->p->base.id);
+ }
+ }
}
break;
case st_const:{
diff --git a/sql/backends/monet5/sql_optimizer.c
b/sql/backends/monet5/sql_optimizer.c
--- a/sql/backends/monet5/sql_optimizer.c
+++ b/sql/backends/monet5/sql_optimizer.c
@@ -30,139 +30,53 @@
#include "sql_gencode.h"
#include "opt_pipes.h"
-#define TOSMALL 10
+static lng
+SQLgetSpace(mvc *m, MalBlkPtr mb)
+{
+ sql_trans *tr = m->session->tr;
+ lng space = 0, i;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
