Changeset: bd2ec2b69993 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bd2ec2b69993
Added Files:
monetdb5/optimizer/Tests/groups00.mal
monetdb5/optimizer/Tests/groups00.stable.err
monetdb5/optimizer/Tests/groups00.stable.out
Modified Files:
monetdb5/optimizer/Makefile.ag
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_support.h
Branch: default
Log Message:
Experimental grouping optimizer
This optimizer is a step towards a new implementation where all arguments are
known.
Th immediate benefit is that we enforce locality of access to arguments, which
may
improve somewhat dataflow driven execution (avoid large intermediates to he
shuffled
to distk).
diffs (truncated from 331 to 300 lines):
diff --git a/monetdb5/optimizer/Makefile.ag b/monetdb5/optimizer/Makefile.ag
--- a/monetdb5/optimizer/Makefile.ag
+++ b/monetdb5/optimizer/Makefile.ag
@@ -41,12 +41,12 @@ lib_optimizer = {
opt_datacyclotron.mx \
opt_dataflow.mx \
opt_deadcode.mx \
- opt_derivepath.mx \
opt_dictionary.mx \
opt_emptySet.mx \
opt_evaluate.mx \
opt_factorize.mx \
opt_garbageCollector.mx \
+ opt_groups.mx \
opt_heuristics.mx \
opt_history.mx \
opt_inline.mx \
@@ -90,7 +90,7 @@ headers_mal = {
opt_accumulators.mx opt_qep.mx opt_mergetable.mx \
opt_remoteQueries.mx opt_joinselect.mx \
opt_reorder.mx opt_prejoin.mx opt_compression.mx \
- opt_evaluate.mx opt_inline.mx opt_pushranges.mx
opt_derivepath.mx \
+ opt_evaluate.mx opt_inline.mx opt_pushranges.mx opt_groups.mx \
opt_accessmode.mx opt_joinpath.mx opt_heuristics.mx
opt_remap.mx \
opt_statistics.mal opt_trace.mx opt_recycler.mx
opt_dataflow.mx \
opt_cluster.mx opt_dictionary.mx opt_mapreduce.mx opt_origin.mx
\
diff --git a/monetdb5/optimizer/Tests/groups00.mal
b/monetdb5/optimizer/Tests/groups00.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/Tests/groups00.mal
@@ -0,0 +1,26 @@
+function tst();
+ b:= bat.new(:oid,:int);
+ bat.append(b,1);
+ bat.append(b,2);
+ bat.append(b,3);
+ c:= bat.new(:oid,:str);
+ bat.append(c,"hello");
+ bat.append(c,"hello");
+ bat.append(c,"world");
+ d:= bat.new(:oid,:flt);
+ bat.append(d,3.14);
+ bat.append(d,3.14);
+ bat.append(d,3.14);
+ (ext1,grp1) := group.new(b);
+ io.print(ext1);
+ io.print(grp1);
+ (ext2,grp2) := group.derive(ext1,grp1, c);
+ io.print(ext2);
+ io.print(grp2);
+ (ext3,grp3) := group.derive(ext2,grp2, d);
+ io.print(ext3);
+ io.print(grp3);
+end tst;
+optimizer.groups("user","tst");
+mdb.list("user","tst");
+tst();
diff --git a/monetdb5/optimizer/Tests/groups00.stable.err
b/monetdb5/optimizer/Tests/groups00.stable.err
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/Tests/groups00.stable.err
@@ -0,0 +1,31 @@
+stderr of test 'groups00` in directory 'optimizer` itself:
+
+
+# 12:47:43 >
+# 12:47:43 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"gdk_dbfarm=/export/scratch1/mk/current//Linux/var/MonetDB" "--set"
"mapi_open=true" "--set" "mapi_port=39187" "--set" "monet_prompt=" "--trace"
"--forcemito" "--set" "mal_listing=2" "--dbname=mTests_optimizer" "groups00.mal"
+# 12:47:43 >
+
+# builtin opt gdk_dbname = demo
+# builtin opt gdk_dbfarm =
/export/scratch1/mk/current//Linux/var/monetdb5/dbfarm
+# builtin opt gdk_debug = 0
+# builtin opt gdk_alloc_map = no
+# builtin opt gdk_vmtrim = yes
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt gdk_dbfarm = /export/scratch1/mk/current//Linux/var/MonetDB
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 39187
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbname = mTests_optimizer
+
+# 12:47:43 >
+# 12:47:43 > "Done."
+# 12:47:43 >
+
diff --git a/monetdb5/optimizer/Tests/groups00.stable.out
b/monetdb5/optimizer/Tests/groups00.stable.out
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/Tests/groups00.stable.out
@@ -0,0 +1,115 @@
+stdout of test 'groups00` in directory 'optimizer` itself:
+
+
+# 12:47:43 >
+# 12:47:43 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"gdk_dbfarm=/export/scratch1/mk/current//Linux/var/MonetDB" "--set"
"mapi_open=true" "--set" "mapi_port=39187" "--set" "monet_prompt=" "--trace"
"--forcemito" "--set" "mal_listing=2" "--dbname=mTests_optimizer" "groups00.mal"
+# 12:47:43 >
+
+# MonetDB 5 server v11.10.0
+# This is an unreleased version
+# Serving database 'mTests_optimizer', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically
linked
+# Found 15.630 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2012 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://vienna.ins.cwi.nl:39187/
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+function user.tst():void;
+ b := bat.new(:oid,:int);
+ bat.append(b,1);
+ bat.append(b,2);
+ bat.append(b,3);
+ c := bat.new(:oid,:str);
+ bat.append(c,"hello");
+ bat.append(c,"hello");
+ bat.append(c,"world");
+ d := bat.new(:oid,:flt);
+ bat.append(d,3.1400001);
+ bat.append(d,3.1400001);
+ bat.append(d,3.1400001);
+ (ext1,grp1) := group.new(b);
+ io.print(ext1);
+ io.print(grp1);
+ (ext2,grp2) := group.derive(ext1,grp1,c);
+ io.print(ext2);
+ io.print(grp2);
+ (ext3,grp3) := group.derive(ext2,grp2,d);
+ io.print(ext3);
+ io.print(grp3);
+end tst;
+function user.main():void;
+ mdb.list("user","tst");
+ user.tst();
+end main;
+function user.tst():void;
+ b := bat.new(:oid,:int);
+ bat.append(b,1);
+ bat.append(b,2);
+ bat.append(b,3);
+ c := bat.new(:oid,:str);
+ bat.append(c,"hello");
+ bat.append(c,"hello");
+ bat.append(c,"world");
+ d := bat.new(:oid,:flt);
+ bat.append(d,3.1400001);
+ bat.append(d,3.1400001);
+ bat.append(d,3.1400001);
+ (ext1,grp1) := group.new(b);
+ io.print(ext1);
+ io.print(grp1);
+ (ext2,grp2) := group.new(b,c);
+ io.print(ext2);
+ io.print(grp2);
+ (ext3,grp3) := group.new(b,c,d);
+ io.print(ext3);
+ io.print(grp3);
+end tst;
+#-----------------#
+# h t # name
+# void wrd # type
+#-----------------#
+[ 0@0, 1 ]
+[ 1@0, 1 ]
+[ 2@0, 1 ]
+#-----------------#
+# h t # name
+# void void # type
+#-----------------#
+[ 0@0, 0@0 ]
+[ 1@0, 1@0 ]
+[ 2@0, 2@0 ]
+#-----------------#
+# h t # name
+# void wrd # type
+#-----------------#
+[ 0@0, 1 ]
+[ 1@0, 1 ]
+[ 2@0, 1 ]
+#-----------------#
+# h t # name
+# void void # type
+#-----------------#
+[ 0@0, 0@0 ]
+[ 1@0, 1@0 ]
+[ 2@0, 2@0 ]
+#-----------------#
+# h t # name
+# void wrd # type
+#-----------------#
+[ 0@0, 1 ]
+[ 1@0, 1 ]
+[ 2@0, 1 ]
+#-----------------#
+# h t # name
+# void void # type
+#-----------------#
+[ 0@0, 0@0 ]
+[ 1@0, 1@0 ]
+[ 2@0, 2@0 ]
+
+# 12:47:43 >
+# 12:47:43 > "Done."
+# 12:47:43 >
+
diff --git a/monetdb5/optimizer/opt_pipes.c b/monetdb5/optimizer/opt_pipes.c
--- a/monetdb5/optimizer/opt_pipes.c
+++ b/monetdb5/optimizer/opt_pipes.c
@@ -82,6 +82,29 @@ struct PIPELINES {
"optimizer.accumulators();"
"optimizer.garbageCollector();",
"stable", 0, 0},
+ {"groups_pipe",
+ "optimizer.inline();"
+ "optimizer.remap();"
+ "optimizer.evaluate();"
+ "optimizer.costModel();"
+ "optimizer.coercions();"
+ "optimizer.emptySet();"
+ "optimizer.aliases();"
+ "optimizer.mitosis();"
+ "optimizer.mergetable();"
+ "optimizer.deadcode();"
+ "optimizer.commonTerms();"
+ "optimizer.joinPath();"
+ "optimizer.groups();"
+ "optimizer.reorder();"
+ "optimizer.deadcode();"
+ "optimizer.reduce();"
+ "optimizer.dataflow();"
+ "optimizer.history();"
+ "optimizer.multiplex();"
+ "optimizer.accumulators();"
+ "optimizer.garbageCollector();",
+ "stable", 0, 0},
/* The no_mitosis pipe line is (and should be kept!) identical to the
* default pipeline, except that optimizer mitosis is omitted. It is
* used mainly to make some tests work deterministically, and to check
diff --git a/monetdb5/optimizer/opt_prelude.c b/monetdb5/optimizer/opt_prelude.c
--- a/monetdb5/optimizer/opt_prelude.c
+++ b/monetdb5/optimizer/opt_prelude.c
@@ -72,7 +72,6 @@ str decompressRef;
str deleteRef;
str depositRef;
str deriveRef;
-str derivePathRef;
str differenceRef;
str divRef;
str disconnectRef;
@@ -85,6 +84,7 @@ str finishRef;
str getRef;
str grabRef;
str groupRef;
+str groupsRef;
str groupbyRef;
str hashRef;
str histogramRef;
@@ -294,7 +294,6 @@ void optimizerInit(void){
deleteRef = putName("delete",6);
depositRef = putName("deposit",7);
deriveRef = putName("derive",6);
- derivePathRef = putName("derivePath",10);
differenceRef= putName("difference",10);
divRef = putName("/",1);
disconnectRef= putName("disconnect",10);
@@ -307,6 +306,7 @@ void optimizerInit(void){
getRef = putName("get",3);
grabRef = putName("grab",4);
groupRef = putName("group",5);
+ groupsRef = putName("groups",6);
groupbyRef = putName("groupby",7);
hashRef = putName("hash",4);
histogramRef = putName("histogram",9);
diff --git a/monetdb5/optimizer/opt_prelude.h b/monetdb5/optimizer/opt_prelude.h
--- a/monetdb5/optimizer/opt_prelude.h
+++ b/monetdb5/optimizer/opt_prelude.h
@@ -71,7 +71,6 @@ opt_export str dblRef;
opt_export str deleteRef;
opt_export str depositRef;
opt_export str deriveRef;
-opt_export str derivePathRef;
opt_export str differenceRef;
opt_export str divRef;
opt_export str disconnectRef;
@@ -84,6 +83,7 @@ opt_export str finishRef;
opt_export str getRef;
opt_export str grabRef;
opt_export str groupRef;
+opt_export str groupsRef;
opt_export str groupbyRef;
opt_export str hashRef;
opt_export str histogramRef;
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
