Changeset: 3c3df2363a78 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3c3df2363a78
Added Files:
monetdb5/optimizer/opt_partition.c
monetdb5/optimizer/opt_partition.h
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
monetdb5/optimizer/Makefile.ag
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_wrapper.c
monetdb5/optimizer/optimizer.mal
Branch: partition
Log Message:
Add empty optimizer framework
diffs (290 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8166,6 +8166,7 @@ Ready.
[ "optimizer", "deadcode", "pattern optimizer.deadcode():str ",
"OPTwrapper;", "" ]
[ "optimizer", "deadcode", "pattern optimizer.deadcode(mod:str,
fcn:str):str ", "OPTwrapper;", "Dead code optimizer" ]
[ "optimizer", "default_pipe", "function optimizer.default_pipe():void;",
"", "" ]
+[ "optimizer", "default_pipe", "function optimizer.default_pipe():void;",
"", "" ]
[ "optimizer", "emptybind", "pattern optimizer.emptybind():str ",
"OPTwrapper;", "" ]
[ "optimizer", "emptybind", "pattern optimizer.emptybind(mod:str,
fcn:str):str ", "OPTwrapper;", "Evaluate empty set expressions." ]
[ "optimizer", "evaluate", "pattern optimizer.evaluate():str ",
"OPTwrapper;", "" ]
@@ -8197,6 +8198,8 @@ Ready.
[ "optimizer", "optimize", "pattern optimizer.optimize(mod:str,
fcn:str):void ", "QOToptimize;", "Optimize a specific operation" ]
[ "optimizer", "orcam", "pattern optimizer.orcam(mod:str, fcn:str,
targetmod:str, targetfcn:str):void ", "OPTorcam;", "Inverse macro,
find pattern and replace with a function call." ]
[ "optimizer", "orcam", "pattern optimizer.orcam(targetmod:str,
targetfcn:str):void ", "OPTorcam;", "Inverse macro processor for current
function" ]
+[ "optimizer", "partition", "pattern optimizer.partition():str ",
"OPTwrapper;", "" ]
+[ "optimizer", "partition", "pattern optimizer.partition(mod:str,
fcn:str):str ", "OPTwrapper;", "Partition a column for aggregates" ]
[ "optimizer", "postfix", "pattern optimizer.postfix():str ",
"OPTwrapper;", "" ]
[ "optimizer", "postfix", "pattern optimizer.postfix(mod:str,
fcn:str):str ", "OPTwrapper;", "Postfix the plan,e.g. pushing projections"
]
[ "optimizer", "prelude", "pattern optimizer.prelude():void ",
"optimizer_prelude;", "Initialize the optimizer" ]
@@ -8655,6 +8658,7 @@ Ready.
[ "user", "main", "function user.main():void;", "", "" ]
[ "user", "main", "function user.main():void;", "", "" ]
[ "user", "main", "function user.main():void;", "", "" ]
+[ "user", "main", "function user.main():void;", "", "" ]
[ "user", "s2_1", "function user.s2_1():void;", "", "" ]
[ "user", "sql.init", "function user.sql.init():void;", "",
"" ]
[ "uuid", "#cmp", "command uuid.#cmp():void ", "UUIDcompare;", ""
]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -10528,6 +10528,7 @@ Ready.
[ "optimizer", "deadcode", "pattern optimizer.deadcode():str ",
"OPTwrapper;", "" ]
[ "optimizer", "deadcode", "pattern optimizer.deadcode(mod:str,
fcn:str):str ", "OPTwrapper;", "Dead code optimizer" ]
[ "optimizer", "default_pipe", "function optimizer.default_pipe():void;",
"", "" ]
+[ "optimizer", "default_pipe", "function optimizer.default_pipe():void;",
"", "" ]
[ "optimizer", "emptybind", "pattern optimizer.emptybind():str ",
"OPTwrapper;", "" ]
[ "optimizer", "emptybind", "pattern optimizer.emptybind(mod:str,
fcn:str):str ", "OPTwrapper;", "Evaluate empty set expressions." ]
[ "optimizer", "evaluate", "pattern optimizer.evaluate():str ",
"OPTwrapper;", "" ]
@@ -10559,6 +10560,8 @@ Ready.
[ "optimizer", "optimize", "pattern optimizer.optimize(mod:str,
fcn:str):void ", "QOToptimize;", "Optimize a specific operation" ]
[ "optimizer", "orcam", "pattern optimizer.orcam(mod:str, fcn:str,
targetmod:str, targetfcn:str):void ", "OPTorcam;", "Inverse macro,
find pattern and replace with a function call." ]
[ "optimizer", "orcam", "pattern optimizer.orcam(targetmod:str,
targetfcn:str):void ", "OPTorcam;", "Inverse macro processor for current
function" ]
+[ "optimizer", "partition", "pattern optimizer.partition():str ",
"OPTwrapper;", "" ]
+[ "optimizer", "partition", "pattern optimizer.partition(mod:str,
fcn:str):str ", "OPTwrapper;", "Partition a column for aggregates" ]
[ "optimizer", "postfix", "pattern optimizer.postfix():str ",
"OPTwrapper;", "" ]
[ "optimizer", "postfix", "pattern optimizer.postfix(mod:str,
fcn:str):str ", "OPTwrapper;", "Postfix the plan,e.g. pushing projections"
]
[ "optimizer", "prelude", "pattern optimizer.prelude():void ",
"optimizer_prelude;", "Initialize the optimizer" ]
@@ -11020,6 +11023,7 @@ Ready.
[ "user", "main", "function user.main():void;", "", "" ]
[ "user", "main", "function user.main():void;", "", "" ]
[ "user", "main", "function user.main():void;", "", "" ]
+[ "user", "main", "function user.main():void;", "", "" ]
[ "user", "s2_1", "function user.s2_1():void;", "", "" ]
[ "user", "sql.init", "function user.sql.init():void;", "",
"" ]
[ "uuid", "#cmp", "command uuid.#cmp():void ", "UUIDcompare;", ""
]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1582,6 +1582,7 @@ str OPTmultiplexSimple(Client cntxt, Mal
str OPToltpImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
str OPTorcam(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str OPTorcamImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
p);
+str OPTpartitionImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
str OPTpostfixImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
str OPTprofilerImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
str OPTprojectionpathImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
diff --git a/monetdb5/optimizer/Makefile.ag b/monetdb5/optimizer/Makefile.ag
--- a/monetdb5/optimizer/Makefile.ag
+++ b/monetdb5/optimizer/Makefile.ag
@@ -51,6 +51,7 @@ lib_optimizer = {
opt_support.c opt_support.h \
opt_pushselect.c opt_pushselect.h \
opt_profiler.c opt_profiler.h \
+ opt_partition.c opt_partition.h \
opt_postfix.c opt_postfix.h \
opt_volcano.c opt_volcano.h \
opt_wrapper.c
diff --git a/monetdb5/optimizer/opt_partition.c
b/monetdb5/optimizer/opt_partition.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_partition.c
@@ -0,0 +1,93 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2017 MonetDB B.V.
+ */
+
+/* example: select count(distinct userid) from hits_10m;
+ * Massage aggregate operations using value based partitioning
+ X_16 := algebra.projection(X_4, X_15);
+ (X_17, X_18, X_19) := group.groupdone(X_16);
+ X_20 := algebra.projection(X_18, X_16);
+ X_21 := aggr.count(X_20, true);
+
+ The new could should become:
+ X_16 := algebra.projection(X_4, X_15);
+ (P1,P2,P3,P4) := bat.partition(X_16)
+ (O1, G1 , S1) := group.groupdone(P1);
+ (O2, G2 , S2) := group.groupdone(P2);
+ (O3, G3 , S3) := group.groupdone(P3);
+ (O4, G4 , S4) := group.groupdone(P4);
+ X_17 := mat.pack(O1,O3,O3,O4)
+ X_18 := mat.pack(G1,G3,G3,G4)
+ X_19 := mat.pack(S1,S3,S3,S4)
+
+ Q1 := algebra.projection(G1, P1);
+ Q2 := algebra.projection(G2, P2);
+ Q3 := algebra.projection(G3, P3);
+ Q4 := algebra.projection(G4, P4);
+ X_20 := mat.pack(Q1,Q2,Q3,Q4);
+
+ X_21 := aggr.count(X_20, true);
+ */
+#include "monetdb_config.h"
+#include "mal_instruction.h"
+#include "opt_partition.h"
+
+#define isCandidateList(M,P,I) ((M)->var[getArg(P,I)].id[0]== 'C')
+
+str
+OPTpartitionImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
p)
+{
+ int i, slimit, limit, actions=0;
+ InstrPtr *old = 0;
+ lng usec = GDKusec();
+ char buf[256];
+
+ (void) stk;
+ (void) cntxt;
+
+ limit = mb->stop;
+ slimit = mb->ssize;
+ /* check for aggregates */
+ for ( i = 0; i < limit; i++){
+ p= getInstrPtr(mb,i);
+ if( getFunctionId(p) == groupdoneRef && getModuleId(p)==
groupRef && p->argc == 2){
+ actions ++;
+ }
+ }
+ if( actions == 0)
+ goto finished;
+
+ old = mb->stmt;
+ if (newMalBlkStmt(mb, mb->ssize) < 0)
+ throw(MAL,"optimizer.postfix", SQLSTATE(HY001) MAL_MALLOC_FAIL);
+
+ /* construct the new plan */
+ for ( i = 0; i < limit; i++)
+ {
+ p= old[i];
+ pushInstruction(mb,p);
+ }
+ for( ;i < slimit; i++)
+ if( old[i])
+ freeInstruction(old[i]);
+
+ /* Defense line against incorrect plans */
+ if( actions ){
+ chkTypes(cntxt->usermodule, mb, FALSE);
+ chkFlow(mb);
+ chkDeclarations(mb);
+ }
+ /* keep all actions taken as a post block comment and update statics */
+ GDKfree(old);
+finished:
+ usec= GDKusec() - usec;
+ snprintf(buf,256,"%-20s actions=%2d time=" LLFMT " usec", "postfix",
actions, usec);
+ newComment(mb,buf);
+ addtoMalBlkHistory(mb);
+
+ return MAL_SUCCEED;
+}
diff --git a/monetdb5/optimizer/opt_partition.h
b/monetdb5/optimizer/opt_partition.h
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_partition.h
@@ -0,0 +1,17 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2017 MonetDB B.V.
+ */
+
+#ifndef _OPT_PARTITION_
+#define _OPT_PARTITION_
+#include "opt_prelude.h"
+#include "opt_support.h"
+#include "mal_exception.h"
+
+mal_export str OPTpartitionImplementation(Client cntxt, MalBlkPtr mb,
MalStkPtr stk, InstrPtr p);
+
+#endif
diff --git a/monetdb5/optimizer/opt_pipes.c b/monetdb5/optimizer/opt_pipes.c
--- a/monetdb5/optimizer/opt_pipes.c
+++ b/monetdb5/optimizer/opt_pipes.c
@@ -96,6 +96,41 @@ static struct PIPELINES {
"optimizer.wlc();"
"optimizer.garbageCollector();",
"stable", NULL, NULL, 1},
+// Experiment with partition
+ {"default_pipe",
+ "optimizer.inline();"
+ "optimizer.remap();"
+ "optimizer.costModel();"
+ "optimizer.coercions();"
+ "optimizer.evaluate();"
+ "optimizer.emptybind();"
+ "optimizer.pushselect();"
+ "optimizer.aliases();"
+ "optimizer.partition();"
+ "optimizer.mitosis();"
+ "optimizer.mergetable();"
+ "optimizer.deadcode();"
+ "optimizer.aliases();"
+ "optimizer.constants();"
+ "optimizer.commonTerms();"
+ "optimizer.projectionpath();"
+ "optimizer.deadcode();"
+ "optimizer.reorder();"
+// "optimizer.reduce();" deprecated
+ "optimizer.matpack();"
+ "optimizer.dataflow();"
+ "optimizer.querylog();"
+ "optimizer.multiplex();"
+ "optimizer.generator();"
+ "optimizer.profiler();"
+ "optimizer.candidates();"
+ "optimizer.postfix();"
+ "optimizer.deadcode();"
+// "optimizer.jit();" awaiting the new batcalc api
+// "optimizer.oltp();"awaiting the autocommit front-end changes
+ "optimizer.wlc();"
+ "optimizer.garbageCollector();",
+ "stable", NULL, NULL, 1},
/*
* Volcano style execution produces a sequence of blocks from the source
relation
*/
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -50,6 +50,7 @@ struct OPTcatalog {
{"multiplex", 0, 0, 0},
{"oltp", 0, 0, 0},
{"postfix", 0, 0, 0},
+{"partition", 0, 0, 0},
{"reduce", 0, 0, 0},
{"remap", 0, 0, 0},
{"remote", 0, 0, 0},
diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -39,6 +39,7 @@
#include "opt_json.h"
#include "opt_oltp.h"
#include "opt_postfix.h"
+#include "opt_partition.h"
#include "opt_mergetable.h"
#include "opt_mitosis.h"
#include "opt_multiplex.h"
@@ -79,6 +80,7 @@ struct{
{"multiplex", &OPTmultiplexImplementation,0,0},
{"oltp", &OPToltpImplementation,0,0},
{"postfix", &OPTpostfixImplementation,0,0},
+ {"partition", &OPTpartitionImplementation,0,0},
{"profiler", &OPTprofilerImplementation,0,0},
{"projectionpath", &OPTprojectionpathImplementation,0,0},
{"pushselect", &OPTpushselectImplementation,0,0},
diff --git a/monetdb5/optimizer/optimizer.mal b/monetdb5/optimizer/optimizer.mal
--- a/monetdb5/optimizer/optimizer.mal
+++ b/monetdb5/optimizer/optimizer.mal
@@ -281,5 +281,13 @@ pattern optimizer.postfix(mod:str, fcn:s
address OPTwrapper
comment "Postfix the plan,e.g. pushing projections";
+#opt_postfix.mal
+
+pattern optimizer.partition():str
+address OPTwrapper;
+pattern optimizer.partition(mod:str, fcn:str):str
+address OPTwrapper
+comment "Partition a column for aggregates";
+
optimizer.prelude();
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
