Changeset: 62ff93eca509 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=62ff93eca509
Added Files:
monetdb5/optimizer/opt_matpack.c
monetdb5/optimizer/opt_matpack.h
Modified Files:
monetdb5/modules/mal/mat.c
monetdb5/modules/mal/mat.h
monetdb5/modules/mal/mat.mal
monetdb5/optimizer/Makefile.ag
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_support.h
monetdb5/optimizer/opt_wrapper.c
monetdb5/optimizer/optimizer.mal
Branch: default
Log Message:
Incremental MAT pack optimizer
mat.pack behaves like a blocking operator, which it isn't.
The semantics simply requires to append the arguments to form
a new BAT.
Before dataflow optimizer steps in, we should break compound
mat.pack operations into an incremental one. This will reduce
the resource footprint and improve use of parallelism.
diffs (292 lines):
diff --git a/monetdb5/modules/mal/mat.c b/monetdb5/modules/mal/mat.c
--- a/monetdb5/modules/mal/mat.c
+++ b/monetdb5/modules/mal/mat.c
@@ -133,6 +133,55 @@ MATpackInternal(MalStkPtr stk, InstrPtr
return MAL_SUCCEED;
}
+/*
+ * Enable incremental packing. The SQL front-end requires
+ * fixed oid sequences.
+ */
+str
+MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
+{
+ int bid, *ret = (int*) getArgReference(stk,p,0);
+ BAT *b, *bb, *bn;
+
+ (void) cntxt;
+ b = BATdescriptor( bid = stk->stk[getArg(p,1)].val.ival);
+ if ( b == NULL)
+ throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING);
+ if ( bid < 0 )
+ b = BATmirror(b);
+ assert(BAThdense(b));
+
+ if ( getArgType(mb,p,2) == TYPE_int){
+ /* first step */
+ bn = BATnew(TYPE_void, b->ttype, BATcount(b) *
stk->stk[getArg(p,2)].val.ival);
+ if (bn == NULL)
+ throw(MAL, "mat.pack", MAL_MALLOC_FAIL);
+ BATsettrivprop(bn);
+ BATseqbase(bn, b->H->seq);
+ BATseqbase(BATmirror(bn), b->T->seq);
+ BATappend(bn,b,FALSE);
+ BBPreleaseref(b->batCacheid);
+ assert(!bn->H->nil || !bn->H->nonil);
+ assert(!bn->T->nil || !bn->T->nonil);
+ BBPkeepref(*ret = bn->batCacheid);
+ } else {
+ /* remaining steps */
+ bb = BATdescriptor(stk->stk[getArg(p,2)].val.ival);
+ if ( bb ){
+ if (BATcount(b) == 0)
+ BATseqbase(b, bb->H->seq);
+ if (BATcount(b) == 0)
+ BATseqbase(BATmirror(b), bb->T->seq);
+ BATappend(b,bb,FALSE);
+ }
+ assert(!b->H->nil || !b->H->nonil);
+ assert(!b->T->nil || !b->T->nonil);
+ BBPkeepref(*ret = b->batCacheid);
+ BBPreleaseref(bb->batCacheid);
+ }
+ return MAL_SUCCEED;
+}
+
static str
MATpackSliceInternal(MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
diff --git a/monetdb5/modules/mal/mat.h b/monetdb5/modules/mal/mat.h
--- a/monetdb5/modules/mal/mat.h
+++ b/monetdb5/modules/mal/mat.h
@@ -36,6 +36,7 @@
mat_export str MATpack(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
mat_export str MATpack2(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
mat_export str MATpack3(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
+mat_export str MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
mat_export str MATpackValues(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
mat_export str MATpackSlice(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
mat_export str MATnewIterator(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr p);
diff --git a/monetdb5/modules/mal/mat.mal b/monetdb5/modules/mal/mat.mal
--- a/monetdb5/modules/mal/mat.mal
+++ b/monetdb5/modules/mal/mat.mal
@@ -21,6 +21,14 @@ pattern pack3(b:bat[:oid,:any_2]...):bat
address MATpack3
comment "Materialize the MAT into a BAT by considering the heads as void.
(used in centipede)";
+pattern packIncrement(b:bat[:oid,:any_2],pieces:int):bat[:oid,:any_2]
+address MATpackIncrement
+comment "Prepare incremental mat pack";
+
+pattern packIncrement(b:bat[:oid,:any_2],c:bat[:oid,:any_2]):bat[:oid,:any_2]
+address MATpackIncrement
+comment "Prepare incremental mat pack";
+
pattern slice(first:wrd, last:wrd, b:bat[:oid,:any_2]...):bat[:oid,:any_2]
address MATpackSlice
comment "Materialize a sliced MAT into a BAT";
diff --git a/monetdb5/optimizer/Makefile.ag b/monetdb5/optimizer/Makefile.ag
--- a/monetdb5/optimizer/Makefile.ag
+++ b/monetdb5/optimizer/Makefile.ag
@@ -50,6 +50,7 @@ lib_optimizer = {
opt_joinpath.c opt_joinpath.h \
opt_macro.c opt_macro.h \
opt_mapreduce.c opt_mapreduce.h \
+ opt_matpack.c opt_matpack.h \
opt_mergetable.c opt_mergetable.h \
opt_mitosis.c opt_mitosis.h \
opt_multiplex.c opt_multiplex.h \
diff --git a/monetdb5/optimizer/opt_matpack.c b/monetdb5/optimizer/opt_matpack.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_matpack.c
@@ -0,0 +1,69 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2012 MonetDB B.V.
+ * All Rights Reserved.
+ *
+*/
+/*
+ * This simple module unrolls the mat.pack into an incremental sequence.
+ * This could speedup parallel processing and releases resources faster.
+ */
+#include "monetdb_config.h"
+#include "opt_matpack.h"
+
+int
+OPTmatpackImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci)
+{
+ int v, i, j, limit;
+ InstrPtr p,q;
+ int actions = 0;
+ InstrPtr *old;
+ char *packIncrementRef = putName("packIncrement", 13);
+
+ (void) pci;
+ (void) cntxt;
+ (void) stk; /* to fool compilers */
+ old= mb->stmt;
+ limit= mb->stop;
+ if ( newMalBlkStmt(mb,mb->stop) < 0)
+ return 0;
+ for (i = 0; i < limit; i++) {
+ p = old[i];
+ if( getModuleId(p) == matRef && getFunctionId(p) == packRef &&
isaBatType(getArgType(mb,p,1))) {
+ q = newStmt(mb, matRef, packIncrementRef);
+ v = getArg(q,0);
+ setVarType(mb,v,getArgType(mb,p,1));
+ q = pushArgument(mb, q, getArg(p,1));
+ q = pushInt(mb,q, p->argc - p->retc);
+
+ for ( j = 2; j < p->argc; j++) {
+ q = newStmt(mb,matRef, packIncrementRef);
+ q = pushArgument(mb, q, v);
+ q = pushArgument(mb, q, getArg(p,j));
+ setVarType(mb,getArg(q,0),getVarType(mb,v));
+ v = getArg(q,0);
+ }
+ getArg(q,0) = getArg(p,0);
+ freeInstruction(p);
+ continue;
+ }
+ pushInstruction(mb,p);
+ }
+ GDKfree(old);
+ DEBUGoptimizers
+ mnstr_printf(cntxt->fdout,"#opt_matpack: %d statements
marked\n", actions);
+ return actions;
+}
diff --git a/monetdb5/optimizer/opt_matpack.h b/monetdb5/optimizer/opt_matpack.h
new file mode 100644
--- /dev/null
+++ b/monetdb5/optimizer/opt_matpack.h
@@ -0,0 +1,34 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2012 MonetDB B.V.
+ * All Rights Reserved.
+*/
+#ifndef _OPT_MATPACK_
+#define _OPT_MATPACK_
+#include "opt_prelude.h"
+#include "opt_support.h"
+#include "mal_interpreter.h"
+#include "mal_instruction.h"
+#include "mal_function.h"
+
+opt_export int OPTmatpackImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr
stk, InstrPtr pci);
+
+/* #define _DEBUG_OPT_MATPACK_ */
+#undef DEBUG_OPT_MATPACK
+#define DEBUG_OPT_MATPACK 1
+#define OPTDEBUGmatpack if ( optDebug & (1 <<DEBUG_OPT_MATPACK) )
+
+#endif
diff --git a/monetdb5/optimizer/opt_pipes.c b/monetdb5/optimizer/opt_pipes.c
--- a/monetdb5/optimizer/opt_pipes.c
+++ b/monetdb5/optimizer/opt_pipes.c
@@ -70,8 +70,8 @@ struct PIPELINES {
"optimizer.emptySet();"
"optimizer.aliases();"
"optimizer.pushselect();"
- "optimizer.mitosis();"
- "optimizer.mergetable();"
+ "optimizer.mitosis();"
+ "optimizer.mergetable();"
"optimizer.deadcode();"
"optimizer.commonTerms();"
//"optimizer.groups();"
@@ -79,6 +79,7 @@ struct PIPELINES {
"optimizer.reorder();"
"optimizer.deadcode();"
"optimizer.reduce();"
+ //"optimizer.matpack();"
"optimizer.dataflow();"
"optimizer.history();"
"optimizer.multiplex();"
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -164,6 +164,7 @@ struct OPTcatalog {
{"joinPath", 0, 0, 0, DEBUG_OPT_JOINPATH},
{"macro", 0, 0, 0, DEBUG_OPT_MACRO},
{"mapreduce", 0, 0, 0, DEBUG_OPT_MAPREDUCE},
+{"matpack", 0, 0, 0, DEBUG_OPT_MATPACK},
{"mergetable", 0, 0, 0, DEBUG_OPT_MERGETABLE},
{"mitosis", 0, 0, 0, DEBUG_OPT_MITOSIS},
{"multiplex", 0, 0, 0, DEBUG_OPT_MULTIPLEX},
diff --git a/monetdb5/optimizer/opt_support.h b/monetdb5/optimizer/opt_support.h
--- a/monetdb5/optimizer/opt_support.h
+++ b/monetdb5/optimizer/opt_support.h
@@ -62,6 +62,7 @@
#define DEBUG_OPT_INLINE 20
#define DEBUG_OPT_JOINPATH 21
#define DEBUG_OPT_MACRO 23
+#define DEBUG_OPT_MATPACK 53
#define DEBUG_OPT_MERGETABLE 24
#define DEBUG_OPT_OCTOPUS 25
#define DEBUG_OPT_ORIGIN 52
diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -58,6 +58,7 @@ All Rights Reserved.
#include "opt_inline.h"
#include "opt_joinpath.h"
#include "opt_mapreduce.h"
+#include "opt_matpack.h"
#include "opt_mergetable.h"
#include "opt_mitosis.h"
#include "opt_multiplex.h"
@@ -99,6 +100,7 @@ struct{
{"inline", &OPTinlineImplementation},
{"joinPath", &OPTjoinPathImplementation},
{"mapreduce", &OPTmapreduceImplementation},
+ {"matpack", &OPTmatpackImplementation},
{"mergetable", &OPTmergetableImplementation},
{"mitosis", &OPTmitosisImplementation},
{"multiplex", &OPTmultiplexImplementation},
diff --git a/monetdb5/optimizer/optimizer.mal b/monetdb5/optimizer/optimizer.mal
--- a/monetdb5/optimizer/optimizer.mal
+++ b/monetdb5/optimizer/optimizer.mal
@@ -358,6 +358,13 @@ pattern optimizer.multiplex(mod:str,fcn:
address OPTwrapper
comment "Compiler for multiplexed instructions.";
+#opt_matpack
+pattern optimizer.matpack():str
+address OPTwrapper;
+pattern optimizer.matpack(mod:str, fcn:str):str
+address OPTwrapper
+comment "Unroll the mat.pack operation";
+
#opt_octopus.mal
module octopus;
pattern optimizer.octopus():str
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
