Changeset: 521416a9ac2e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=521416a9ac2e
Added Files:
monetdb5/modules/mal/Tests/manifold.mal
monetdb5/modules/mal/Tests/manifold.stable.err
monetdb5/modules/mal/Tests/manifold.stable.out
monetdb5/modules/mal/manifold.c
monetdb5/modules/mal/manifold.h
monetdb5/modules/mal/manifold.mal
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/exports.stable.out
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/Tests/All
monetdb5/modules/mal/mal_init.mal
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_remap.c
monetdb5/optimizer/opt_remap.h
monetdb5/optimizer/optimizer.mal
Branch: default
Log Message:
Manifold implementation
A multiplex operation leads to a MAL loop when the underlying
BAT version is not available.
This is quite expensive and interferes with the dataflow optimizer.
The manifold function partly rectifies this situation by simulating
the BAT loop, provided the underlying MAL command returns a single BAT
and the argument types are fixed width.
Simulation on mmath.sin() with 1M elm shows the following:
vectorise: 52ms
manifold : 70ms
multiplex:1469ms
diffs (truncated from 747 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -39939,10 +39939,6 @@ command lsst.xmatch(l:bat[:oid,:lng],r:b
address LSSTxmatch;
comment Return the HtmID pairs that lie within the same triangle at level depth
-pattern mal.multiplex(a:any...):any
-address OPTremapMultiplex;
-pattern mal.multiplex(a:any...):any...
-address OPTremapMultiplex;
command mapreduce.getCloud(nme:str):bat[:oid,:str]
address MRgetCloud;
comment Localize the elements of a named cloud
@@ -40251,6 +40247,10 @@ pattern mat.slice(first:wrd,last:wrd,b:b
address MATpackSlice;
comment Materialize a sliced MAT into a BAT
+pattern mal.manifold(mod:str,fcn:str,a:any...):bat[:oid,:any]
+address MANIFOLDevaluate;
+pattern mal.multiplex(mod:str,fcn:str,a:any...):any...
+address OPTremapMultiplex;
command mbr.#cmp():int
address mbrCOMP;
command mbr.#fromstr():mbr
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1532,6 +1532,8 @@ str MALpipeline(Client c);
str MALreader(Client c);
void MALresourceFairness(lng usec);
str MALstartDataflow(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+str MANIFOLDevaluate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+MALfcn MANIFOLDtypecheck(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
str MANUALcompletion(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str MANUALcreate0(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str MANUALcreate1(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
@@ -2021,6 +2023,9 @@ str QOTshowPlan(Client cntxt, MalBlkPtr
void QOTstatisticsExit(void);
void QOTupdateStatistics(str nme, int prop, lng val);
QueryQueue QRYqueue;
+str RAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+str RAPIpostlude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+str RAPIprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str RECYCLEappendSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str RECYCLEcolumn(Client cntxt, str sch, str tbl, str col);
str RECYCLEdeleteSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
@@ -2056,6 +2061,7 @@ str RUNcostPrediction(Client cntxt, MalB
str RUNisolation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str RUNpickResult(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str RUNvolumeCost(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
+char *R_HomeDir();
str SABAOTHdeserialise(sabdb **ret, str *sabdb);
str SABAOTHfreeStatus(sabdb **ret);
str SABAOTHgetDBfarm(str *ret);
diff --git a/monetdb5/modules/mal/Makefile.ag b/monetdb5/modules/mal/Makefile.ag
--- a/monetdb5/modules/mal/Makefile.ag
+++ b/monetdb5/modules/mal/Makefile.ag
@@ -47,6 +47,7 @@ lib_mal = {
mat.c mat.h \
mdb.c mdb.h \
mkey.c mkey.h \
+ manifold.c manifold.h \
pcre.c \
pqueue.c pqueue.h \
profiler.c profiler.h \
@@ -72,7 +73,7 @@ headers_mal = {
DIR = libdir/monetdb5
SOURCES = language.mal constraints.mal mal_init.mal box.mal bbp.mal \
profiler.mal const.mal batExtensions.mal \
- inspect.mal manual.mal mal_io.mal pqueue.mal mkey.mal \
+ inspect.mal manual.mal mal_io.mal pqueue.mal mkey.mal
manifold.mal \
iterator.mal clients.mal \
factories.mal groupby.mal mdb.mal pcre.mal mat.mal \
urlbox.mal transaction.mal \
@@ -84,7 +85,7 @@ headers_mal = {
}
EXTRA_DIST = batExtensions.mal iterator.mal constraints.mal \
- groupby.mal mal_init.mal manual.mal mkey.mal pcre.mal \
+ groupby.mal mal_init.mal manual.mal mkey.mal manifold.mal pcre.mal \
profiler.mal recycle.mal remote.mal sabaoth.mal trader.mal \
transaction.mal txtsim.mal tablet.mal tablet.h sample.mal json_util.mal
\
mal_mapi.mal mat.mal tokenizer.mal pqueue.mal calc.mal \
diff --git a/monetdb5/modules/mal/Tests/All b/monetdb5/modules/mal/Tests/All
--- a/monetdb5/modules/mal/Tests/All
+++ b/monetdb5/modules/mal/Tests/All
@@ -69,6 +69,8 @@ cluster00
tokenizer00
zorder
+manifold
+
#HAVE_RAPTOR?rdf
# might show different output if openssl is compiled without full sha2
diff --git a/monetdb5/modules/mal/Tests/manifold.mal
b/monetdb5/modules/mal/Tests/manifold.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mal/Tests/manifold.mal
@@ -0,0 +1,48 @@
+# the multiplex code blocks.
+
+t0:= alarm.usec();
+b:= bat.new(:oid,:dbl);
+
+INT_MAX := 2147483647;
+dbgmsk_restore := mdb.getDebug();
+dbgmsk_unset := 8+8388608;
+dbgmsk_keep := calc.xor(INT_MAX,dbgmsk_unset);
+dbgmsk_set := calc.and(dbgmsk_restore,dbgmsk_keep);
+mdb.setDebug(dbgmsk_set);
+
+r:= mmath.srand(0);
+barrier i:= 0:lng;
+ k:= mmath.rand();
+ l:= calc.dbl(k);
+ bat.append(b,l);
+ redo i:= iterator.next(1:lng,1000000:lng);
+exit i;
+
+mdb.setDebug(dbgmsk_restore);
+mdb.setDebug(dbgmsk_set);
+
+t0:= alarm.usec();
+z:bat[:oid,:dbl]:= mal.manifold("mmath","sin",b);
+t1:= alarm.usec();
+d0:= t1-t0;
+io.printf("#mal.manifold(sin) %d\n",d0);
+s:= aggr.sum(z);
+io.printf("#sum %f\n",s);
+
+
+t0:= alarm.usec();
+x:bat[:oid,:dbl]:= mal.multiplex("mmath","sin",b);
+t1:= alarm.usec();
+d0:= t1-t0;
+io.printf("#mal.multiplex(sin) %d\n",d0);
+s:= aggr.sum(x);
+io.printf("#sum %f\n",s);
+
+
+t0:= alarm.usec();
+x:= batmmath.sin(b);
+t1:= alarm.usec();
+d0:= t1-t0;
+io.printf("#mmath.sin() %d\n",d0);
+s:= aggr.sum(x);
+io.printf("#sum %f\n",s);
diff --git a/monetdb5/modules/mal/Tests/manifold.stable.err
b/monetdb5/modules/mal/Tests/manifold.stable.err
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mal/Tests/manifold.stable.err
@@ -0,0 +1,29 @@
+stderr of test 'manifold` in directory 'monetdb5/modules/mal` itself:
+
+
+# 23:08:15 >
+# 23:08:15 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=34659" "--set"
"mapi_usock=/var/tmp/mtest-26669/.s.monetdb.34659" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/export/scratch1/mk/current//Linux/var/MonetDB/mTests_monetdb5_modules_mal"
"manifold.mal"
+# 23:08:15 >
+
+# builtin opt gdk_dbpath =
/export/scratch1/mk/current//Linux/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = yes
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 34659
+# cmdline opt mapi_usock = /var/tmp/mtest-26669/.s.monetdb.34659
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbpath =
/export/scratch1/mk/current//Linux/var/MonetDB/mTests_monetdb5_modules_mal
+
+# 23:08:16 >
+# 23:08:16 > "Done."
+# 23:08:16 >
+
diff --git a/monetdb5/modules/mal/Tests/manifold.stable.out
b/monetdb5/modules/mal/Tests/manifold.stable.out
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mal/Tests/manifold.stable.out
@@ -0,0 +1,76 @@
+stdout of test 'manifold` in directory 'monetdb5/modules/mal` itself:
+
+
+# 23:08:15 >
+# 23:08:15 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=34659" "--set"
"mapi_usock=/var/tmp/mtest-26669/.s.monetdb.34659" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/export/scratch1/mk/current//Linux/var/MonetDB/mTests_monetdb5_modules_mal"
"manifold.mal"
+# 23:08:15 >
+
+# MonetDB 5 server v11.18.0
+# This is an unreleased version
+# Serving database 'mTests_monetdb5_modules_mal', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically
linked
+# Found 15.591 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2013 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://vienna.ins.cwi.nl:34659/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-26669/.s.monetdb.34659
+# MonetDB/GIS module loaded
+# MonetDB/JAQL module loaded
+# MonetDB/SQL module loaded
+#WARNING To speedup mmath.sin a bulk operator implementation is needed
+function user.main():void;
+# the multiplex code blocks.
+ t0 := alarm.usec();
+ b := bat.new(:oid,:dbl);
+ INT_MAX := 2147483647;
+ dbgmsk_restore := mdb.getDebug();
+ dbgmsk_unset := calc.+(8,8388608);
+ dbgmsk_keep := calc.xor(INT_MAX,dbgmsk_unset);
+ dbgmsk_set := calc.and(dbgmsk_restore,dbgmsk_keep);
+ mdb.setDebug(dbgmsk_set);
+ r := mmath.srand(0);
+barrier i := 0:lng;
+ k := mmath.rand();
+ l := calc.dbl(k);
+ bat.append(b,l);
+ redo i := iterator.next(1:lng,1000000:lng);
+exit i;
+ mdb.setDebug(dbgmsk_restore);
+ mdb.setDebug(dbgmsk_set);
+ t0 := alarm.usec();
+ z:bat[:oid,:dbl] := mal.manifold("mmath","sin",b);
+ t1 := alarm.usec();
+ d0 := calc.-(t1,t0);
+ io.printf("#mal.manifold(sin) %d\n",d0);
+ s := aggr.sum(z);
+ io.printf("#sum %f\n",s);
+ t0 := alarm.usec();
+ X_43 := bat.reverse(b);
+ X_45 := bat.new(nil:oid,nil:dbl);
+barrier (X_49,X_50) := iterator.new(b);
+ X_52 := mmath.sin(X_50);
+ X_53 := algebra.fetch(X_43,X_49);
+ bat.insert(X_45,X_53,X_52);
+ redo (X_49,X_50) := iterator.next(b);
+exit (X_49,X_50);
+ x:bat[:oid,:dbl] := X_45;
+ t1 := alarm.usec();
+ d0 := calc.-(t1,t0);
+ io.printf("#mal.multiplex(sin) %d\n",d0);
+ s := aggr.sum(x);
+ io.printf("#sum %f\n",s);
+ t0 := alarm.usec();
+ x:bat[:oid,:dbl] := batmmath.sin(b);
+ t1 := alarm.usec();
+ d0 := calc.-(t1,t0);
+ io.printf("#mmath.sin() %d\n",d0);
+ s := aggr.sum(x);
+ io.printf("#sum %f\n",s);
+end main;
+
+
+# 23:08:16 >
+# 23:08:16 > "Done."
+# 23:08:16 >
+
diff --git a/monetdb5/modules/mal/mal_init.mal
b/monetdb5/modules/mal/mal_init.mal
--- a/monetdb5/modules/mal/mal_init.mal
+++ b/monetdb5/modules/mal/mal_init.mal
@@ -85,6 +85,7 @@ include bbp;
include const;
include box;
include mal_io;
+include manifold;
include factories;
include recycle;
include remote;
diff --git a/monetdb5/modules/mal/manifold.c b/monetdb5/modules/mal/manifold.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/modules/mal/manifold.c
@@ -0,0 +1,270 @@
+/*
+ * The contents of this file are subject to the MonetDB Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.monetdb.org/Legal/MonetDBLicense
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is the MonetDB Database System.
+ *
+ * The Initial Developer of the Original Code is CWI.
+ * Portions created by CWI are Copyright (C) 1997-July 2008 CWI.
+ * Copyright August 2008-2013 MonetDB B.V.
+ * All Rights Reserved.
+ */
+
+/*
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
