Changeset: abbdfa27a43d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=abbdfa27a43d
Modified Files:
MonetDB5/src/modules/mal/Tests/remote07.mal
MonetDB5/src/modules/mal/Tests/remote07.stable.err
MonetDB5/src/modules/mal/Tests/remote07.stable.out
MonetDB5/src/modules/mal/remote.mx
Branch: default
Log Message:
Dramatically improve remote.put speed for BATs.
remote.put now uses remote.batload as helper on the remote site to
more efficiently transfer the BAT data. Before this commit,
remote07.mal took roughly 300 usecs for the put operation, and 10 for
the get operation. After this commit, the put takes 7 usecs, the get
still 10.
The put operation is faster than the get because put doesn't use Mapi,
but plain comma-separated lines instead. The win is in the less
parsing/data overhead.
diffs (truncated from 451 to 300 lines):
diff -r 4b6c272eb1ff -r abbdfa27a43d MonetDB5/src/modules/mal/Tests/remote07.mal
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MonetDB5/src/modules/mal/Tests/remote07.mal Fri Sep 10 14:53:35
2010 +0200
@@ -0,0 +1,40 @@
+# let connect figure out itself how to connect to the running db
+uri := sabaoth.getLocalConnectionURI();
+conn:str := remote.connect(uri, "monetdb", "monetdb");
+
+d:bat[:str,:int] := bat.new(:str, :int);
+bat.insert(d, "bla", 1);
+bat.insert(d, nil:str, 2);
+bat.insert(d, "boe", nil:int);
+bat.insert(d, d); # 6
+bat.insert(d, d); # 12
+bat.insert(d, d); # 24
+bat.insert(d, d); # 48
+bat.insert(d, d); # 96
+bat.insert(d, d); # 192
+bat.insert(d, d); # 384
+bat.insert(d, d); # 768
+bat.insert(d, d); # 1536
+bat.insert(d, d); # 3072
+
+# store the values
+t1 := alarm.time();
+l:str := remote.put(conn, d);
+t2 := alarm.time();
+t3 := t2 - t1;
+io.printf("# put: %d\n", t3);
+
+# retrieve the values
+t1 := alarm.time();
+u:bat[:str,:int] := remote.get(conn, l);
+t2 := alarm.time();
+t3 := t2 - t1;
+io.printf("# get: %d\n", t3);
+
+# see if it really is here
+c := aggr.count(u);
+io.print(c);
+
+# help testweb a bit, since currently no cleanup is done on server
+# shutdown
+remote.disconnect(conn);
diff -r 4b6c272eb1ff -r abbdfa27a43d
MonetDB5/src/modules/mal/Tests/remote07.stable.err
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MonetDB5/src/modules/mal/Tests/remote07.stable.err Fri Sep 10
14:53:35 2010 +0200
@@ -0,0 +1,75 @@
+stderr of test 'remote07` in directory 'src/modules/mal` itself:
+
+
+# 10:41:05 >
+# 10:41:05 > mserver5
"--config=/ufs/fabian/scratch/monetdb/current/program-x86_64/etc/monetdb5.conf"
--debug=10 --set gdk_nr_threads=0 --set
"monet_mod_path=/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/lib:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/bin"
--set
"gdk_dbfarm=/net/volund.ins.cwi.nl/export/scratch0/fabian/vtmp/mtest-current-volund.ins.cwi.nl/five/dbfarm"
--set mapi_open=true --set xrpc_open=true --set mapi_port=30248 --set
xrpc_port=47120 --set monet_prompt= --set mal_listing=2 --trace
--dbname=mTests_src_modules_mal remote07.mal
+# 10:41:05 >
+
+# builtin opt gdk_arch = 64bitx86_64-pc-linux-gnu
+# builtin opt gdk_version = 1.39.0
+# builtin opt prefix = /ufs/fabian/scratch/monetdb/current/program-x86_64
+# builtin opt exec_prefix = ${prefix}
+# builtin opt gdk_dbname = demo
+# builtin opt gdk_dbfarm = ${prefix}/var/MonetDB/dbfarm
+# builtin opt gdk_debug = 0
+# builtin opt gdk_alloc_map = no
+# builtin opt gdk_vmtrim = yes
+# builtin opt monet_admin = adm
+# builtin opt monet_prompt = >
+# builtin opt monet_welcome = yes
+# builtin opt monet_mod_path =
/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB
+# builtin opt monet_daemon = no
+# builtin opt host = localhost
+# builtin opt mapi_port = 50000
+# builtin opt mapi_clients = 2
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_debug = 0
+# builtin opt standoff_ns =
+# builtin opt standoff_start = start
+# builtin opt standoff_end = end
+# config opt prefix = /ufs/fabian/scratch/monetdb/current/program-x86_64
+# config opt config = ${prefix}/etc/monetdb5.conf
+# config opt prefix = /ufs/fabian/scratch/monetdb/current/program-x86_64
+# config opt exec_prefix = ${prefix}
+# config opt gdk_dbfarm = ${prefix}/var/MonetDB5/dbfarm
+# config opt monet_mod_path =
/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/lib:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/bin
+# config opt mero_pidfile = ${prefix}/var/run/MonetDB/merovingian.pid
+# config opt mero_controlport = 50001
+# config opt sql_optimizer = default_pipe
+# config opt minimal_pipe = inline,remap,deadcode,multiplex,garbageCollector
+# config opt default_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt no_mitosis_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt sequential_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,history,multiplex,garbageCollector
+# config opt nov2009_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt replication_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,replication,multiplex,garbageCollector
+# config opt accumulator_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,accumulators,dataflow,history,multiplex,garbageCollector
+# config opt recycler_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,deadcode,constants,commonTerms,joinPath,deadcode,recycle,reduce,dataflow,history,multiplex,garbageCollector
+# config opt cracker_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,selcrack,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt sidcrack_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,sidcrack,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt datacell_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,deadcode,constants,commonTerms,joinPath,datacell,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt octopus_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,constants,commonTerms,joinPath,octopus,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt tarantula_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,constants,commonTerms,joinPath,tarantula,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt mapreduce_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mapreduce,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt datacyclotron_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,datacyclotron,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,replication,multiplex,garbageCollector
+# config opt derive_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,commonTerms,derivePath,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt dictionary_pipe =
inline,remap,dictionary,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt compression_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,compression,dataflow,history,multiplex,garbageCollector
+# cmdline opt config =
/ufs/fabian/scratch/monetdb/current/program-x86_64/etc/monetdb5.conf
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt monet_mod_path =
/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/lib:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/bin
+# cmdline opt gdk_dbfarm =
/net/volund.ins.cwi.nl/export/scratch0/fabian/vtmp/mtest-current-volund.ins.cwi.nl/five/dbfarm
+# cmdline opt mapi_open = true
+# cmdline opt xrpc_open = true
+# cmdline opt mapi_port = 30248
+# cmdline opt xrpc_port = 47120
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbname = mTests_src_modules_mal
+#warning: please don't forget to set your vault key!
+#(see /ufs/fabian/scratch/monetdb/current/program-x86_64/etc/monetdb5.conf)
+
+# 10:41:05 >
+# 10:41:05 > Done.
+# 10:41:05 >
+
diff -r 4b6c272eb1ff -r abbdfa27a43d
MonetDB5/src/modules/mal/Tests/remote07.stable.out
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MonetDB5/src/modules/mal/Tests/remote07.stable.out Fri Sep 10
14:53:35 2010 +0200
@@ -0,0 +1,73 @@
+stdout of test 'remote07` in directory 'src/modules/mal` itself:
+
+
+# 10:41:05 >
+# 10:41:05 > mserver5
"--config=/ufs/fabian/scratch/monetdb/current/program-x86_64/etc/monetdb5.conf"
--debug=10 --set gdk_nr_threads=0 --set
"monet_mod_path=/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/lib:/ufs/fabian/scratch/monetdb/current/program-x86_64/lib/MonetDB5/bin"
--set
"gdk_dbfarm=/net/volund.ins.cwi.nl/export/scratch0/fabian/vtmp/mtest-current-volund.ins.cwi.nl/five/dbfarm"
--set mapi_open=true --set xrpc_open=true --set mapi_port=30248 --set
xrpc_port=47120 --set monet_prompt= --set mal_listing=2 --trace
--dbname=mTests_src_modules_mal remote07.mal
+# 10:41:05 >
+
+# MonetDB server v5.21.0, based on kernel v1.39.0
+# Release current-hg
+# Serving database 'mTests_src_modules_mal', using 4 threads
+# Compiled for x86_64-pc-linux-gnu/64bit with 64bit OIDs dynamically linked
+# Found 7.751 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2010 MonetDB B.V., all rights reserved
+# Visit http://monetdb.cwi.nl/ for further information
+# Listening for connection requests on mapi:monetdb://volund.ins.cwi.nl:30248/
+# MonetDB/SQL module v2.39.0 loaded
+# MonetDB/GIS module v0.19.0 loaded
+function user.main():void;
+# let connect figure out itself how to connect to the running db
+ uri := sabaoth.getLocalConnectionURI();
+ conn:str := remote.connect(uri,"monetdb","monetdb");
+ d:bat[:str,:int] := bat.new(:str,:int);
+ bat.insert(d,"bla",1);
+ bat.insert(d,nil:str,2);
+ bat.insert(d,"boe",nil:int);
+ bat.insert(d,d);
+# 6
+ bat.insert(d,d);
+# 12
+ bat.insert(d,d);
+# 24
+ bat.insert(d,d);
+# 48
+ bat.insert(d,d);
+# 96
+ bat.insert(d,d);
+# 192
+ bat.insert(d,d);
+# 384
+ bat.insert(d,d);
+# 768
+ bat.insert(d,d);
+# 1536
+ bat.insert(d,d);
+# 3072
+# store the values
+ t1 := alarm.time();
+ l:str := remote.put(conn,d);
+ t2 := alarm.time();
+ t3 := calc.-(t2,t1);
+ io.printf("# put: %d\n",t3);
+# retrieve the values
+ t1 := alarm.time();
+ u:bat[:str,:int] := remote.get(conn,l);
+ t2 := alarm.time();
+ t3 := calc.-(t2,t1);
+ io.printf("# get: %d\n",t3);
+# see if it really is here
+ c := aggr.count(u);
+ io.print(c);
+# help testweb a bit, since currently no cleanup is done on server
+# shutdown
+ remote.disconnect(conn);
+end main;
+# put: 310
+# get: 10
+[ 3072 ]
+
+# 10:41:05 >
+# 10:41:05 > Done.
+# 10:41:05 >
+
diff -r 4b6c272eb1ff -r abbdfa27a43d MonetDB5/src/modules/mal/remote.mx
--- a/MonetDB5/src/modules/mal/remote.mx Fri Sep 03 15:34:33 2010 +0200
+++ b/MonetDB5/src/modules/mal/remote.mx Fri Sep 10 14:53:35 2010 +0200
@@ -125,6 +125,13 @@
address RMTexec
comment "remotely executes <mod>.<func> using the argument list of remote
objects and returns the handle to its result";
+
+# helper functions not primilarily intended for normal use
+
+pattern batload(ht:any_1, tt:any_2, size:int):bat[:any_1,:any_2]
+address RMTbatload
+comment "create a BAT of the given type and size, and load values from the
input stream";
+
@{
@h
@@ -747,11 +754,12 @@
BATiter bi;
/* naive approach using bat.new() and bat.insert() calls */
char *head, *tail;
- char qbuf[BUFSIZ + 1]; /* FIXME: this should be dynamic */
+ char qbuf[BUFSIZ];
int bid;
BAT *b = NULL;
BUN p, q;
str headv, tailv;
+ stream *sout;
head = getTypeIdentifier(getHeadType(type));
tail = getTypeIdentifier(getTailType(type));
@@ -762,59 +770,51 @@
throw(MAL, "remote.put", RUNTIME_OBJECT_MISSING);
}
- qbuf[BUFSIZ] = '\0'; /* ensure a terminated string */
- snprintf(qbuf, BUFSIZ, "%s := bat.new(:%s, :%s, " BUNFMT ");",
+ /* bypass Mapi from this point to efficiently write all data to
+ * the server */
+ sout = mapi_get_to(c->mconn);
+
+ /* call our remote helper to do this more efficiently */
+ mnstr_printf(sout,
+ "%s := remote.batload(:%s, :%s, " BUNFMT ");\n",
ident, head, tail, (bid == 0 ? 0 :
BATcount(b)));
-#ifdef _DEBUG_REMOTE
- mnstr_printf(cntxt->fdout, "#remote.put:%s:%s\n", c->name,
qbuf);
-#endif
- if ((tmp = RMTquery(&mhdl, "remote.put", c->mconn, qbuf))
- != MAL_SUCCEED)
- {
- mal_unset_lock(c->lock, "remote.put");
- return tmp;
- }
- mapi_close_handle(mhdl);
+ mnstr_flush(sout);
/* b can be NULL if bid == 0 (only type given, ugh) */
if (b) {
- headv = tailv = NULL;
bi = bat_iterator(b);
BATloop(b, p, q) {
+ headv = tailv = NULL;
ATOMformat(getHeadType(type), BUNhead(bi, p),
&headv);
ATOMformat(getTailType(type), BUNtail(bi, p),
&tailv);
- if ( getTailType(type) <=TYPE_str &&
- getHeadType(type) <= TYPE_str)
- snprintf(qbuf, BUFSIZ, "bat.insert(%s, %s:%s,
%s:%s);",
- ident, headv, head, tailv,
tail);
- else
- if ( getTailType(type) >TYPE_str &&
- getHeadType(type) > TYPE_str)
- snprintf(qbuf, BUFSIZ, "bat.insert(%s,
\"%s\":%s, \"%s\":%s);",
- ident, headv, head, tailv,
tail);
- else
- if ( getTailType(type) >TYPE_str )
- snprintf(qbuf, BUFSIZ, "bat.insert(%s, %s:%s,
\"%s\":%s);",
- ident, headv, head, tailv,
tail);
- else
- snprintf(qbuf, BUFSIZ, "bat.insert(%s,
\"%s\":%s, %s:%s);",
- ident, headv, head, tailv,
tail);
-#ifdef _DEBUG_REMOTE
- mnstr_printf(cntxt->fdout,
- "#remote.put:%s:%s\n", c->name,
qbuf);
-#endif
- if ((tmp = RMTquery(&mhdl, "remote.put",
c->mconn, qbuf))
- != MAL_SUCCEED)
+ if (getTailType(type) <= TYPE_str &&
+ getHeadType(type) <= TYPE_str)
{
- mal_unset_lock(c->lock, "remote.put");
- return tmp;
+ mnstr_printf(sout, "%s,%s\n", headv,
tailv);
+ } else if (getTailType(type) > TYPE_str &&
+ getHeadType(type) > TYPE_str)
+ {
+ mnstr_printf(sout, "\"%s\",\"%s\"\n",
headv, tailv);
+ } else if (getTailType(type) > TYPE_str) {
+ mnstr_printf(sout, "%s,\"%s\"\n",
headv, tailv);
+ } else {
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
