Changeset: ab7257bc9881 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ab7257bc9881
Modified Files:
Branch: default
Log Message:
merging
diffs (truncated from 833 to 300 lines):
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB4/src/modules/plain/decimal.mx
--- a/MonetDB4/src/modules/plain/decimal.mx Fri May 21 16:30:04 2010 +0200
+++ b/MonetDB4/src/modules/plain/decimal.mx Fri May 21 16:38:48 2010 +0200
@@ -1887,6 +1887,8 @@
return GDK_FAIL;
bm = BATmirror(bn);
+ memset(&hp, 0, sizeof hp);
+
/* create space for decimals */
if (prec) {
yy = MIN(PREC(dv) + MAX(1, *prec) - 1, PRECISION_MAX) + 1;
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/Tests/All
--- a/MonetDB5/src/mal/Tests/All Fri May 21 16:30:04 2010 +0200
+++ b/MonetDB5/src/mal/Tests/All Fri May 21 16:38:48 2010 +0200
@@ -49,6 +49,7 @@
tst051
tst052
tst053
+tst054
tst055
tst056
tst057
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/Tests/tst054.mal
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MonetDB5/src/mal/Tests/tst054.mal Fri May 21 16:38:48 2010 +0200
@@ -0,0 +1,8 @@
+function z()(
+:int,:int,:int,:int,:int,:int,:int,:int,:int,:int,
+:int,:int,:int,:int,:int,:int,:int,:int,:int,:int,
+:int,:int,:int,:int,:int,:int,:int,:int,:int,:int);
+io.print("hello");
+return (1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0);
+end z;
+z();
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/Tests/tst054.stable.err
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MonetDB5/src/mal/Tests/tst054.stable.err Fri May 21 16:38:48 2010 +0200
@@ -0,0 +1,72 @@
+stderr of test 'tst054` in directory 'src/mal` itself:
+
+
+# 11:11:08 >
+# 11:11:08 > mserver5 "--config=/ufs/mk/current//Linux/etc/monetdb5.conf"
--debug=10 --set gdk_nr_threads=0 --set
"monet_mod_path=/ufs/mk/current//Linux/lib/MonetDB5:/ufs/mk/current//Linux/lib/MonetDB5/lib:/ufs/mk/current//Linux/lib/MonetDB5/bin"
--set "gdk_dbfarm=/ufs/mk/current//Linux/var/MonetDB5/dbfarm" --set
mapi_open=true --set xrpc_open=true --set mapi_port=31658 --set xrpc_port=44186
--set monet_prompt= --set mal_listing=2 --trace --dbname=mTests_src_mal
tst054.mal
+# 11:11:08 >
+
+# builtin opt gdk_arch = 64bitx86_64-unknown-linux-gnu
+# builtin opt gdk_version = 1.39.0
+# builtin opt prefix = /ufs/mk/current//Linux
+# builtin opt exec_prefix = ${prefix}
+# builtin opt gdk_dbname = demo
+# builtin opt gdk_dbfarm = ${prefix}/var/MonetDB/dbfarm
+# builtin opt gdk_debug = 0
+# builtin opt gdk_alloc_map = no
+# builtin opt gdk_vmtrim = yes
+# builtin opt monet_admin = adm
+# builtin opt monet_prompt = >
+# builtin opt monet_welcome = yes
+# builtin opt monet_mod_path = ${exec_prefix}/lib/MonetDB
+# builtin opt monet_daemon = no
+# builtin opt host = localhost
+# builtin opt mapi_port = 50000
+# builtin opt mapi_clients = 2
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_debug = 0
+# builtin opt standoff_ns =
+# builtin opt standoff_start = start
+# builtin opt standoff_end = end
+# config opt prefix = /ufs/mk/current//Linux
+# config opt config = ${prefix}/etc/monetdb5.conf
+# config opt prefix = /ufs/mk/current//Linux
+# config opt exec_prefix = ${prefix}
+# config opt gdk_dbfarm = ${prefix}/var/MonetDB5/dbfarm
+# config opt monet_mod_path =
${exec_prefix}/lib/MonetDB5:${exec_prefix}/lib/MonetDB5/lib:${exec_prefix}/lib/MonetDB5/bin
+# config opt mero_pidfile = ${prefix}/var/run/MonetDB/merovingian.pid
+# config opt mero_controlport = 50001
+# config opt sql_optimizer = default_pipe
+# config opt minimal_pipe = inline,remap,deadcode,multiplex,garbageCollector
+# config opt default_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt nov2009_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt replication_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,replication,multiplex,garbageCollector
+# config opt accumulator_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,accumulators,dataflow,history,multiplex,garbageCollector
+# config opt recycler_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,deadcode,constants,commonTerms,joinPath,deadcode,recycle,reduce,dataflow,history,multiplex,garbageCollector
+# config opt cracker_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,selcrack,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt sidcrack_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,sidcrack,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt datacell_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,deadcode,constants,commonTerms,joinPath,datacell,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt octopus_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,constants,commonTerms,joinPath,octopus,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt mapreduce_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mapreduce,mergetable,deadcode,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt datacyclotron_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,datacyclotron,mergetable,deadcode,constants,commonTerms,joinPath,reorder,deadcode,reduce,dataflow,history,replication,multiplex,garbageCollector
+# config opt derive_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mitosis,mergetable,deadcode,commonTerms,derivePath,joinPath,reorder,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt dictionary_pipe =
inline,remap,dictionary,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,history,multiplex,garbageCollector
+# config opt compression_pipe =
inline,remap,evaluate,costModel,coercions,emptySet,aliases,mergetable,deadcode,constants,commonTerms,joinPath,deadcode,reduce,dataflow,compression,dataflow,history,multiplex,garbageCollector
+# cmdline opt config = /ufs/mk/current//Linux/etc/monetdb5.conf
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt monet_mod_path =
/ufs/mk/current//Linux/lib/MonetDB5:/ufs/mk/current//Linux/lib/MonetDB5/lib:/ufs/mk/current//Linux/lib/MonetDB5/bin
+# cmdline opt gdk_dbfarm = /ufs/mk/current//Linux/var/MonetDB5/dbfarm
+# cmdline opt mapi_open = true
+# cmdline opt xrpc_open = true
+# cmdline opt mapi_port = 31658
+# cmdline opt xrpc_port = 44186
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbname = mTests_src_mal
+#warning: please don't forget to set your vault key!
+#(see /ufs/mk/current//Linux/etc/monetdb5.conf)
+
+# 11:11:08 >
+# 11:11:08 > Done.
+# 11:11:08 >
+
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/Tests/tst054.stable.out
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MonetDB5/src/mal/Tests/tst054.stable.out Fri May 21 16:38:48 2010 +0200
@@ -0,0 +1,31 @@
+stdout of test 'tst054` in directory 'src/mal` itself:
+
+
+# 11:11:08 >
+# 11:11:08 > mserver5 "--config=/ufs/mk/current//Linux/etc/monetdb5.conf"
--debug=10 --set gdk_nr_threads=0 --set
"monet_mod_path=/ufs/mk/current//Linux/lib/MonetDB5:/ufs/mk/current//Linux/lib/MonetDB5/lib:/ufs/mk/current//Linux/lib/MonetDB5/bin"
--set "gdk_dbfarm=/ufs/mk/current//Linux/var/MonetDB5/dbfarm" --set
mapi_open=true --set xrpc_open=true --set mapi_port=31658 --set xrpc_port=44186
--set monet_prompt= --set mal_listing=2 --trace --dbname=mTests_src_mal
tst054.mal
+# 11:11:08 >
+
+# MonetDB server v5.21.0, based on kernel v1.39.0
+# Serving database 'mTests_src_mal', using 4 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs dynamically
linked
+# Found 7.751 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2010 MonetDB B.V., all rights reserved
+# Visit http://monetdb.cwi.nl/ for further information
+# Listening for connection requests on mapi:monetdb://eir.ins.cwi.nl:31658/
+# MonetDB/SQL module v2.39.0 loaded
+# MonetDB/GIS module v0.19.0 loaded
+!SyntaxException:parseError:return
(1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0);
+!SyntaxException:parseError: ^<identifier> expected
+function user.z()
(_1:int,_2:int,_3:int,_4:int,_5:int,_6:int,_7:int,_8:int,_9:int,_10:int,_11:int,_12:int,_13:int,_14:int,_15:int,_16:int,_17:int,_18:int,_19:int,_20:int,_21:int,_22:int,_23:int,_24:int,_25:int,_26:int,_27:int,_28:int,_29:int,_30:int);
+ io.print("hello");
+end z;
+!TypeException:user.main[1]:'user.z' undefined in: _1:any := user.z()
+function user.main():void;
+ user.z();
+end main;
+
+# 11:11:08 >
+# 11:11:08 > Done.
+# 11:11:08 >
+
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/Tests/tst450.stable.out
--- a/MonetDB5/src/mal/Tests/tst450.stable.out Fri May 21 16:30:04 2010 +0200
+++ b/MonetDB5/src/mal/Tests/tst450.stable.out Fri May 21 16:38:48 2010 +0200
@@ -119,6 +119,7 @@
[ "mal", nil ]
[ "manual", nil ]
[ "mapi", nil ]
+[ "mapreduce", nil ]
[ "master", nil ]
[ "mat", nil ]
[ "mbr", nil ]
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/mal_instruction.mx
--- a/MonetDB5/src/mal/mal_instruction.mx Fri May 21 16:30:04 2010 +0200
+++ b/MonetDB5/src/mal/mal_instruction.mx Fri May 21 16:38:48 2010 +0200
@@ -2670,7 +2670,6 @@
advance(t,base,len);
}
switch (p->token<0?-p->token:p->token) {
- case NOOPsymbol:
case FCNcall:
case FACcall:
case PATcall:
@@ -2745,6 +2744,7 @@
}
return fcnDefinition(mb, p, s, flg, base, len);
case REMsymbol:
+ case NOOPsymbol:
if(getVar(mb, getArg(p, 0))->value.val.sval)
snprintf(t,(len-(t-base)), "#%s ", getVar(mb, getArg(p,
0))->value.val.sval);
else
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/mal/mal_parser.mx
--- a/MonetDB5/src/mal/mal_parser.mx Fri May 21 16:30:04 2010 +0200
+++ b/MonetDB5/src/mal/mal_parser.mx Fri May 21 16:38:48 2010 +0200
@@ -1484,7 +1484,7 @@
if( l==0 || i) {
parseError(cntxt,"<identifier> expected\n");
skipToEnd(cntxt);
- pushInstruction(curBlk,curInstr);
+ freeInstruction(curInstr);
return;
}
@:GETvariable@
diff -r 4ed7d0ca8b5d -r ab7257bc9881
MonetDB5/src/modules/mal/Tests/inspect05.stable.out
--- a/MonetDB5/src/modules/mal/Tests/inspect05.stable.out Fri May 21
16:30:04 2010 +0200
+++ b/MonetDB5/src/modules/mal/Tests/inspect05.stable.out Fri May 21
16:38:48 2010 +0200
@@ -1009,6 +1009,7 @@
[ "new", "command", "cluster",
"(b:bat[:oid,:bte],bits:int,offset:int)
(psum:bat[:oid,:wrd],map:bat[:oid,:wrd]) ",
"CLS_create_bte;" ]
[ "split", "pattern", "cluster",
"(clustered:bat[:oid,:any_1],psum:bat[:oid,:wrd]):bat[:oid,:any_1]... ",
"CLS_split;" ]
[ "table", "pattern", "cluster",
"(b:bat[:oid,:any]...):bat[:oid,:oid] ",
"CLUSTER_table;"
]
+[ "getCloud", "command", "mapreduce",
"(nme:str):bat[:oid,:str] ",
"MRgetCloud;"
]
[ "multiplex", "pattern", "mal",
"(a:any...):any ",
"OPTremapMultiplex;"
]
[ "argRecord", "pattern", "sql",
"(a:any...):str ",
"SQLargRecord;"
]
[ "argRecord", "pattern", "sql", "():str ",
"SQLargRecord;"
]
diff -r 4ed7d0ca8b5d -r ab7257bc9881 MonetDB5/src/optimizer/opt_mapreduce.mx
--- a/MonetDB5/src/optimizer/opt_mapreduce.mx Fri May 21 16:30:04 2010 +0200
+++ b/MonetDB5/src/optimizer/opt_mapreduce.mx Fri May 21 16:38:48 2010 +0200
@@ -20,6 +20,11 @@
@- Map-Reduce
The Map-Reduce infrastructure requires a little optimizer to turn
an arbitrary query into a plan to be executed on the elements of the Cloud.
+Each cloud consists of a series of named servers, managed by Merovingian
+with the pattern "*/Cloudname/node/*".
+
+Determining the clould is an expensive operation and for the time being
+performed each time when a query is compiled.
In the first implementation we don't optimize the plan against the mapping
scheme.
We simply assume that the complete query can be executed and that only the
@@ -108,14 +113,20 @@
return (Y201,Y202,Y203);
end exec;
@end verbatim
+...@{
The code can be considered a refinement of the Octopus.
-...@{
+The mal primitives are meant for debugging.
@mal
pattern optimizer.mapreduce():str
address OPTmapreduce;
pattern optimizer.mapreduce(mod:str, fcn:str):str
address OPTmapreduce
comment "Modify the plan to exploit parallel processing on multiple systems
using map-reduce";
+
+module mapreduce;
+command getCloud(nme:str):bat[:oid,:str]
+address MRgetCloud
+comment "Localize the elements of a named cloud";
@h
#ifndef _OPT_MAPREDUCE_
#define _OPT_MAPREDUCE_
@@ -123,6 +134,7 @@
#include "opt_support.h"
opt_export str MRexec(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
+opt_export str MRgetCloud(int *ret, str *nme);
@c
#include "mal_config.h"
#include "opt_mapreduce.h"
@@ -134,6 +146,7 @@
the number of nodes participating in the cloud setting.
It calls the map-reduce executor to produce a result
possible with the aid of a replica.
+The mapnode is an auxilary structure.
@c
typedef struct _mapnode {
@@ -142,68 +155,92 @@
str pass;
} mapnode;
-#define MAXNODES 256
-static mapnode mapnodes[MAXNODES];
+static mapnode *mapnodes;
+static BAT *cloud;
+
+static void
+MRcleanCloud()
+{
+ int i;
+
+ mal_set_lock(mal_contextLock,"mapreduce");
+ for (i = 0; mapnodes[i].uri; i++) {
+ if (mapnodes[i].uri != NULL)
+ GDKfree(mapnodes[i].uri);
+ if (mapnodes[i].user != NULL)
+ GDKfree(mapnodes[i].user);
+ if (mapnodes[i].pass != NULL)
+ GDKfree(mapnodes[i].pass);
+ mapnodes[i].uri = mapnodes[i].user = mapnodes[i].pass = 0;
+ }
+ if ( cloud)
+ BBPreleaseref(cloud->batCacheid);
+ cloud = 0;
+ mal_unset_lock(mal_contextLock,"mapreduce");
+}
+
+str
+MRgetCloud(int *ret, str *mrcluster)
+{
+ str msg;
+ BUN p, q;
+ BATiter bi;
+ char nodes[BUFSIZ];
+ char *n = nodes;
+ int mapcount = 0;
+
+ snprintf(nodes, sizeof(nodes), "*/%s/node/*", *mrcluster);
+
+ msg = RMTresolve(ret, &n);
+ if (msg )
+ return msg;
+
+ if ( cloud )
+ /* cleanup remains of previous call */
_______________________________________________
Checkin-list mailing list
Checkin-list@monetdb.org
http://mail.monetdb.org/mailman/listinfo/checkin-list
