Changeset: 5349ad317813 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5349ad317813
Removed Files:
monetdb5/scheduler/srvpool.c
monetdb5/scheduler/srvpool.h
monetdb5/scheduler/srvpool.mal
Modified Files:
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
geom/monetdb5/Makefile.ag
monetdb5/extras/mal_optimizer_template/Makefile.ag
monetdb5/mal/mal_linker.c
monetdb5/modules/kernel/Makefile.ag
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/mal_init.mal
monetdb5/scheduler/Makefile.ag
sql/backends/monet5/LSST/Makefile.ag
sql/backends/monet5/Makefile.ag
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/vaults/Makefile.ag
sql/backends/monet5/vaults/fits/Makefile.ag
sql/backends/monet5/vaults/netcdf/Makefile.ag
Branch: default
Log Message:
Merge with Dec2016 branch.
diffs (truncated from 969 to 300 lines):
diff --git a/NT/installer32/MonetDB5-SQL-Installer.vdproj
b/NT/installer32/MonetDB5-SQL-Installer.vdproj
--- a/NT/installer32/MonetDB5-SQL-Installer.vdproj
+++ b/NT/installer32/MonetDB5-SQL-Installer.vdproj
@@ -585,12 +585,6 @@
}
"Entry"
{
- "MsmKey" = "8:_8F6BBCFB08084CD1AF561D7CF43D5BD5"
- "OwnerKey" = "8:_UNDEFINED"
- "MsmSig" = "8:_UNDEFINED"
- }
- "Entry"
- {
"MsmKey" = "8:_90BC4B557F7942DD8DEAC13CD0EB6FE9"
"OwnerKey" = "8:_UNDEFINED"
"MsmSig" = "8:_UNDEFINED"
@@ -2912,26 +2906,6 @@
"IsDependency" = "11:FALSE"
"IsolateTo" = "8:"
}
-
"{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_8F6BBCFB08084CD1AF561D7CF43D5BD5"
- {
- "SourcePath" = "8:lib\\monetdb5\\srvpool.mal"
- "TargetName" = "8:srvpool.mal"
- "Tag" = "8:"
- "Folder" = "8:_B593DAA0787744D6A44D17884E0C3951"
- "Condition" = "8:"
- "Transitive" = "11:FALSE"
- "Vital" = "11:TRUE"
- "ReadOnly" = "11:FALSE"
- "Hidden" = "11:FALSE"
- "System" = "11:FALSE"
- "Permanent" = "11:FALSE"
- "SharedLegacy" = "11:FALSE"
- "PackageAs" = "3:1"
- "Register" = "3:1"
- "Exclude" = "11:FALSE"
- "IsDependency" = "11:FALSE"
- "IsolateTo" = "8:"
- }
"{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_90BC4B557F7942DD8DEAC13CD0EB6FE9"
{
"SourcePath" = "8:lib\\monetdb5\\createdb\\14_inet.sql"
diff --git a/NT/installer64/MonetDB5-SQL-Installer.vdproj
b/NT/installer64/MonetDB5-SQL-Installer.vdproj
--- a/NT/installer64/MonetDB5-SQL-Installer.vdproj
+++ b/NT/installer64/MonetDB5-SQL-Installer.vdproj
@@ -585,12 +585,6 @@
}
"Entry"
{
- "MsmKey" = "8:_8F6BBCFB08084CD1AF561D7CF43D5BD5"
- "OwnerKey" = "8:_UNDEFINED"
- "MsmSig" = "8:_UNDEFINED"
- }
- "Entry"
- {
"MsmKey" = "8:_90BC4B557F7942DD8DEAC13CD0EB6FE9"
"OwnerKey" = "8:_UNDEFINED"
"MsmSig" = "8:_UNDEFINED"
@@ -2912,26 +2906,6 @@
"IsDependency" = "11:FALSE"
"IsolateTo" = "8:"
}
-
"{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_8F6BBCFB08084CD1AF561D7CF43D5BD5"
- {
- "SourcePath" = "8:lib\\monetdb5\\srvpool.mal"
- "TargetName" = "8:srvpool.mal"
- "Tag" = "8:"
- "Folder" = "8:_B593DAA0787744D6A44D17884E0C3951"
- "Condition" = "8:"
- "Transitive" = "11:FALSE"
- "Vital" = "11:TRUE"
- "ReadOnly" = "11:FALSE"
- "Hidden" = "11:FALSE"
- "System" = "11:FALSE"
- "Permanent" = "11:FALSE"
- "SharedLegacy" = "11:FALSE"
- "PackageAs" = "3:1"
- "Register" = "3:1"
- "Exclude" = "11:FALSE"
- "IsDependency" = "11:FALSE"
- "IsolateTo" = "8:"
- }
"{1FB2D0AE-D3B9-43D4-B9DD-F88EC61E35DE}:_90BC4B557F7942DD8DEAC13CD0EB6FE9"
{
"SourcePath" = "8:lib\\monetdb5\\createdb\\14_inet.sql"
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8270,7 +8270,6 @@ Ready.
[ "sample", "subuniform", "command sample.subuniform(b:bat[:any],
p:dbl):bat[:oid] ", "SAMPLEuniform_dbl;", "Returns the oids of a uniform
sample of size = (p x count(b)), where 0 <= p <= 1.0" ]
[ "scheduler", "pattern", "pattern scheduler.pattern(pat:str):void ",
"SRVsetServers;", "Activate the peers matching the pattern and return
number found." ]
[ "scheduler", "peers", "pattern scheduler.peers(N:int):void ",
"SRVsetServers;", "Simulate N peers" ]
-[ "scheduler", "srvpool", "pattern scheduler.srvpool():int ",
"SRVPOOLscheduler;", "Run a program block in parallel." ]
[ "shp", "attach", "pattern shp.attach(filename:str):void ",
"SHPattach;", "Register an ESRI Shapefile in the vault catalog" ]
[ "shp", "import", "pattern shp.import(fileid:int):void ",
"SHPimport;", "Import an ESRI Shapefile with given id into the vault" ]
[ "shp", "import", "pattern shp.import(fileid:int, po:wkb):void ",
"SHPpartialimport;", "Partially import an ESRI Shapefile with given id into
the vault" ]
@@ -8473,11 +8472,6 @@ Ready.
[ "sqlcatalog", "revoke", "pattern sqlcatalog.revoke(sname:str,
tbl:str, grantee:str, privs:int, cname:str, grant:int, grantor:int):void ",
"SQLrevoke;", "Catalog operation revoke" ]
[ "sqlcatalog", "revoke_function", "pattern
sqlcatalog.revoke_function(sname:str, fcnid:int, grantee:str, privs:int,
grant:int, grantor:int):void ", "SQLrevoke_function;", "Catalog
operation revoke_function" ]
[ "sqlcatalog", "revoke_roles", "pattern
sqlcatalog.revoke_roles(sname:str, auth:str, grantor:int, admin:int):void ",
"SQLrevoke_roles;", "Catalog operation revoke_roles" ]
-[ "srvpool", "connect", "command srvpool.connect(dbname:str):str ",
"SRVPOOLconnect;", "Returns the connection to the server dbname." ]
-[ "srvpool", "exec", "pattern srvpool.exec(fname:str, arg:any...):any... ",
"SRVPOOLexec;", "Remote execution of the function on dbname if channel is
free." ]
-[ "srvpool", "local", "command srvpool.local(b:bit):void ",
"SRVPOOLlocal;", "Change location from remote to local execution"
]
-[ "srvpool", "query", "pattern srvpool.query(plan:str):str... ",
"SRVPOOLquery;", "Allocate a list of servers for query execution"
]
-[ "srvpool", "reset", "pattern srvpool.reset(dbname:str):str ",
"SRVPOOLreset;", "Clear the server cache" ]
[ "status", "batStatistics", "command status.batStatistics()
(X_0:bat[:str], X_1:bat[:str]) ", "SYSgdkEnv;", "Show distribution of
bats by kind" ]
[ "status", "cpuStatistics", "command status.cpuStatistics()
(X_0:bat[:str], X_1:bat[:int]) ", "SYScpuStatistics;", "Global cpu
usage information" ]
[ "status", "getThreads", "command status.getThreads() (X_0:bat[:int],
X_1:bat[:str]) ", "SYSgdkThread;", "Produce overview of active threads"
]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -10632,7 +10632,6 @@ Ready.
[ "sample", "subuniform", "command sample.subuniform(b:bat[:any],
p:dbl):bat[:oid] ", "SAMPLEuniform_dbl;", "Returns the oids of a uniform
sample of size = (p x count(b)), where 0 <= p <= 1.0" ]
[ "scheduler", "pattern", "pattern scheduler.pattern(pat:str):void ",
"SRVsetServers;", "Activate the peers matching the pattern and return
number found." ]
[ "scheduler", "peers", "pattern scheduler.peers(N:int):void ",
"SRVsetServers;", "Simulate N peers" ]
-[ "scheduler", "srvpool", "pattern scheduler.srvpool():int ",
"SRVPOOLscheduler;", "Run a program block in parallel." ]
[ "shp", "attach", "pattern shp.attach(filename:str):void ",
"SHPattach;", "Register an ESRI Shapefile in the vault catalog" ]
[ "shp", "import", "pattern shp.import(fileid:int):void ",
"SHPimport;", "Import an ESRI Shapefile with given id into the vault" ]
[ "shp", "import", "pattern shp.import(fileid:int, po:wkb):void ",
"SHPpartialimport;", "Partially import an ESRI Shapefile with given id into
the vault" ]
@@ -10837,11 +10836,6 @@ Ready.
[ "sqlcatalog", "revoke", "pattern sqlcatalog.revoke(sname:str,
tbl:str, grantee:str, privs:int, cname:str, grant:int, grantor:int):void ",
"SQLrevoke;", "Catalog operation revoke" ]
[ "sqlcatalog", "revoke_function", "pattern
sqlcatalog.revoke_function(sname:str, fcnid:int, grantee:str, privs:int,
grant:int, grantor:int):void ", "SQLrevoke_function;", "Catalog
operation revoke_function" ]
[ "sqlcatalog", "revoke_roles", "pattern
sqlcatalog.revoke_roles(sname:str, auth:str, grantor:int, admin:int):void ",
"SQLrevoke_roles;", "Catalog operation revoke_roles" ]
-[ "srvpool", "connect", "command srvpool.connect(dbname:str):str ",
"SRVPOOLconnect;", "Returns the connection to the server dbname." ]
-[ "srvpool", "exec", "pattern srvpool.exec(fname:str, arg:any...):any... ",
"SRVPOOLexec;", "Remote execution of the function on dbname if channel is
free." ]
-[ "srvpool", "local", "command srvpool.local(b:bit):void ",
"SRVPOOLlocal;", "Change location from remote to local execution"
]
-[ "srvpool", "query", "pattern srvpool.query(plan:str):str... ",
"SRVPOOLquery;", "Allocate a list of servers for query execution"
]
-[ "srvpool", "reset", "pattern srvpool.reset(dbname:str):str ",
"SRVPOOLreset;", "Clear the server cache" ]
[ "status", "batStatistics", "command status.batStatistics()
(X_0:bat[:str], X_1:bat[:str]) ", "SYSgdkEnv;", "Show distribution of
bats by kind" ]
[ "status", "cpuStatistics", "command status.cpuStatistics()
(X_0:bat[:str], X_1:bat[:int]) ", "SYScpuStatistics;", "Global cpu
usage information" ]
[ "status", "getThreads", "command status.getThreads() (X_0:bat[:int],
X_1:bat[:str]) ", "SYSgdkThread;", "Produce overview of active threads"
]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1704,14 +1704,6 @@ str SERVERtrace(void *ret, int *mid, int
int SQLBLOBfromstr(char *instr, int *l, blob **val);
int SQLBLOBtostr(str *tostr, int *l, blob *pin);
BUN SQLload_file(Client cntxt, Tablet *as, bstream *b, stream *out, char
*csep, char *rsep, char quote, lng skip, lng maxrow, int best);
-str SRVPOOLconnect(str *c, str *dbname);
-str SRVPOOLexec(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str SRVPOOLlocal(void *res, bit *flag);
-str SRVPOOLquery(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str SRVPOOLregister(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str SRVPOOLreset(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
-str SRVPOOLscheduler(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
-str SRVsetServers(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str STRBytes(int *res, const str *arg1);
str STRConcat(str *res, const str *val1, const str *val2);
str STRFromWChr(str *res, const int *at);
diff --git a/geom/monetdb5/Makefile.ag b/geom/monetdb5/Makefile.ag
--- a/geom/monetdb5/Makefile.ag
+++ b/geom/monetdb5/Makefile.ag
@@ -34,5 +34,3 @@ headers_autoload = {
DIR = libdir/monetdb5/autoload
SOURCES = 30_geom.mal
}
-
-EXTRA_DIST = 30_geom.mal geom.mal
diff --git a/monetdb5/extras/mal_optimizer_template/Makefile.ag
b/monetdb5/extras/mal_optimizer_template/Makefile.ag
--- a/monetdb5/extras/mal_optimizer_template/Makefile.ag
+++ b/monetdb5/extras/mal_optimizer_template/Makefile.ag
@@ -36,6 +36,4 @@ headers_opt_sql_append_autoload = {
SOURCES = 91_opt_sql_append.mal
}
-EXTRA_DIST = 91_opt_sql_append.mal
-
EXTRA_DIST_DIR = Tests
diff --git a/monetdb5/mal/mal_linker.c b/monetdb5/mal/mal_linker.c
--- a/monetdb5/mal/mal_linker.c
+++ b/monetdb5/mal/mal_linker.c
@@ -65,8 +65,7 @@ getAddress(stream *out, str modname, str
{
void *dl;
MALfcn adr;
- static int idx=0;
-
+ int idx=0;
static int prev= -1;
/* First try the last module loaded */
@@ -82,7 +81,9 @@ getAddress(stream *out, str modname, str
* obtained from the source-file MAL script.
*/
for (idx =0; idx < lastfile; idx++)
- if (filesLoaded[idx].handle) {
+ if (idx != prev && /* skip already searched module
*/
+ filesLoaded[idx].handle &&
+ (idx == 0 || filesLoaded[idx].handle !=
filesLoaded[0].handle)) {
adr = (MALfcn) dlsym(filesLoaded[idx].handle, fcnname);
if (adr != NULL) {
prev = idx;
diff --git a/monetdb5/modules/kernel/Makefile.ag
b/monetdb5/modules/kernel/Makefile.ag
--- a/monetdb5/modules/kernel/Makefile.ag
+++ b/monetdb5/modules/kernel/Makefile.ag
@@ -46,6 +46,4 @@ headers_hge = {
SOURCES = 00_aggr_hge.mal
}
-EXTRA_DIST = alarm.mal logger.mal microbenchmark.mal aggr.mal 00_aggr_hge.mal
group.mal algebra.mal
-
EXTRA_DIST_DIR = Tests
diff --git a/monetdb5/modules/mal/Makefile.ag b/monetdb5/modules/mal/Makefile.ag
--- a/monetdb5/modules/mal/Makefile.ag
+++ b/monetdb5/modules/mal/Makefile.ag
@@ -81,15 +81,4 @@ headers_hge = {
00_mal_mapi_hge.mal 00_mkey_hge.mal 00_batExtensions_hge.mal
}
-EXTRA_DIST = batExtensions.mal iterator.mal \
- groupby.mal mal_init.mal manual.mal mkey.mal manifold.mal pcre.mal \
- profiler.mal remote.mal sabaoth.mal orderidx.mal \
- transaction.mal txtsim.mal tablet.mal tablet.h sample.mal json_util.mal
\
- mal_mapi.mal mat.mal tokenizer.mal \
- batmtime.mal querylog.mal sysmon.mal \
- 01_calc.mal 01_batcalc.mal \
- 00_calc_hge.mal 00_batcalc_hge.mal \
- 00_iterator_hge.mal 00_language_hge.mal \
- 00_mal_mapi_hge.mal 00_mkey_hge.mal 00_batExtensions_hge.mal
-
EXTRA_DIST_DIR = Tests
diff --git a/monetdb5/modules/mal/mal_init.mal
b/monetdb5/modules/mal/mal_init.mal
--- a/monetdb5/modules/mal/mal_init.mal
+++ b/monetdb5/modules/mal/mal_init.mal
@@ -87,11 +87,6 @@ include iterator;
include txtsim;
include tokenizer;
-# include transaction; no direct access anymore to avoid SQL interference
-
-# scheduler components, experimental components
-include srvpool;
-
include mal_mapi;
include oltp;
diff --git a/monetdb5/scheduler/Makefile.ag b/monetdb5/scheduler/Makefile.ag
--- a/monetdb5/scheduler/Makefile.ag
+++ b/monetdb5/scheduler/Makefile.ag
@@ -20,15 +20,13 @@ lib_scheduler = {
run_adder.c run_adder.h \
run_isolate.c run_isolate.h \
run_memo.c run_memo.h \
- srvpool.c srvpool.h \
run_pipeline.c run_pipeline.h
}
headers_mal = {
HEADERS = mal
DIR = libdir/monetdb5
- SOURCES = run_adder.mal run_isolate.mal run_memo.mal srvpool.mal
+ SOURCES = run_adder.mal run_isolate.mal run_memo.mal
}
EXTRA_DIST_DIR = Tests
-EXTRA_DIST = run_adder.mal run_isolate.mal run_memo.mal srvpool.mal
diff --git a/monetdb5/scheduler/srvpool.c b/monetdb5/scheduler/srvpool.c
deleted file mode 100644
--- a/monetdb5/scheduler/srvpool.c
+++ /dev/null
@@ -1,517 +0,0 @@
-/*
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Copyright 1997 - July 2008 CWI, August 2008 - 2017 MonetDB B.V.
- */
-
-/*
- * @a M. Kersten
- * The experimental octopus/centipede modules enabled compute cloud based
processing of SQL queries.
- * Their scheduler takes over control of a MAL execution by
- * re-directing requests to multiple sites. If there are no sites known,
- * then the code is executed locally as is.
- * The scheduler runs all subqueries asynchronously if possible.
- *
- * To make our live easier, we assume that all subqueries are
- * grouped together in a guarded block as follows:
- *
- * @verbatim
- * barrier parallel:= scheduler.srvpool();
- * (s1,...,sn) := srvpool.query("queryplan");
- * a:= user.S0_1_stub(s1,arg...);
- * ...
- * b:= srvpool.S0_1_stub(sn,arg...);
- * z:= mat.pack(a,...,b);
- * exit parallel;
- * @end verbatim
- *
- * We assume that the database versions are synchronised.
- *
- * To make the scheduler work, it needs a list of database instances.
- * This list it gets from Merovingian by resolving
- * all with the property 'shared=octopus' or 'shared=centipede' (set by
monetdb).
- * The default is to use the local database as a target.
- *
- * Ideally, we would not like to ship query plans with each query, but
- * this requires knowledge that the remote server has not been restarted
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
