Changeset: 786276ea7767 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=786276ea7767 Modified Files: monetdb5/ChangeLog monetdb5/mal/mal.c monetdb5/mal/mal_client.mx monetdb5/mal/mal_debugger.mx monetdb5/modules/mal/tablet.c monetdb5/modules/mal/tablet.h monetdb5/modules/mal/tablet.mal Branch: default Log Message:
m5: turned MAL_MAXCLIENTS into a runtime variable While 64 may be enough for most cases these days with single clients, it certainly isn't for environments where hundreds of clients are "shot" at the database as seen with e.g. the LOFAR pipeline. Given that dataflow also eats threads per core, on modern hardware the 64 client slots are easily halfed. Hence, allow the user to start the database with a max_clients setting, to easily override the default 64 in cases where this makes sense. diffs (217 lines): diff --git a/monetdb5/ChangeLog b/monetdb5/ChangeLog --- a/monetdb5/ChangeLog +++ b/monetdb5/ChangeLog @@ -1,21 +1,25 @@ # ChangeLog file for MonetDB5 # This file is updated with Maddlog -* Tue Aug 2 2011 [email protected] +* Fri Sep 23 2011 Fabian Groffen <[email protected]> +- Introduced new variable max_clients that allows to define how many + concurrent connections are allowed to be made against the database. + +* Tue Aug 2 2011 Martin Kersten <[email protected]> - mal_interpreter.mx Protect against concurrent exceptions If multiple parallel blocks access the exception variables and perform GDKfree on old messages, then we may enter a case that one thread attempts a second free. Simple lock-based protection is the first line of defense. -* Tue Aug 2 2011 [email protected] +* Tue Aug 2 2011 Martin Kersten <[email protected]> - bpm and partitions have been moved to the attic. It is replaced by the partition optimizer, still under construction. -* Tue Aug 2 2011 [email protected] +* Tue Aug 2 2011 Martin Kersten <[email protected]> - The MAL debugger next/step operations semantics have been fixed. The profiler also now clearly shows entrance/exit of a MAL function. -* Tue Aug 2 2011 [email protected] +* Tue Aug 2 2011 Martin Kersten <[email protected]> - The dataflow optimizer uses a less strict side-effect test for BAT new operations and better recognizes trivial plans. diff --git a/monetdb5/mal/mal.c b/monetdb5/mal/mal.c --- a/monetdb5/mal/mal.c +++ b/monetdb5/mal/mal.c @@ -188,6 +188,7 @@ int nrservers = 0; #include "mal_parser.h" #include "mal_interpreter.h" #include "mal_namespace.h" /* for initNamespace() */ +#include "mal_debugger.h" /* for mdbInit() */ #include "mal_client.h" #include "mal_sabaoth.h" #include "mal_recycle.h" @@ -236,7 +237,8 @@ int mal_init(void){ GDKprotect(); tstAligned(); - memset((char*)mal_clients, 0, sizeof(ClientRec)*MAL_MAXCLIENTS); + MCinit(); + mdbInit(); if (monet_memory == 0) monet_memory = MT_npages() * MT_pagesize(); initNamespace(); diff --git a/monetdb5/mal/mal_client.mx b/monetdb5/mal/mal_client.mx --- a/monetdb5/mal/mal_client.mx +++ b/monetdb5/mal/mal_client.mx @@ -76,7 +76,6 @@ All Rights Reserved. #include "mal_profiler.h" #include "mal.h" -#define MAL_MAXCLIENTS 64 #define CONSOLE 0 #define isAdministrator(X) (X==mal_clients) @@ -261,7 +260,10 @@ typedef struct CLIENT { int exception_buf_initialized; } *Client, ClientRec; -mal_export ClientRec mal_clients[MAL_MAXCLIENTS+1]; +mal_export void MCinit(void); + +mal_export int MAL_MAXCLIENTS; +mal_export ClientRec *mal_clients; mal_export int MCdefault; mal_export Client MCgetClient(int id); @@ -321,10 +323,32 @@ int bstreamClient(Client c, str prompt); static void freeClient (Client c); -ClientRec mal_clients[MAL_MAXCLIENTS+1]; +int MAL_MAXCLIENTS = 0; +ClientRec *mal_clients; /*int MCdefault= threadFlag | bigfootFlag;*/ int MCdefault= 0; +void +MCinit(void) +{ + char *max_clients = GDKgetenv("max_clients"); + int threads = GDKnr_threads; + int maxclients = 0; + + if (max_clients != NULL) + maxclients = atoi(max_clients); + if (maxclients <= 0) { + maxclients = 64; + GDKsetenv("max_clients", "64"); + } + + MAL_MAXCLIENTS = + /* console */ 1 + + /* workers */ threads + + /* client connections */ maxclients; + mal_clients = GDKzalloc(sizeof(ClientRec) * MAL_MAXCLIENTS); +} + int MCpushClientInput(Client c, bstream *new_input, int listing, char *prompt) { ClientInput *x = (ClientInput*)GDKmalloc(sizeof(ClientInput)); diff --git a/monetdb5/mal/mal_debugger.mx b/monetdb5/mal/mal_debugger.mx --- a/monetdb5/mal/mal_debugger.mx +++ b/monetdb5/mal/mal_debugger.mx @@ -481,6 +481,7 @@ typedef struct MDBSTATE{ int pc; } MdbState; +mal_export void mdbInit(void); mal_export void mdbSetBreakpoint(Client cntxt, MalBlkPtr mb, int pc, char cmd); mal_export void mdbSetBreakRequest(Client cntxt, MalBlkPtr mb, str request, char cmd); mal_export void mdbClrBreakRequest(Client cntxt, str name); @@ -538,13 +539,19 @@ int MDBdelay; /* do not immediately re static void printStackElm(stream *f, MalBlkPtr mb, ValPtr v, int index, BUN cnt, BUN first); static void printStackHdr(stream *f, MalBlkPtr mb, ValPtr v, int index); -/* - * @- - * Each client has its own breakpoint administration, kept in a global table. - * Although a little space consumptive, it is the easiest to maintain - * and much less expensive as reserving debugger space in each instruction. - */ -mdbStateRecord mdbTable[MAL_MAXCLIENTS]; +static mdbStateRecord *mdbTable; + +void +mdbInit(void) +{ + /* + * Each client has its own breakpoint administration, kept in a + * global table. Although a little space consumptive, it is the + * easiest to maintain and much less expensive as reserving debugger + * space in each instruction. + */ + mdbTable = GDKzalloc(sizeof(mdbStateRecord) * MAL_MAXCLIENTS); +} static char isBreakpoint(Client cntxt, MalBlkPtr mb, InstrPtr p, int pc) diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c --- a/monetdb5/modules/mal/tablet.c +++ b/monetdb5/modules/mal/tablet.c @@ -309,14 +309,22 @@ static int setTabwidth(Column *c); while(n-->0) \ mnstr_printf(s, "\t"); \ } while (0) -/* - * @+ - * The table formatting information is stored in a system wide table. - * Access is granted to a single client thread only. - * The table structure depends on the columns to be printed, - * it will be dynamically extended to accommodate the space. - */ -static Tablet *tableReports[MAL_MAXCLIENTS]; + +static Tablet **tableReports; + +str +TABprelude(int *ret) +{ + /* + * The table formatting information is stored in a system wide table. + * Access is granted to a single client thread only. + * The table structure depends on the columns to be printed, + * it will be dynamically extended to accommodate the space. + */ + tableReports = GDKzalloc(sizeof(Tablet *) * MAL_MAXCLIENTS); + *ret = 0; + return MAL_SUCCEED; +} static void diff --git a/monetdb5/modules/mal/tablet.h b/monetdb5/modules/mal/tablet.h --- a/monetdb5/modules/mal/tablet.h +++ b/monetdb5/modules/mal/tablet.h @@ -125,6 +125,8 @@ typedef struct Table_t { Column columns[1]; /* at least one column, enlarged upon need */ } Tablet; +tablet_export str TABprelude(int *ret); + tablet_export BAT *TABLETload(Tablet *as, char *datafile); tablet_export BUN TABLEToutput(BAT *order, BAT *seps, BAT *bats, stream *s); tablet_export void TABLETdump(BAT *names, BAT *seps, BAT *bats, char *datafile, BUN nr); diff --git a/monetdb5/modules/mal/tablet.mal b/monetdb5/modules/mal/tablet.mal --- a/monetdb5/modules/mal/tablet.mal +++ b/monetdb5/modules/mal/tablet.mal @@ -17,6 +17,10 @@ module tablet; +command prelude():void +address TABprelude +comment "initialise the tablet module"; + command load( names:bat[:oid,:str], seps:bat[:oid,:str], types:bat[:oid,:str], datafile:str, nr:int ) :bat[:str,:bat] address CMDtablet_load @@ -133,4 +137,6 @@ comment "Return the size in number of pa command getPage(i:int):void address TABgetPage comment "Produce the i-th page of output"; -# @- + +# initialise our tblrecord +tablet.prelude(); _______________________________________________ Checkin-list mailing list [email protected] http://mail.monetdb.org/mailman/listinfo/checkin-list
