Changeset: a926a05e53f7 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a926a05e53f7
Modified Files:
configure.ag
gdk/ChangeLog.Apr2019
gdk/gdk_utils.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/modules/mal/language.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/wlc.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/wlr.c
sql/storage/bat/bat_storage.c
sql/storage/store.c
Branch: default
Log Message:
Merge with Apr2019 branch.
diffs (truncated from 554 to 300 lines):
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2539,6 +2539,7 @@ AC_CHECK_FUNCS([\
getlogin \
getopt \
getopt_long \
+ getrlimit \
gettimeofday \
getuid \
gmtime_r \
diff --git a/gdk/ChangeLog.Apr2019 b/gdk/ChangeLog.Apr2019
--- a/gdk/ChangeLog.Apr2019
+++ b/gdk/ChangeLog.Apr2019
@@ -1,3 +1,8 @@
# ChangeLog file for MonetDB
# This file is updated with Maddlog
+* Mon Jul 15 2019 Sjoerd Mullender <[email protected]>
+- We now look at the limits imposed by cgroups and the setrlimit system
+ call to initialize some internal values related to how much (virtual)
+ memory we think is available.
+
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -42,6 +42,9 @@ int GDKverbose = 0;
#ifdef HAVE_SYS_SYSCTL_H
# include <sys/sysctl.h>
#endif
+#if defined(HAVE_SYS_RESOURCE_H) && defined(HAVE_GETRLIMIT)
+#include <sys/resource.h>
+#endif
#ifdef __CYGWIN__
#include <sysinfoapi.h>
@@ -398,6 +401,48 @@ MT_init(void)
#else
# error "don't know how to get the amount of physical memory for your OS"
#endif
+ /* limit values to whatever cgroups gives us */
+ FILE *f;
+ /* limit of memory usage */
+ f = fopen("/sys/fs/cgroup/memory/memory.limit_in_bytes", "r");
+ if (f != NULL) {
+ uint64_t mem;
+ if (fscanf(f, "%" SCNu64, &mem) == 1
+ && mem < (uint64_t) _MT_pagesize * _MT_npages) {
+ _MT_npages = (size_t) (mem / _MT_pagesize);
+ }
+ fclose(f);
+ }
+ /* soft limit of memory usage */
+ f = fopen("/sys/fs/cgroup/memory/memory.soft_limit_in_bytes", "r");
+ if (f != NULL) {
+ uint64_t mem;
+ if (fscanf(f, "%" SCNu64, &mem) == 1
+ && mem < (uint64_t) _MT_pagesize * _MT_npages) {
+ _MT_npages = (size_t) (mem / _MT_pagesize);
+ }
+ fclose(f);
+ }
+ /* limit of memory+swap usage
+ * we use this as maximum virtual memory size */
+ f = fopen("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes", "r");
+ if (f != NULL) {
+ uint64_t mem;
+ if (fscanf(f, "%" SCNu64, &mem) == 1
+ && mem < (uint64_t) GDK_vm_maxsize) {
+ GDK_vm_maxsize = (size_t) mem;
+ }
+ fclose(f);
+ }
+#if defined(HAVE_SYS_RESOURCE_H) && defined(HAVE_GETRLIMIT) &&
defined(RLIMIT_AS)
+ struct rlimit l;
+ /* address space (virtual memory) limit */
+ if (getrlimit(RLIMIT_AS, &l) == 0
+ && l.rlim_cur != RLIM_INFINITY
+ && l.rlim_cur < GDK_vm_maxsize) {
+ GDK_vm_maxsize = l.rlim_cur;
+ }
+#endif
}
/*
diff --git a/monetdb5/mal/mal_import.c b/monetdb5/mal/mal_import.c
--- a/monetdb5/mal/mal_import.c
+++ b/monetdb5/mal/mal_import.c
@@ -43,7 +43,7 @@ slash_2_dir_sep(str fname)
}
static str
-malResolveFile(str fname)
+malResolveFile(const char *fname)
{
char path[FILENAME_MAX];
str script;
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -599,27 +599,20 @@ findVariable(MalBlkPtr mb, const char *n
}
/* The second version of findVariable assumes you have not yet
- * allocated a private structure. This is particularly usefull during
+ * allocated a private structure. This is particularly useful during
* parsing, because most variables are already defined. This way we
* safe GDKmalloc/GDKfree. */
int
findVariableLength(MalBlkPtr mb, str name, int len)
{
int i;
- int j;
- for (i = mb->vtop - 1; i >= 0; i--)
- {
- str s = mb->var[i].id;
+ for (i = mb->vtop - 1; i >= 0; i--) {
+ const char *s = getVarName(mb, i);
- j = 0;
- if (s)
- for (j = 0; j < len; j++)
- if (name[j] != s[j])
- break;
- if (j == len && s && s[j] == 0)
- return i;
- }
+ if (s && strncmp(name, s, len) == 0 && s[len] == 0)
+ return i;
+ }
return -1;
}
diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c
--- a/monetdb5/mal/mal_interpreter.c
+++ b/monetdb5/mal/mal_interpreter.c
@@ -457,7 +457,7 @@ str runMALsequence(Client cntxt, MalBlkP
int i, k;
InstrPtr pci = 0;
int exceptionVar;
- str ret = 0, localGDKerrbuf= GDKerrbuf;
+ str ret = MAL_SUCCEED, localGDKerrbuf= GDKerrbuf;
ValRecord backups[16];
ValPtr backup;
int garbages[16], *garbage;
@@ -476,11 +476,10 @@ str runMALsequence(Client cntxt, MalBlkP
pci = getInstrPtr(mb, startpc);
if (pci->argc > 16) {
backup = GDKmalloc(pci->argc * sizeof(ValRecord));
- if( backup == NULL)
- throw(MAL, "mal.interpreter", SQLSTATE(HY001)
MAL_MALLOC_FAIL);
garbage = (int*)GDKzalloc(pci->argc * sizeof(int));
- if( garbage == NULL){
+ if( backup == NULL || garbage == NULL) {
GDKfree(backup);
+ GDKfree(garbage);
throw(MAL, "mal.interpreter", SQLSTATE(HY001)
MAL_MALLOC_FAIL);
}
} else {
@@ -490,11 +489,10 @@ str runMALsequence(Client cntxt, MalBlkP
}
} else if ( mb->maxarg > 16 ){
backup = GDKmalloc(mb->maxarg * sizeof(ValRecord));
- if( backup == NULL)
- throw(MAL, "mal.interpreter", SQLSTATE(HY001)
MAL_MALLOC_FAIL);
garbage = (int*)GDKzalloc(mb->maxarg * sizeof(int));
- if( garbage == NULL){
+ if( backup == NULL || garbage == NULL) {
GDKfree(backup);
+ GDKfree(garbage);
throw(MAL, "mal.interpreter", SQLSTATE(HY001)
MAL_MALLOC_FAIL);
}
} else {
@@ -523,7 +521,7 @@ str runMALsequence(Client cntxt, MalBlkP
pci = getInstrPtr(mb, stkpc);
if (cntxt->mode == FINISHCLIENT){
stkpc = stoppc;
- if (ret == NULL)
+ if (ret == MAL_SUCCEED)
ret= createException(MAL, "mal.interpreter",
"prematurely stopped client");
break;
}
@@ -544,7 +542,8 @@ str runMALsequence(Client cntxt, MalBlkP
if (stk->cmd == 'x' ) {
stk->cmd = 0;
stkpc = mb->stop;
- continue;
+ ret= createException(MAL, "mal.interpreter",
"prematurely stopped client");
+ break;
}
}
#endif
@@ -575,6 +574,12 @@ str runMALsequence(Client cntxt, MalBlkP
}
}
+ if (cntxt->qtimeout && mb->starttime && GDKusec() -
mb->starttime > cntxt->qtimeout) {
+ freeException(ret); /* in case it's set */
+ ret = createException(MAL, "mal.interpreter",
SQLSTATE(HYT00) RUNTIME_QRY_TIMEOUT);
+ break;
+ }
+
/* The interpreter loop
* The interpreter is geared towards execution a MAL
* procedure together with all its descendant
@@ -610,7 +615,7 @@ str runMALsequence(Client cntxt, MalBlkP
}
freeException(ret);
- ret = 0;
+ ret = MAL_SUCCEED;
switch (pci->token) {
case ASSIGNsymbol:
/* Assignment command
@@ -637,8 +642,6 @@ str runMALsequence(Client cntxt, MalBlkP
} else if (lhs->vtype == TYPE_bat &&
!is_bat_nil(lhs->val.bval))
BBPretain(lhs->val.bval);
}
- freeException(ret);
- ret = 0;
break;
case PATcall:
if (pci->fcn == NULL) {
@@ -766,7 +769,7 @@ str runMALsequence(Client cntxt, MalBlkP
} else if (lhs->vtype == TYPE_bat)
BBPretain(lhs->val.bval);
}
- if(!ret) {
+ if(ret == MAL_SUCCEED) {
ret = runMALsequence(cntxt, pci->blk,
1, pci->blk->stop, nstk, stk, pci);
for (ii = 0; ii < nstk->stktop; ii++)
if
(ATOMextern(nstk->stk[ii].vtype))
@@ -786,6 +789,7 @@ str runMALsequence(Client cntxt, MalBlkP
if (pcicaller && garbageControl(getInstrPtr(mb, 0)))
garbageCollector(cntxt, mb, stk, TRUE);
if (cntxt->qtimeout && mb->starttime && GDKusec()-
mb->starttime > cntxt->qtimeout){
+ freeException(ret); /* overrule exception */
ret= createException(MAL, "mal.interpreter",
SQLSTATE(HYT00) RUNTIME_QRY_TIMEOUT);
break;
}
@@ -804,10 +808,6 @@ str runMALsequence(Client cntxt, MalBlkP
} else {
ret = createException(MAL,"interpreter",
"failed instruction2str");
}
- if (cntxt->qtimeout && mb->starttime && GDKusec()-
mb->starttime > cntxt->qtimeout){
- ret= createException(MAL, "mal.interpreter",
SQLSTATE(HYT00) RUNTIME_QRY_TIMEOUT);
- break;
- }
stkpc= mb->stop;
continue;
} }
@@ -876,7 +876,7 @@ str runMALsequence(Client cntxt, MalBlkP
/* Exception handling */
if (localGDKerrbuf && localGDKerrbuf[0]) {
- if( ret == 0)
+ if( ret == MAL_SUCCEED)
ret =
createException(MAL,"mal.interpreter",GDK_EXCEPTION);
// TODO take properly care of the GDK exception
localGDKerrbuf[0]=0;
@@ -917,17 +917,13 @@ str runMALsequence(Client cntxt, MalBlkP
exceptionVar = -1;
msg = strchr(ret, ':');
if (msg) {
- *msg = 0;
exceptionVar = findVariableLength(mb, ret,
(int)(msg - ret));
- *msg = ':';
}
if (exceptionVar == -1)
- exceptionVar = findVariableLength(mb,
"ANYexception", 12);
+ exceptionVar = findVariable(mb, "ANYexception");
/* unknown exceptions lead to propagation */
if (exceptionVar == -1) {
- if (cntxt->qtimeout && mb->starttime &&
GDKusec()- mb->starttime > cntxt->qtimeout)
- ret= createException(MAL,
"mal.interpreter", SQLSTATE(HYT00) RUNTIME_QRY_TIMEOUT);
stkpc = mb->stop;
continue;
}
@@ -941,11 +937,12 @@ str runMALsequence(Client cntxt, MalBlkP
v->vtype = TYPE_str;
v->val.sval = ret;
v->len = strlen(v->val.sval);
- ret = 0;
+ ret = MAL_SUCCEED;
MT_lock_unset(&mal_contextLock);
} else {
mnstr_printf(cntxt->fdout, "%s", ret);
freeException(ret);
+ ret = MAL_SUCCEED;
}
/* position yourself at the catch instruction for
further decisions */
/* skipToCatch(exceptionVar,@2,@3) */
@@ -1128,7 +1125,7 @@ str runMALsequence(Client cntxt, MalBlkP
case RAISEsymbol:
exceptionVar = getDestVar(pci);
//freeException(ret);
- ret = NULL;
+ ret = MAL_SUCCEED;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
