MonetDB: default - we cannot write on the (non blocked) stream w...
Changeset: 353d2d859af3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=353d2d859af3
Modified Files:
monetdb5/modules/mal/mal_mapi.c
Branch: default
Log Message:
we cannot write on the (non blocked) stream when the api requires
it to be an blocked stream. So when thread creation (and blocked
stream creation) fails only output error on console/log file, do
not report this to the client. Only solves block size error, not
the real problem of running out of threads..
diffs (13 lines):
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -384,9 +384,6 @@ SERVERlistenThread(SOCKET *Sock)
continue;
}
if (MT_create_thread(, doChallenge, data, MT_THR_JOINABLE)
< 0) {
- mnstr_printf(data->out, "!internal server error (cannot
fork new "
-"client thread), please try
again later\n");
- mnstr_flush(data->out);
showException(GDKstdout, MAL, "initClient",
"cannot fork new client
thread");
closesocket(msgsock);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - Additional defense lines for failing malloc i...
Changeset: ed8864a8a204 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ed8864a8a204
Modified Files:
gdk/gdk_firstn.c
gdk/gdk_group.c
gdk/gdk_logger.c
gdk/gdk_posix.c
gdk/gdk_project.c
gdk/gdk_system.h
gdk/gdk_value.c
Branch: Jun2016
Log Message:
Additional defense lines for failing malloc in GDK
diffs (151 lines):
diff --git a/gdk/gdk_firstn.c b/gdk/gdk_firstn.c
--- a/gdk/gdk_firstn.c
+++ b/gdk/gdk_firstn.c
@@ -663,6 +663,8 @@ BATfirstn_grouped(BAT **topn, BAT **gids
* can use the base type */
tpe = ATOMbasetype(tpe); /* takes care of wrd and oid */
groups = GDKmalloc(sizeof(*groups) * n);
+ if( groups == NULL)
+ return GDK_FAIL;
oldcand = cand;
if (asc) {
switch (tpe) {
@@ -971,6 +973,8 @@ BATfirstn_grouped_with_groups(BAT **topn
* can use the base type */
tpe = ATOMbasetype(tpe); /* takes care of wrd and oid */
groups = GDKmalloc(sizeof(*groups) * n);
+ if( groups == NULL)
+ return GDK_FAIL;
gv = (const oid *) Tloc(g, BUNfirst(g));
oldcand = cand;
if (asc) {
diff --git a/gdk/gdk_group.c b/gdk/gdk_group.c
--- a/gdk/gdk_group.c
+++ b/gdk/gdk_group.c
@@ -718,6 +718,9 @@ BATgroup_internal(BAT **groups, BAT **ex
unsigned char *restrict bgrps = GDKmalloc(256);
const unsigned char *restrict w = (const unsigned char *)
Tloc(b, BUNfirst(b));
unsigned char v;
+
+ if( bgrps == NULL)
+ goto error;
memset(bgrps, 0xFF, 256);
if (histo)
memset(cnts, 0, maxgrps * sizeof(wrd));
@@ -744,6 +747,9 @@ BATgroup_internal(BAT **groups, BAT **ex
unsigned short *restrict sgrps = GDKmalloc(65536 *
sizeof(short));
const unsigned short *restrict w = (const unsigned short *)
Tloc(b, BUNfirst(b));
unsigned short v;
+
+ if( sgrps == NULL)
+ goto error;
memset(sgrps, 0xFF, 65536 * sizeof(short));
if (histo)
memset(cnts, 0, maxgrps * sizeof(wrd));
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1227,6 +1227,9 @@ bm_subcommit(logger *lg, BAT *list_bid,
BATiter iter = (list_nme)?bat_iterator(list_nme):bat_iterator(list_bid);
gdk_return res;
+ if( n == NULL)
+ return GDK_FAIL;
+
n[i++] = 0; /* n[0] is not used */
BATloop(list_bid, p, q) {
bat col = *(log_bid *) Tloc(list_bid, p);
diff --git a/gdk/gdk_posix.c b/gdk/gdk_posix.c
--- a/gdk/gdk_posix.c
+++ b/gdk/gdk_posix.c
@@ -582,6 +582,12 @@ MT_mremap(const char *path, int mode, vo
if (fd >= 0)
close(fd);
p = malloc(strlen(path) + 5);
+ if ( p == NULL){
+ GDKsyserror("MT_mremap:
malloc() failed\n");
+ fprintf(stderr, "= %s:%d:
MT_mremap(%s,"PTRFMT","SZFMT","SZFMT"): fd < 0\n", __FILE__, __LINE__, path,
PTRFMTCAST old_address, old_size, *new_size);
+ return NULL;
+ }
+
strcat(strcpy(p, path), ".tmp");
fd = open(p, O_RDWR | O_CREAT,
MONETDB_MODE);
@@ -1048,6 +1054,8 @@ reduce_dir_name(const char *src, char *d
if (len >= cap)
buf = malloc(len + 1);
+ if( buf == NULL)
+ return NULL;
while (--len > 0 && src[len - 1] != ':' && src[len] == DIR_SEP)
;
for (buf[++len] = 0; len > 0; buf[len] = src[len])
diff --git a/gdk/gdk_project.c b/gdk/gdk_project.c
--- a/gdk/gdk_project.c
+++ b/gdk/gdk_project.c
@@ -447,6 +447,8 @@ BATprojectchain(BAT **bats)
for (n = 0; bats[n]; n++)
;
ba = GDKmalloc(sizeof(*ba) * n);
+ if( ba == NULL)
+ return NULL;
b = *bats++;
cnt = BATcount(b); /* this will be the size of the output */
hseq = b->hseqbase; /* this will be the seqbase of the output */
diff --git a/gdk/gdk_system.h b/gdk/gdk_system.h
--- a/gdk/gdk_system.h
+++ b/gdk/gdk_system.h
@@ -228,17 +228,19 @@ gdk_export ATOMIC_TYPE volatile GDKlocks
MT_Lock * volatile _p; \
/* save a copy for statistical purposes */ \
_p = GDKmalloc(sizeof(MT_Lock));\
- memcpy(_p, l,
MonetDB: iot - Merged
Changeset: 94b98320b6bc for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=94b98320b6bc Added Files: buildtools/autogen/autogen/filesplit.py monetdb5/mal/Tests/tst023.malC monetdb5/mal/Tests/tst023.stable.err monetdb5/mal/Tests/tst023.stable.out sql/backends/monet5/iot/Tests/iot01.sql sql/backends/monet5/iot/Tests/iot01.stable.err sql/backends/monet5/iot/Tests/iot01.stable.out sql/backends/monet5/iot/Tests/iot11.sql sql/backends/monet5/iot/Tests/iot11.stable.err sql/backends/monet5/iot/Tests/iot11.stable.out sql/benchmarks/tpch/Tests/02-explain.stable.out.32bit sql/benchmarks/tpch/Tests/02-plan.stable.out.32bit sql/benchmarks/tpch/Tests/13-plan.stable.out.32bit sql/benchmarks/tpch/Tests/16-plan.stable.out.32bit sql/test/BugTracker-2010/Tests/offset_limited_32bit.SF-2950579.stable.err.32bit sql/test/BugTracker-2010/Tests/offset_limited_32bit.SF-2950579.stable.out.32bit sql/test/BugTracker-2016/Tests/create_merge_table_withdata.Bug-4041.sql sql/test/BugTracker-2016/Tests/create_merge_table_withdata.Bug-4041.stable.err sql/test/BugTracker-2016/Tests/create_merge_table_withdata.Bug-4041.stable.out sql/test/BugTracker-2016/Tests/innerjoin-leftjoin-or.Bug-4049.sql sql/test/BugTracker-2016/Tests/innerjoin-leftjoin-or.Bug-4049.stable.err sql/test/BugTracker-2016/Tests/innerjoin-leftjoin-or.Bug-4049.stable.out sql/test/BugTracker-2016/Tests/simpletest.sql sql/test/BugTracker-2016/Tests/timestamp_with_tz_fromstr.Bug-4019.sql sql/test/BugTracker-2016/Tests/timestamp_with_tz_fromstr.Bug-4019.stable.err sql/test/BugTracker-2016/Tests/timestamp_with_tz_fromstr.Bug-4019.stable.out sql/test/orderidx/Tests/simpletable.stable.out.32bit sql/test/orderidx/Tests/smalltable.stable.out.32bit sql/test/pg_regress/Tests/oid.stable.out.32bit sql/test/pg_regress/Tests/without_oid.stable.out.32bit Removed Files: sql/benchmarks/tpch/Tests/01-explain.stable.out.32bit sql/benchmarks/tpch/Tests/04-explain.stable.out.32bit sql/benchmarks/tpch/Tests/13-explain.stable.out.32bit sql/benchmarks/tpch/Tests/21-explain.stable.out.32bit sql/benchmarks/tpch/Tests/22-explain.stable.out.32bit sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out.32bit sql/test/BugTracker-2010/Tests/limit_in_prepare.Bug-2552.stable.out.32bit sql/test/BugTracker-2010/Tests/prepare_stmt.Bug-2533.Bug-2534.stable.out.32bit sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out.32bit sql/test/BugTracker-2016/Tests/rename_exps.Bug-3974.stable.out.32bit sql/test/Dependencies/Tests/Dependencies.stable.out.int128 Modified Files: buildtools/autogen/autogen/am.py buildtools/autogen/autogen/codegen.py buildtools/autogen/autogen/msc.py clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.int128 clients/Tests/exports.stable.out common/stream/stream.c gdk/gdk.h gdk/gdk_aggr.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_calc.c gdk/gdk_calc.h gdk/gdk_cross.c gdk/gdk_storage.c gdk/gdk_utils.c gdk/gdk_value.c java/src/main/java/nl/cwi/monetdb/jdbc/MonetConnection.java java/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java java/src/main/java/nl/cwi/monetdb/jdbc/MonetResultSet.java java/src/main/java/nl/cwi/monetdb/jdbc/MonetStatement.java java/src/main/java/nl/cwi/monetdb/mcl/parser/HeaderLineParser.java java/src/main/java/nl/cwi/monetdb/mcl/parser/TupleLineParser.java monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.Windows.single monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out.single monetdb5/mal/Tests/All monetdb5/mal/Tests/tst031.malC monetdb5/mal/mal_builder.c monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_debugger.c monetdb5/mal/mal_debugger.h monetdb5/mal/mal_linker.c monetdb5/mal/mal_module.c monetdb5/mal/mal_profiler.c monetdb5/modules/atoms/batxml.c monetdb5/modules/atoms/json.c monetdb5/modules/kernel/algebra.c monetdb5/modules/mal/Tests/inspect05.stable.out monetdb5/modules/mal/calc.c monetdb5/modules/mal/mdb.c monetdb5/modules/mal/mkey.c monetdb5/modules/mal/tablet.c monetdb5/optimizer/Tests/manifold2.stable.out.single monetdb5/optimizer/Tests/tst4000.malC monetdb5/optimizer/Tests/tst4003.malC monetdb5/optimizer/Tests/tst4010.malC
MonetDB: iot - Updated ais tests and imports
Changeset: 287cab123693 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=287cab123693
Modified Files:
clients/iotapi/requirements.txt
clients/iotapi/src/Streams/datatypes.py
clients/iotapi/src/Streams/streampolling.py
clients/iotapi/src/Streams/streams.py
clients/iotapi/src/Streams/streamscontext.py
clients/iotapi/src/Utilities/customthreading.py
clients/iotclient/requirements.txt
clients/iotclient/src/Flask/app.py
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/Utilities/customthreading.py
clients/iotclient/src/main.py
clients/iotclient/tests/datatypesinsertstests.py
clients/iotclient/tests/main.py
sql/backends/monet5/iot/Tests/ais/Tests/ais01.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais02.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais03.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais04.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais05.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais06.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais07.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais08.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais09.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais10.sql
sql/backends/monet5/iot/Tests/ais/Tests/ais11.sql
sql/backends/monet5/iot/Tests/bug06.sql
Branch: iot
Log Message:
Updated ais tests and imports
diffs (truncated from 605 to 300 lines):
diff --git a/clients/iotapi/requirements.txt b/clients/iotapi/requirements.txt
--- a/clients/iotapi/requirements.txt
+++ b/clients/iotapi/requirements.txt
@@ -4,8 +4,8 @@ jsonschema>=2.5.1
python-dateutil>=2.5.3
python-monetdb>=11.19.3.2
pytz>=2016.4
-requests>=2.10.0
-Sphinx>=1.4.4
+requests>=2.11.1
+Sphinx>=1.4.6
sphinx-rtd-theme>=0.1.9
tornado>=3.2.2
tzlocal>=1.2.2
diff --git a/clients/iotapi/src/Streams/datatypes.py
b/clients/iotapi/src/Streams/datatypes.py
--- a/clients/iotapi/src/Streams/datatypes.py
+++ b/clients/iotapi/src/Streams/datatypes.py
@@ -1,9 +1,9 @@
import struct
-
from abc import ABCMeta, abstractmethod
from datetime import date, time, datetime
+from os import SEEK_END
+
from dateutil.relativedelta import relativedelta
-from os import SEEK_END
LITTLE_ENDIAN_ALIGNMENT = '<' # for now is little-endian for Intel CPU's
diff --git a/clients/iotapi/src/Streams/streampolling.py
b/clients/iotapi/src/Streams/streampolling.py
--- a/clients/iotapi/src/Streams/streampolling.py
+++ b/clients/iotapi/src/Streams/streampolling.py
@@ -1,12 +1,14 @@
from collections import OrderedDict, defaultdict
from json import dumps
+
+from Settings.iotlogger import add_log
+from Settings.mapiconnection import mapi_get_database_streams
+from Utilities.customthreading import PeriodicalThread
+
from .datatypes import TextType, LimitedTextType, SmallIntegerType,
HugeIntegerType, FloatType, DecimalType,\
BooleanType, DateType, TimeType, TimestampType, INetType, UUIDType,
IntervalType
from .streams import IOTStream
from .streamscontext import Streams_Context
-from Settings.iotlogger import add_log
-from Settings.mapiconnection import mapi_get_database_streams
-from Utilities.customthreading import PeriodicalThread
Switcher = [{'types': ['clob', 'url'], 'class': TextType},
{'types': ['char', 'varchar'], 'class': LimitedTextType},
diff --git a/clients/iotapi/src/Streams/streams.py
b/clients/iotapi/src/Streams/streams.py
--- a/clients/iotapi/src/Streams/streams.py
+++ b/clients/iotapi/src/Streams/streams.py
@@ -1,11 +1,11 @@
import os
+from collections import OrderedDict
-from collections import OrderedDict
-from watchdog.events import FileSystemEventHandler, DirCreatedEvent,
DirDeletedEvent
-from watchdog.observers import Observer
from Settings.filesystem import get_baskets_base_location
from Utilities.readwritelock import RWLock
from WebSockets.websockets import notify_stream_inserts_to_clients
+from watchdog.events import FileSystemEventHandler, DirCreatedEvent,
DirDeletedEvent
+from watchdog.observers import Observer
def represents_int(s):
diff --git a/clients/iotapi/src/Streams/streamscontext.py
b/clients/iotapi/src/Streams/streamscontext.py
--- a/clients/iotapi/src/Streams/streamscontext.py
+++ b/clients/iotapi/src/Streams/streamscontext.py
@@ -1,4 +1,5 @@
from collections import OrderedDict
+
from Utilities.readwritelock import RWLock
from WebSockets.websockets import unsubscribe_removed_streams
diff --git a/clients/iotapi/src/Utilities/customthreading.py
b/clients/iotapi/src/Utilities/customthreading.py
---
MonetDB: iot - Removed Flask log messages
Changeset: e5fc524d0512 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e5fc524d0512
Modified Files:
clients/iotclient/src/Flask/app.py
Branch: iot
Log Message:
Removed Flask log messages
diffs (16 lines):
diff --git a/clients/iotclient/src/Flask/app.py
b/clients/iotclient/src/Flask/app.py
--- a/clients/iotclient/src/Flask/app.py
+++ b/clients/iotclient/src/Flask/app.py
@@ -1,8 +1,12 @@
+import logging
+
from flask import Flask
from flask_restful import Api
from .restresources import StreamInput, StreamsInfo, StreamsHandling
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR) # disable logging messages to the stderr
def start_flask_iot_app(host, port):
iot_app = Flask(__name__)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - More defense lines against failing malloc
Changeset: 9b3d30af4f4d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9b3d30af4f4d
Modified Files:
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_function.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_module.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_resolve.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_stack.h
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/color.c
monetdb5/modules/atoms/identifier.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/json.c
monetdb5/modules/atoms/mtime.c
monetdb5/modules/atoms/streams.c
monetdb5/modules/kernel/algebra.c
monetdb5/modules/mal/inspect.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/tokenizer.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_scenario.c
Branch: Jun2016
Log Message:
More defense lines against failing malloc
diffs (truncated from 817 to 300 lines):
diff --git a/monetdb5/mal/mal_dataflow.c b/monetdb5/mal/mal_dataflow.c
--- a/monetdb5/mal/mal_dataflow.c
+++ b/monetdb5/mal/mal_dataflow.c
@@ -336,7 +336,11 @@ DFLOWworker(void *T)
srand((unsigned int) GDKusec());
#endif
GDKsetbuf(GDKmalloc(GDKMAXERRLEN)); /* where to leave errors */
- GDKerrbuf[0] = 0;
+ if( GDKerrbuf == 0){
+ fprintf(stderr,"DFLOWworker:Could not allocate GDKerrbuf\n");
+ GDKsetbuf(0);
+ } else
+ GDKerrbuf[0] = 0;
MT_lock_set();
cntxt = t->cntxt;
MT_lock_unset();
diff --git a/monetdb5/mal/mal_debugger.c b/monetdb5/mal/mal_debugger.c
--- a/monetdb5/mal/mal_debugger.c
+++ b/monetdb5/mal/mal_debugger.c
@@ -272,7 +272,7 @@ printCall(Client cntxt, MalBlkPtr mb, Ma
{
str msg;
msg = instruction2str(mb, stk, getInstrPtr(mb, pc), LIST_MAL_CALL);
- mnstr_printf(cntxt->fdout, "#%s at %s.%s[%d]\n", msg,
+ mnstr_printf(cntxt->fdout, "#%s at %s.%s[%d]\n", (msg?msg:"failed
instruction2str()") ,
getModuleId(getInstrPtr(mb, 0)),
getFunctionId(getInstrPtr(mb, 0)), pc);
GDKfree(msg);
@@ -287,7 +287,7 @@ printTraceCall(stream *out, MalBlkPtr mb
p = getInstrPtr(mb, pc);
msg = instruction2str(mb, stk, p, flags);
- mnstr_printf(out, "#%s%s\n", (mb->errors ? "!" : ""), msg);
+ mnstr_printf(out, "#%s%s\n", (mb->errors ? "!" : ""), msg?msg:"failed
instruction2str()");
GDKfree(msg);
}
diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c
--- a/monetdb5/mal/mal_function.c
+++ b/monetdb5/mal/mal_function.c
@@ -386,6 +386,10 @@ cloneFunction(stream *out, Module scope,
printInstruction(out,mb,0,p,LIST_MAL_ALL);
#endif
new = newFunction(scope->name, proc->name, getSignature(proc)->token);
+ if( new == NULL){
+ fprintf(stderr,"cloneFunction() failed");
+ return NULL;
+ }
freeMalBlk(new->def);
new->def = copyMalBlk(proc->def);
/* now change the definition of the original proc */
@@ -486,7 +490,7 @@ debugFunction(stream *fd, MalBlkPtr mb,
mnstr_printf(fd,"\n");
}
GDKfree(ps);
- }
+ } else mnstr_printf(fd,"#failed instruction2str()\n");
}
}
@@ -517,7 +521,7 @@ listFunction(stream *fd, MalBlkPtr mb, M
if (l > len)
len = l;
GDKfree(ps);
- }
+ } else mnstr_printf(fd,"#failed instruction2str()\n");
}
mnstr_printf(fd, "%% " SZFMT " # length\n", len);
}
diff --git a/monetdb5/mal/mal_import.c b/monetdb5/mal/mal_import.c
--- a/monetdb5/mal/mal_import.c
+++ b/monetdb5/mal/mal_import.c
@@ -290,8 +290,10 @@ evalFile(Client c, str fname, int listin
c->yycur = 0;
c->bak = NULL;
MSinitClientPrg(c, "user", "main"); /*
re-initialize context */
- MCpushClientInput(c, bstream_create(fd, 128 * BLOCK),
c->listing, "");
- msg = runScenario(c);
+ if( MCpushClientInput(c, bstream_create(fd, 128 *
BLOCK), c->listing, "") < 0){
+ msg = createException(MAL,"mal.eval", "WARNING:
could not switch client input stream\n");
+ } else
+ msg = runScenario(c);
if (msg != MAL_SUCCEED) {
dumpExceptionsToStream(c->fdout, msg);
