Changeset: d8234d835b5d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d8234d835b5d
Modified Files:
sql/backends/monet5/Tests/pyloader04.stable.err
sql/backends/monet5/UDF/pyapi/emit.c
sql/backends/monet5/UDF/pyapi/pyloader.c
Branch: Dec2016
Log Message:
Proper cleanup in loader functions.
diffs (truncated from 355 to 300 lines):
diff --git a/sql/backends/monet5/Tests/pyloader04.stable.err
b/sql/backends/monet5/Tests/pyloader04.stable.err
--- a/sql/backends/monet5/Tests/pyloader04.stable.err
+++ b/sql/backends/monet5/Tests/pyloader04.stable.err
@@ -32,16 +32,18 @@ stderr of test 'pyloader04` in directory
MAPI = (monetdb) /var/tmp/mtest-6147/.s.monetdb.38930
QUERY = COPY LOADER INTO pyloader04table FROM pyloader04();
ERROR = !Python exception
+ !
! 1. def pyfun(_emit,_conn):
- ! 2. _emit.emit({'a1': 3, 'a2': 4, 'a3': 5})
- !> 3.
+ !> 2. _emit.emit({'a1': 3, 'a2': 4, 'a3': 5})
+ ! 3.
!Unmatched element "a3" in dict
MAPI = (monetdb) /var/tmp/mtest-6147/.s.monetdb.38930
QUERY = COPY LOADER INTO pyloader04table FROM pyloader04();
ERROR = !Python exception
+ !
! 1. def pyfun(_emit,_conn):
- ! 2. _emit.emit({'a1': 3, 'a2': 4, 3: 5})
- !> 3.
+ !> 2. _emit.emit({'a1': 3, 'a2': 4, 3: 5})
+ ! 3.
!Unmatched element "3" in dict
MAPI = (monetdb) /var/tmp/mtest-79373/.s.monetdb.33370
QUERY = COPY LOADER INTO pyloader04table FROM pyloader04();
@@ -74,7 +76,7 @@ ERROR = !Python exception
! 1. def pyfun(_emit,_conn):
!> 2. _emit.emit({'a1': 'hello'})
! 3.
- !Conversion Failed: Error converting string.
+ !Failed conversion: Error converting string.
# 12:30:44 >
# 12:30:44 > "Done."
diff --git a/sql/backends/monet5/UDF/pyapi/emit.c
b/sql/backends/monet5/UDF/pyapi/emit.c
--- a/sql/backends/monet5/UDF/pyapi/emit.c
+++ b/sql/backends/monet5/UDF/pyapi/emit.c
@@ -28,12 +28,12 @@
#define PythonUnicodeType Py_UNICODE
#endif
-#define scalar_convert(tpe) {\
- tpe val = (tpe) tpe##_nil; msg = pyobject_to_##tpe(&dictEntry, 42, &val); \
- BUNappend(self->cols[i].b, &val, 0); \
- if (msg != MAL_SUCCEED) { \
- PyErr_Format(PyExc_TypeError, "Conversion Failed: %s", msg); \
- return NULL; \
+#define scalar_convert(tpe) { \
+ tpe val = (tpe) tpe##_nil; \
+ msg = pyobject_to_##tpe(&dictEntry, 42, &val); \
+ if (msg != MAL_SUCCEED || BUNappend(self->cols[i].b, &val, 0) !=
GDK_SUCCEED) { \
+ if (msg == MAL_SUCCEED) msg = GDKstrdup("BUNappend failed."); \
+ goto wrapup; \
}}
@@ -43,6 +43,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
ssize_t el_count = -1; // the amount of elements this emit call will write
to the table
size_t dict_elements, matched_elements;
str msg = MAL_SUCCEED; // return message
+ bool error = false;
if (!PyDict_Check(args)) {
PyErr_SetString(PyExc_TypeError, "need dict");
@@ -93,6 +94,10 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (matched_elements != dict_elements) {
// not all elements in the dictionary were matched, look for the
element that was not matched
PyObject *keys = PyDict_Keys(args);
+ if (!keys) {
+ msg = GDKstrdup(MAL_MALLOC_FAIL);
+ goto wrapup;
+ }
for(i = 0; i < (size_t) PyList_Size(keys); i++) {
PyObject *key = PyList_GetItem(keys, i);
char *val = NULL;
@@ -113,6 +118,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (!found) {
// the current element was present in the dictionary, but
it has no matching column
PyErr_Format(PyExc_TypeError, "Unmatched element \"%s\" in
dict", val);
+ error = true;
goto loop_end;
}
}
@@ -126,7 +132,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (potential_size > self->maxcols) {
// allocate space for new columns (if any new columns show up)
sql_emit_col *old = self->cols;
- self->cols = GDKmalloc(sizeof(sql_emit_col) * potential_size);
+ self->cols = GDKzalloc(sizeof(sql_emit_col) * potential_size);
if (old) {
memcpy(self->cols, old, sizeof(sql_emit_col) * self->maxcols);
GDKfree(old);
@@ -144,6 +150,7 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (msg != MAL_SUCCEED) {
// one of the keys in the dictionary was not a string
PyErr_Format(PyExc_TypeError, "Could not convert object type
%s to a string: %s", PyString_AsString(PyObject_Str(PyObject_Type(key))), msg);
+ error = true;
Py_DECREF(keys);
goto wrapup;
}
@@ -164,7 +171,8 @@ PyEmit_Emit(PyEmitObject *self, PyObject
int bat_type = TYPE_int;
if (!array) {
PyErr_Format(PyExc_TypeError, "Failed to create NumPy
array.");
- return NULL;
+ error = true;
+ goto wrapup;
}
array_type = (PyArray_Descr*)
PyArray_DESCR((PyArrayObject*)array);
bat_type = PyType_ToBat(array_type->type_num);
@@ -175,7 +183,10 @@ PyEmit_Emit(PyEmitObject *self, PyObject
if (self->nvals > 0) {
// insert NULL values up until the current entry
for (ai = 0; ai < self->nvals; ai++) {
- BUNappend(self->cols[self->ncols].b,
ATOMnil(self->cols[self->ncols].b->ttype), 0);
+ if (BUNappend(self->cols[self->ncols].b,
ATOMnil(self->cols[self->ncols].b->ttype), 0) != GDK_SUCCEED) {
+ msg = GDKstrdup("BUNappend failed.");
+ goto wrapup;
+ }
}
self->cols[i].b->tnil = 1;
self->cols[i].b->tnonil = 0;
@@ -225,19 +236,23 @@ PyEmit_Emit(PyEmitObject *self, PyObject
{
str val = NULL;
msg = pyobject_to_str(&dictEntry, 42, &val);
- BUNappend(self->cols[i].b, val, 0);
- if (val) {
- free(val);
+ gdk_return retval;
+ if (msg != MAL_SUCCEED) {
+ goto wrapup;
}
- if (msg != MAL_SUCCEED) {
- PyErr_Format(PyExc_TypeError, "Conversion Failed:
%s", msg);
- return NULL;
+ assert(val);
+ retval = BUNappend(self->cols[i].b, val, 0);
+ free(val);
+ if (retval != GDK_SUCCEED) {
+ msg = GDKstrdup("BUNappend failed.");
+ goto wrapup;
}
}
break;
default:
PyErr_Format(PyExc_TypeError, "Unsupported BAT Type
%s", BatType_Format(self->cols[i].b->ttype));
- return NULL;
+ error = true;
+ goto wrapup;
}
} else {
bool *mask = NULL;
@@ -247,17 +262,18 @@ PyEmit_Emit(PyEmitObject *self, PyObject
size_t index_offset = 0;
size_t iu = 0;
if (BATextend(self->cols[i].b, self->nvals + el_count) !=
GDK_SUCCEED) {
- PyErr_Format(PyExc_TypeError, "Failed to allocate memory
to extend BAT.");
- return NULL;
+ msg = GDKstrdup("Failed to allocate memory to extend
BAT.");
+ goto wrapup;
}
msg = PyObject_GetReturnValues(dictEntry, ret);
- if (ret->mask_data != NULL) {
- mask = (bool*) ret->mask_data;
+ if (msg != MAL_SUCCEED) {
+ goto wrapup;
}
if (ret->array_data == NULL) {
- msg = createException(MAL, "pyapi.eval", "No return value
stored in the structure.\n");
+ msg = GDKstrdup("No return value stored in the
structure.");
goto wrapup;
}
+ mask = (bool*) ret->mask_data;
data = (char*) ret->array_data;
assert((size_t) el_count == (size_t) ret->count);
switch (self->cols[i].b->ttype) {
@@ -303,13 +319,16 @@ PyEmit_Emit(PyEmitObject *self, PyObject
break;
default:
PyErr_Format(PyExc_TypeError, "Unsupported BAT Type
%s", BatType_Format(self->cols[i].b->ttype));
- return NULL;
+ error = true;
+ goto wrapup;
}
self->cols[i].b->tnonil = 1 - self->cols[i].b->tnil;
}
} else {
for (ai = 0; ai < (size_t) el_count; ai++) {
- BUNappend(self->cols[i].b, ATOMnil(self->cols[i].b->ttype), 0);
+ if (BUNappend(self->cols[i].b,
ATOMnil(self->cols[i].b->ttype), 0) != GDK_SUCCEED) {
+ goto wrapup;
+ }
}
self->cols[i].b->tnil = 1;
self->cols[i].b->tnonil = 0;
@@ -318,11 +337,10 @@ PyEmit_Emit(PyEmitObject *self, PyObject
}
self->nvals += el_count;
- Py_RETURN_NONE;
wrapup:
if (msg != MAL_SUCCEED) {
PyErr_Format(PyExc_TypeError, "Failed conversion: %s", msg);
- } else {
+ } else if (!error) {
Py_RETURN_NONE;
}
return NULL;
diff --git a/sql/backends/monet5/UDF/pyapi/pyloader.c
b/sql/backends/monet5/UDF/pyapi/pyloader.c
--- a/sql/backends/monet5/UDF/pyapi/pyloader.c
+++ b/sql/backends/monet5/UDF/pyapi/pyloader.c
@@ -72,9 +72,8 @@ str PyAPIevalLoader(Client cntxt, MalBlk
sqlfun = sqlmorefun->func;
exprStr = *getArgReference_str(stk, pci, pci->retc + 1);
-
args = (str*) GDKzalloc(pci->argc * sizeof(str));
- if (args == NULL) {
+ if (!args) {
throw(MAL, "pyapi.eval", MAL_MALLOC_FAIL" arguments.");
}
@@ -91,7 +90,7 @@ str PyAPIevalLoader(Client cntxt, MalBlk
// We name all the unknown arguments
for (i = pci->retc + 2; i < argcount; i++) {
- if (args[i] == NULL) {
+ if (!args[i]) {
char argbuf[64];
snprintf(argbuf, sizeof(argbuf), "arg%i", i - pci->retc - 1);
args[i] = GDKstrdup(argbuf);
@@ -135,10 +134,9 @@ str PyAPIevalLoader(Client cntxt, MalBlk
// TODO deal with sql types
}
- pConnection = Py_Connection_Create(cntxt, 0, 0, 0);
if (sqlmorefun->colnames) {
n = sqlmorefun->colnames->h;
- cols = GDKmalloc(sizeof(sql_emit_col) * pci->retc);
+ cols = GDKzalloc(sizeof(sql_emit_col) * pci->retc);
if (!cols) {
msg = createException(MAL, "pyapi.eval_loader",
MAL_MALLOC_FAIL"column list");
goto wrapup;
@@ -146,7 +144,7 @@ str PyAPIevalLoader(Client cntxt, MalBlk
i = 0;
while (n) {
assert(i < pci->retc);
- cols[i].name = *((char**) n->data);
+ cols[i].name = GDKstrdup(*((char**) n->data));
n = n->next;
cols[i].b = COLnew(0, getBatType(getArgType(mb, pci, i)), 0,
TRANSIENT);
cols[i].b->tnil = 0;
@@ -160,8 +158,9 @@ str PyAPIevalLoader(Client cntxt, MalBlk
retvals = 0;
create_table = true;
}
+
+ pConnection = Py_Connection_Create(cntxt, 0, 0, 0);
pEmit = PyEmit_Create(cols, retvals);
-
if (!pConnection || !pEmit) {
msg = createException(MAL, "pyapi.eval_loader", MAL_MALLOC_FAIL"python
object");
goto wrapup;
@@ -171,8 +170,10 @@ str PyAPIevalLoader(Client cntxt, MalBlk
PyTuple_SetItem(pArgs, ai++, pConnection);
pycall = FormatCode(exprStr, args, argcount, 4, &code_object, &msg,
loader_additional_args, additional_columns);
- if (pycall == NULL && code_object == NULL) {
- if (msg == NULL) { msg = createException(MAL, "pyapi.eval_loader",
"Error while parsing Python code."); }
+ if (!pycall && !code_object) {
+ if (msg == MAL_SUCCEED) {
+ msg = createException(MAL, "pyapi.eval_loader", "Error while
parsing Python code.");
+ }
goto wrapup;
}
@@ -213,7 +214,6 @@ str PyAPIevalLoader(Client cntxt, MalBlk
if (PyErr_Occurred()) {
Py_DECREF(pFunc);
- Py_DECREF(pArgs);
msg = PyError_CreateException("Python exception", pycall);
if (code_object == NULL) { PyRun_SimpleString("del pyfun"); }
goto wrapup;
@@ -222,7 +222,6 @@ str PyAPIevalLoader(Client cntxt, MalBlk
if (ret != Py_None) {
if (PyEmit_Emit((PyEmitObject *) pEmit, ret) == NULL) {
Py_DECREF(pFunc);
- Py_DECREF(pArgs);
msg = PyError_CreateException("Python exception", pycall);
goto wrapup;
}
@@ -233,6 +232,7 @@ str PyAPIevalLoader(Client cntxt, MalBlk
retvals = (int) ((PyEmitObject *) pEmit)->ncols;
Py_DECREF(pFunc);
Py_DECREF(pArgs);
+ pArgs = NULL;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
