Changeset: ba0540d156db for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ba0540d156db
Modified Files:
monetdb5/extras/pyapi/convert_loops.h
monetdb5/extras/pyapi/emit.c
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyloader.c
monetdb5/extras/pyapi/type_conversion.c
Branch: pythonloader
Log Message:
Now correctly recognize Python type "None" as NULL and fix issue with masked
arrays.
diffs (truncated from 402 to 300 lines):
diff --git a/monetdb5/extras/pyapi/convert_loops.h
b/monetdb5/extras/pyapi/convert_loops.h
--- a/monetdb5/extras/pyapi/convert_loops.h
+++ b/monetdb5/extras/pyapi/convert_loops.h
@@ -135,6 +135,7 @@
goto wrapup;
\
}
\
((mtpe_to*) Tloc(bat, BUNfirst(bat)))[index + iu] = value;
\
+ if (bat->T->nil == 0) bat->T->nil = value == mtpe_to##_nil ? 1 :
0; \
}
\
}
\
else
\
@@ -153,6 +154,7 @@
goto wrapup;
\
}
\
((mtpe_to*) Tloc(bat, BUNfirst(bat)))[index + iu] = value;
\
+ if (bat->T->nil == 0) bat->T->nil = value == mtpe_to##_nil ? 1
: 0; \
}
\
}
\
} }
diff --git a/monetdb5/extras/pyapi/emit.c b/monetdb5/extras/pyapi/emit.c
--- a/monetdb5/extras/pyapi/emit.c
+++ b/monetdb5/extras/pyapi/emit.c
@@ -138,8 +138,8 @@ static PyObject *
goto wrapup;
}
data = (char*) ret->array_data;
- switch (self->cols[i].b->T->type)
- {
+ assert((size_t) el_count == (size_t) ret->count);
+ switch (self->cols[i].b->T->type) {
case TYPE_bit:
NP_INSERT_BAT(self->cols[i].b, bit, self->nvals);
break;
@@ -174,6 +174,8 @@ static PyObject *
PyErr_Format(PyExc_TypeError, "Unsupported BAT Type
%s", BatType_Format(self->cols[i].b->T->type));
return NULL;
}
+ BATsetcount(self->cols[i].b, self->nvals + el_count);
+ self->cols[i].b->T->nonil = 1 - self->cols[i].b->T->nil;
}
} else {
for (ai = 0; ai < (size_t) el_count; ai++) {
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -2641,7 +2641,7 @@ ssize_t PyType_Size(PyObject *obj) {
if (PyType_IsPyScalar(obj)) {
return 1;
}
- if (PyArray_CheckExact(obj)) {
+ if (PyArray_Check(obj)) {
return PyArray_Size(obj);
}
if (PyList_Check(obj)) {
diff --git a/monetdb5/extras/pyapi/pyloader.c b/monetdb5/extras/pyapi/pyloader.c
--- a/monetdb5/extras/pyapi/pyloader.c
+++ b/monetdb5/extras/pyapi/pyloader.c
@@ -26,10 +26,9 @@ str _loader_init(void)
return msg;
}
-
str PyAPIevalLoader(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) {
sql_func * sqlfun;
- sql_subfunc * sqlmorefun;
+ sql_subfunc * sqlmorefun;
str exprStr;
const int additional_columns = 2;
@@ -82,9 +81,9 @@ str PyAPIevalLoader(Client cntxt, MalBlk
// We name all the unknown arguments
for (i = pci->retc + 2; i < argcount; i++) {
if (args[i] == NULL) {
- char argbuf[64];
- snprintf(argbuf, sizeof(argbuf), "arg%i", i - pci->retc
- 1);
- args[i] = GDKstrdup(argbuf);
+ char argbuf[64];
+ snprintf(argbuf, sizeof(argbuf), "arg%i", i - pci->retc - 1);
+ args[i] = GDKstrdup(argbuf);
}
}
gstate = Python_ObtainGIL();
@@ -105,44 +104,44 @@ str PyAPIevalLoader(Client cntxt, MalBlk
msg = createException(MAL, "pyapi.eval_loader", "Only scalar
arguments are supported.");
goto wrapup;
}
- inp.scalar = true;
- inp.bat_type = getArgType(mb, pci, i);
- inp.count = 1;
- if (inp.bat_type == TYPE_str) {
- inp.dataptr = getArgReference_str(stk, pci, i);
- }
- else {
- inp.dataptr = getArgReference(stk, pci, i);
- }
- val = PyArrayObject_FromScalar(&inp, &msg);
- if (msg != MAL_SUCCEED) {
+ inp.scalar = true;
+ inp.bat_type = getArgType(mb, pci, i);
+ inp.count = 1;
+ if (inp.bat_type == TYPE_str) {
+ inp.dataptr = getArgReference_str(stk, pci, i);
+ }
+ else {
+ inp.dataptr = getArgReference(stk, pci, i);
+ }
+ val = PyArrayObject_FromScalar(&inp, &msg);
+ if (msg != MAL_SUCCEED) {
goto wrapup;
- }
- if (PyTuple_SetItem(pArgs, ai++, val) != 0) {
+ }
+ if (PyTuple_SetItem(pArgs, ai++, val) != 0) {
msg = createException(MAL, "pyapi.eval_loader", "Failed to set
tuple (this shouldn't happen).");
goto wrapup;
- }
- // TODO deal with sql types
+ }
+ // TODO deal with sql types
}
pConnection = Py_Connection_Create(cntxt, 0, 0, 0);
- n = sqlmorefun->colnames->h;
- cols = GDKmalloc(sizeof(EmitCol) * pci->retc);
- if (!cols) {
+ n = sqlmorefun->colnames->h;
+ cols = GDKmalloc(sizeof(EmitCol) * pci->retc);
+ if (!cols) {
msg = createException(MAL, "pyapi.eval_loader", MAL_MALLOC_FAIL"column
list");
goto wrapup;
- }
- i = 0;
- while (n) {
+ }
+ i = 0;
+ while (n) {
assert(i < pci->retc);
- cols[i].name = *((char**) n->data);
- n = n->next;
- cols[i].b = BATnew(TYPE_void, getColumnType(getArgType(mb, pci,
i)), 0, TRANSIENT);
+ cols[i].name = *((char**) n->data);
+ n = n->next;
+ cols[i].b = BATnew(TYPE_void, getColumnType(getArgType(mb, pci, i)),
0, TRANSIENT);
cols[i].b->T->nil = 0;
cols[i].b->T->nonil = 0;
- i++;
- }
- pEmit = Py_Emit_Create(cols, pci->retc);
+ i++;
+ }
+ pEmit = Py_Emit_Create(cols, pci->retc);
if (!pConnection || !pEmit) {
msg = createException(MAL, "pyapi.eval_loader", MAL_MALLOC_FAIL"python
object");
@@ -158,40 +157,39 @@ str PyAPIevalLoader(Client cntxt, MalBlk
goto wrapup;
}
+ {
+ PyObject *pFunc, *pModule, *v, *d;
- {
- PyObject *pFunc, *pModule, *v, *d;
+ // First we will load the main module, this is required
+ pModule = PyImport_AddModule("__main__");
+ if (!pModule) {
+ msg = PyError_CreateException("Failed to load module", NULL);
+ goto wrapup;
+ }
- // First we will load the main module, this is required
- pModule = PyImport_AddModule("__main__");
- if (!pModule) {
- msg = PyError_CreateException("Failed to load module",
NULL);
- goto wrapup;
- }
+ // Now we will add the UDF to the main module
+ d = PyModule_GetDict(pModule);
+ if (code_object == NULL) {
+ v = PyRun_StringFlags(pycall, Py_file_input, d, d, NULL);
+ if (v == NULL) {
+ msg = PyError_CreateException("Could not parse Python code",
pycall);
+ goto wrapup;
+ }
+ Py_DECREF(v);
- // Now we will add the UDF to the main module
- d = PyModule_GetDict(pModule);
- if (code_object == NULL) {
- v = PyRun_StringFlags(pycall, Py_file_input, d, d,
NULL);
- if (v == NULL) {
- msg = PyError_CreateException("Could not parse
Python code", pycall);
- goto wrapup;
- }
- Py_DECREF(v);
-
- // Now we need to obtain a pointer to the function, the
function is called "pyfun"
- pFunc = PyObject_GetAttrString(pModule, "pyfun");
- if (!pFunc || !PyCallable_Check(pFunc)) {
- msg = PyError_CreateException("Failed to load
function", NULL);
- goto wrapup;
- }
- } else {
- pFunc = PyFunction_New(code_object, d);
- if (!pFunc || !PyCallable_Check(pFunc)) {
- msg = PyError_CreateException("Failed to load
function", NULL);
- goto wrapup;
- }
- }
+ // Now we need to obtain a pointer to the function, the function
is called "pyfun"
+ pFunc = PyObject_GetAttrString(pModule, "pyfun");
+ if (!pFunc || !PyCallable_Check(pFunc)) {
+ msg = PyError_CreateException("Failed to load function", NULL);
+ goto wrapup;
+ }
+ } else {
+ pFunc = PyFunction_New(code_object, d);
+ if (!pFunc || !PyCallable_Check(pFunc)) {
+ msg = PyError_CreateException("Failed to load function", NULL);
+ goto wrapup;
+ }
+ }
PyObject_CallObject(pFunc, pArgs);
Py_DECREF(pFunc);
@@ -207,17 +205,17 @@ str PyAPIevalLoader(Client cntxt, MalBlk
gstate = Python_ReleaseGIL(gstate);
{
- size_t nval = ((Py_EmitObject *) pEmit)->nvals;
- for (i = 0; i < pci->retc; i++) {
- BAT *b = cols[i].b;
- BATsetcount(b, nval);
+ size_t nval = ((Py_EmitObject *) pEmit)->nvals;
+ for (i = 0; i < pci->retc; i++) {
+ BAT *b = cols[i].b;
+ BATsetcount(b, nval);
b->tkey = 0;
b->tsorted = 0;
b->trevsorted = 0;
*getArgReference_bat(stk, pci, i) = b->batCacheid;
BBPkeepref(b->batCacheid);
- }
+ }
}
wrapup:
@@ -228,7 +226,7 @@ wrapup:
if (args) GDKfree(args);
if (cols) GDKfree(cols);
- // TODO: fix leaks of which there are many
+ // TODO: fix leaks of which there are many
return(msg);
}
diff --git a/monetdb5/extras/pyapi/type_conversion.c
b/monetdb5/extras/pyapi/type_conversion.c
--- a/monetdb5/extras/pyapi/type_conversion.c
+++ b/monetdb5/extras/pyapi/type_conversion.c
@@ -172,38 +172,40 @@ wrapup:
}
#ifndef IS_PY3K
-#define PY_TO_(type, inttpe)
\
-str pyobject_to_##type(PyObject **pyobj, size_t maxsize, type *value)
\
-{
\
- PyObject *ptr = *pyobj; \
- str retval = MAL_SUCCEED; \
- (void) maxsize; \
- if (PyLong_CheckExact(ptr)) {
\
- PyLongObject *p = (PyLongObject*) ptr;
\
- inttpe h = 0;
\
- inttpe prev = 0;
\
- ssize_t i = Py_SIZE(p);
\
- int sign = i < 0 ? -1 : 1;
\
- i *= sign;
\
- while (--i >= 0) {
\
- prev = h; (void)prev;
\
- h = (h << PyLong_SHIFT) + p->ob_digit[i];
\
- if ((h >> PyLong_SHIFT) != prev) {
\
- return GDKstrdup("Overflow when converting value.");
\
- }
\
- }
\
- *value = (type)(h * sign);
\
- } else if (PyInt_CheckExact(ptr) || PyBool_Check(ptr)) {
\
- *value = (type)((PyIntObject*)ptr)->ob_ival;
\
- } else if (PyFloat_CheckExact(ptr)) {
\
- *value = (type) ((PyFloatObject*)ptr)->ob_fval;
\
- } else if (PyString_CheckExact(ptr)) {
\
- return str_to_##type(((PyStringObject*)ptr)->ob_sval, -1, value); \
- } else if (PyByteArray_CheckExact(ptr)) {
\
- return str_to_##type(((PyByteArrayObject*)ptr)->ob_bytes, -1, value);\
- } else if (PyUnicode_CheckExact(ptr)) {
\
- return unicode_to_##type(((PyUnicodeObject*)ptr)->str, -1, value);
\
- }
\
+#define PY_TO_(type, inttpe)
\
+str pyobject_to_##type(PyObject **pyobj, size_t maxsize, type *value)
\
+{
\
+ PyObject *ptr = *pyobj;
\
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
