Changeset: bb647b65e1c9 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb647b65e1c9
Modified Files:
sql/backends/monet5/Tests/pyapi09.sql
sql/backends/monet5/Tests/pyapi21.sql
sql/backends/monet5/UDF/pyapi/conversion.c
Branch: default
Log Message:
Add support for blob/sqlblob conversions to Python UDFs.
diffs (truncated from 1793 to 300 lines):
diff --git a/sql/backends/monet5/Tests/pyapi09.sql
b/sql/backends/monet5/Tests/pyapi09.sql
--- a/sql/backends/monet5/Tests/pyapi09.sql
+++ b/sql/backends/monet5/Tests/pyapi09.sql
@@ -40,7 +40,7 @@ DROP TABLE pyapi09multiplication;
#Third test: Store python object in table and load it in later function
-CREATE FUNCTION pyapi09create() returns TABLE(s STRING)
+CREATE FUNCTION pyapi09create() returns TABLE(s BLOB)
language P
{
import pickle
@@ -58,7 +58,7 @@ language P
import pickle
res = _conn.execute('SELECT s FROM pyapi09objects;')
array = pickle.loads(res['s'][0])
- print array
+ print(array)
return array[:10]
};
diff --git a/sql/backends/monet5/Tests/pyapi21.sql
b/sql/backends/monet5/Tests/pyapi21.sql
--- a/sql/backends/monet5/Tests/pyapi21.sql
+++ b/sql/backends/monet5/Tests/pyapi21.sql
@@ -2,7 +2,7 @@
START TRANSACTION;
-CREATE FUNCTION pyapi21_create() returns TABLE(s STRING)
+CREATE FUNCTION pyapi21_create() returns TABLE(s BLOB)
language P
{
import pickle
@@ -18,12 +18,12 @@ language P
return result
};
-CREATE FUNCTION pyapi21_load(objects STRING) returns BOOLEAN
+CREATE FUNCTION pyapi21_load(objects BLOB) returns BOOLEAN
LANGUAGE P
{
import pickle
for x in [pickle.loads(y) for y in objects]:
- print str(type(x)) + ": " + str(x)
+ print(str(type(x)) + ": " + str(x))
return True
};
diff --git a/sql/backends/monet5/UDF/pyapi/conversion.c
b/sql/backends/monet5/UDF/pyapi/conversion.c
--- a/sql/backends/monet5/UDF/pyapi/conversion.c
+++ b/sql/backends/monet5/UDF/pyapi/conversion.c
@@ -4,6 +4,7 @@
#include "pytypes.h"
#include "type_conversion.h"
#include "unicode.h"
+#include "blob.h"
#ifndef HAVE_EMBEDDED
#include "gdk_interprocess.h"
#endif
@@ -23,899 +24,960 @@ CREATE_SQL_FUNCTION_PTR(str,batdbl_num2d
//! Wrapper to get eclass of SQL type
int GetSQLType(sql_subtype *sql_subtype);
+static bool IsBlobType(int type) {
+ return type == TYPE_blob || type == TYPE_sqlblob;
+}
+
PyObject *PyArrayObject_FromScalar(PyInput* inp, char **return_message)
{
- PyObject *vararray = NULL;
- char *msg = NULL;
- assert(inp->scalar); //input has to be a scalar
+ PyObject *vararray = NULL;
+ char *msg = NULL;
+ assert(inp->scalar); //input has to be a scalar
- switch(inp->bat_type)
- {
- case TYPE_bit:
- vararray = PyInt_FromLong((long)(*(bit*)inp->dataptr));
- break;
- case TYPE_bte:
- vararray = PyInt_FromLong((long)(*(bte*)inp->dataptr));
- break;
- case TYPE_sht:
- vararray = PyInt_FromLong((long)(*(sht*)inp->dataptr));
- break;
- case TYPE_int:
- vararray = PyInt_FromLong((long)(*(int*)inp->dataptr));
- break;
- case TYPE_lng:
- vararray = PyLong_FromLongLong((*(lng*)inp->dataptr));
- break;
- case TYPE_flt:
- vararray = PyFloat_FromDouble((double)(*(flt*)inp->dataptr));
- break;
- case TYPE_dbl:
- vararray = PyFloat_FromDouble((double)(*(dbl*)inp->dataptr));
- break;
+ switch(inp->bat_type)
+ {
+ case TYPE_bit:
+ vararray = PyInt_FromLong((long)(*(bit*)inp->dataptr));
+ break;
+ case TYPE_bte:
+ vararray = PyInt_FromLong((long)(*(bte*)inp->dataptr));
+ break;
+ case TYPE_sht:
+ vararray = PyInt_FromLong((long)(*(sht*)inp->dataptr));
+ break;
+ case TYPE_int:
+ vararray = PyInt_FromLong((long)(*(int*)inp->dataptr));
+ break;
+ case TYPE_lng:
+ vararray = PyLong_FromLongLong((*(lng*)inp->dataptr));
+ break;
+ case TYPE_flt:
+ vararray =
PyFloat_FromDouble((double)(*(flt*)inp->dataptr));
+ break;
+ case TYPE_dbl:
+ vararray =
PyFloat_FromDouble((double)(*(dbl*)inp->dataptr));
+ break;
#ifdef HAVE_HGE
- case TYPE_hge:
- vararray = PyLong_FromHge(*((hge *) inp->dataptr));
- break;
+ case TYPE_hge:
+ vararray = PyLong_FromHge(*((hge *) inp->dataptr));
+ break;
#endif
- case TYPE_str:
- vararray = PyUnicode_FromString(*((char**) inp->dataptr));
- break;
- default:
- msg = createException(MAL, "pyapi.eval", "Unsupported scalar type
%i.", inp->bat_type);
- goto wrapup;
- }
- if (vararray == NULL)
- {
- msg = createException(MAL, "pyapi.eval", "Something went wrong
converting the MonetDB scalar to a Python scalar.");
- goto wrapup;
- }
+ case TYPE_str:
+ vararray = PyUnicode_FromString(*((char**)
inp->dataptr));
+ break;
+ default:
+ msg = createException(MAL, "pyapi.eval", "Unsupported
scalar type %i.", inp->bat_type);
+ goto wrapup;
+ }
+ if (vararray == NULL)
+ {
+ msg = createException(MAL, "pyapi.eval", "Something went wrong
converting the MonetDB scalar to a Python scalar.");
+ goto wrapup;
+ }
wrapup:
- *return_message = msg;
- return vararray;
+ *return_message = msg;
+ return vararray;
}
PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
char **return_message, bool copy)
{
- BAT *b = inp->bat;
- char *msg;
- PyObject *vararray;
+ BAT *b = inp->bat;
+ char *msg;
+ PyObject *vararray;
- vararray = PyArrayObject_FromBAT(inp, t_start, t_end, return_message,
copy);
- if (vararray == NULL) {
- return NULL;
- }
- // To deal with null values, we use the numpy masked array structure
- // The masked array structure is an object with two arrays of equal size,
a data array and a mask array
- // The mask array is a boolean array that has the value 'True' when the
element is NULL, and 'False' otherwise
- // If the BAT has Null values, we construct this masked array
- if (!(b->tnil == 0 && b->tnonil == 1))
- {
- PyObject *mask;
- PyObject *mafunc =
PyObject_GetAttrString(PyImport_Import(PyString_FromString("numpy.ma")),
"masked_array");
- PyObject *maargs;
- PyObject *nullmask = PyNullMask_FromBAT(b, t_start, t_end);
+ vararray = PyArrayObject_FromBAT(inp, t_start, t_end, return_message,
copy);
+ if (vararray == NULL) {
+ return NULL;
+ }
+ // To deal with null values, we use the numpy masked array structure
+ // The masked array structure is an object with two arrays of equal
size, a data array and a mask array
+ // The mask array is a boolean array that has the value 'True' when the
element is NULL, and 'False' otherwise
+ // If the BAT has Null values, we construct this masked array
+ if (!(b->tnil == 0 && b->tnonil == 1))
+ {
+ PyObject *mask;
+ PyObject *mafunc =
PyObject_GetAttrString(PyImport_Import(PyString_FromString("numpy.ma")),
"masked_array");
+ PyObject *maargs;
+ PyObject *nullmask = PyNullMask_FromBAT(b, t_start, t_end);
- if (nullmask == Py_None) {
- maargs = PyTuple_New(1);
- PyTuple_SetItem(maargs, 0, vararray);
- } else {
- maargs = PyTuple_New(2);
- PyTuple_SetItem(maargs, 0, vararray);
- PyTuple_SetItem(maargs, 1, (PyObject*) nullmask);
- }
+ if (nullmask == Py_None) {
+ maargs = PyTuple_New(1);
+ PyTuple_SetItem(maargs, 0, vararray);
+ } else {
+ maargs = PyTuple_New(2);
+ PyTuple_SetItem(maargs, 0, vararray);
+ PyTuple_SetItem(maargs, 1, (PyObject*) nullmask);
+ }
- // Now we will actually construct the mask by calling the masked array
constructor
- mask = PyObject_CallObject(mafunc, maargs);
- if (!mask) {
- msg = createException(MAL, "pyapi.eval", "Failed to create mask");
- goto wrapup;
- }
- Py_DECREF(maargs);
- Py_DECREF(mafunc);
+ // Now we will actually construct the mask by calling the
masked array constructor
+ mask = PyObject_CallObject(mafunc, maargs);
+ if (!mask) {
+ msg = createException(MAL, "pyapi.eval", "Failed to
create mask");
+ goto wrapup;
+ }
+ Py_DECREF(maargs);
+ Py_DECREF(mafunc);
- vararray = mask;
- }
- return vararray;
+ vararray = mask;
+ }
+ return vararray;
wrapup:
- *return_message = msg;
- return NULL;
+ *return_message = msg;
+ return NULL;
}
PyObject *PyArrayObject_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
char **return_message, bool copy)
{
- // This variable will hold the converted Python object
- PyObject *vararray = NULL;
- char *msg;
- size_t j = 0;
- BUN p = 0, q = 0;
- BATiter li;
- BAT *b = inp->bat;
- npy_intp elements[1] = { t_end-t_start };
+ // This variable will hold the converted Python object
+ PyObject *vararray = NULL;
+ char *msg;
+ size_t j = 0;
+ BUN p = 0, q = 0;
+ BATiter li;
+ BAT *b = inp->bat;
+ npy_intp elements[1] = { t_end-t_start };
- assert(!inp->scalar); //input has to be a BAT
+ assert(!inp->scalar); //input has to be a BAT
- if (!b) {
- // No BAT was found, we can't do anything in this case
- msg = createException(MAL, "pyapi.eval", MAL_MALLOC_FAIL" bat.");
- goto wrapup;
- }
+ if (!b) {
+ // No BAT was found, we can't do anything in this case
+ msg = createException(MAL, "pyapi.eval", MAL_MALLOC_FAIL"
bat.");
+ goto wrapup;
+ }
- if (!IsStandardBATType(inp->bat_type) ||
ConvertableSQLType(inp->sql_subtype)) { // if the sql type is set, we have to
do some conversion
- if (inp->scalar) {
- // todo: scalar SQL types
- msg = createException(MAL, "pyapi.eval", "Scalar SQL types haven't
been implemented yet... sorry");
- goto wrapup;
- } else {
- BAT *ret_bat = NULL;
- msg = ConvertFromSQLType(inp->bat, inp->sql_subtype, &ret_bat,
&inp->bat_type);
- if (msg != MAL_SUCCEED) {
- msg = createException(MAL, "pyapi.eval", "Failed to convert
BAT.");
- goto wrapup;
- }
- BBPunfix(inp->bat->batCacheid);
- inp->bat = ret_bat;
- }
- }
+ if (!IsBlobType(inp->bat_type) && (!IsStandardBATType(inp->bat_type) ||
ConvertableSQLType(inp->sql_subtype))) { // if the sql type is set, we have to
do some conversion
+ if (inp->scalar) {
+ // FIXME: scalar SQL types
+ msg = createException(MAL, "pyapi.eval", "Scalar SQL
types haven't been implemented yet... sorry");
+ goto wrapup;
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
