Changeset: 480b07b893e0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=480b07b893e0
Added Files:
sql/backends/monet5/Tests/pyapi17.sql
sql/backends/monet5/Tests/pyapi17.stable.err
sql/backends/monet5/Tests/pyapi17.stable.out
Modified Files:
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pytypes.h
sql/backends/monet5/Tests/All
Branch: pyapi
Log Message:
Now also accept Python dict() types as return values.
diffs (truncated from 383 to 300 lines):
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -849,9 +849,24 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
if (code_object == NULL) { PyRun_SimpleString("del pyfun"); }
- // Now we need to do some error checking on the result object, because
the result object has to have the correct type/size
- // We will also do some converting of result objects to a common type
(such as scalar -> [[scalar]])
- pResult = PyObject_CheckForConversion(pResult, pci->retc, NULL, &msg);
+ if (PyDict_Check(pResult)) { // Handle dictionary returns
+ // For dictionary returns we need to map each of the (key,value)
pairs to the proper return value
+ // We first analyze the SQL Function structure for a list of
return value names
+ char **retnames = NULL;
+ if (sqlfun != NULL) {
+ retnames = GDKzalloc(sizeof(char*) * sqlfun->res->cnt);
+ argnode = sqlfun->res->h;
+ for(i = 0; i < sqlfun->res->cnt; i++) {
+ retnames[i] = ((sql_arg*)argnode->data)->name;
+ argnode = argnode->next;
+ }
+ }
+ pResult = PyDict_CheckForConversion(pResult, pci->retc, retnames,
&msg);
+ } else {
+ // Now we need to do some error checking on the result object,
because the result object has to have the correct type/size
+ // We will also do some converting of result objects to a common
type (such as scalar -> [[scalar]])
+ pResult = PyObject_CheckForConversion(pResult, pci->retc, NULL,
&msg);
+ }
if (pResult == NULL) {
goto wrapup;
}
@@ -1835,6 +1850,54 @@ PyObject *PyNullMask_FromBAT(BAT *b, siz
}
+PyObject *PyDict_CheckForConversion(PyObject *pResult, int expected_columns,
char **retcol_names, char **return_message)
+{
+ char *msg = MAL_SUCCEED;
+ PyObject *result = PyList_New(expected_columns), *keys =
PyDict_Keys(pResult);
+ int i;
+
+ if (PyList_Size(keys) != expected_columns) {
+ if (retcol_names == NULL) {
+ msg = createException(MAL, "pyapi.eval", "Expected a dictionary
with %d return values, but a dictionary with %zu was returned instead.",
expected_columns, PyList_Size(keys));
+ goto wrapup;
+ }
+#ifdef _PYAPI_WARNINGS_
+ if (PyList_Size(keys) > expected_columns) {
+ WARNING_MESSAGE("WARNING: Expected %d return values, but a
dictionary with %zu values was returned instead.\n", expected_columns,
PyList_Size(keys));
+ }
+#endif
+ }
+
+ for(i = 0; i < expected_columns; i++) {
+ PyObject *object;
+ if (retcol_names == NULL) {
+ object = PyDict_GetItem(pResult, PyList_GetItem(keys, i));
+ } else {
+ object = PyDict_GetItemString(pResult, retcol_names[i]);
+ if (object == NULL) {
+ msg = createException(MAL, "pyapi.eval", "Expected a return
value with name \"%s\", but this key was not present in the dictionary.",
retcol_names[i]);
+ goto wrapup;
+ }
+ }
+ object = PyObject_CheckForConversion(object, 1, NULL, &msg);
+ if (object == NULL) {
+ goto wrapup;
+ }
+ if (PyList_CheckExact(object)) {
+ PyList_SetItem(result, i, PyList_GetItem(object, 0));
+ } else {
+ msg = createException(MAL, "pyapi.eval", "Why is this not a
list?");
+ goto wrapup;
+ }
+ }
+ return result;
+wrapup:
+ *return_message = msg;
+ Py_DECREF(result);
+ Py_DECREF(keys);
+ return NULL;
+}
+
PyObject *PyObject_CheckForConversion(PyObject *pResult, int expected_columns,
int *actual_columns, char **return_message)
{
char *msg;
@@ -1905,7 +1968,7 @@ PyObject *PyObject_CheckForConversion(Py
IsSingleArray = TRUE;
} else if (!PyType_IsNumpyMaskedArray(data)) {
//it is neither a python array, numpy array or numpy masked
array, thus the result is unsupported! Throw an exception!
- msg = createException(MAL, "pyapi.eval", "Unsupported result
object. Expected either an array, a numpy array, a numpy masked array or a
pandas data frame, but received an object of type \"%s\"",
PyString_AsString(PyObject_Str(PyObject_Type(data))));
+ msg = createException(MAL, "pyapi.eval", "Unsupported result
object. Expected either a list, dictionary, a numpy array, a numpy masked array
or a pandas data frame, but received an object of type \"%s\"",
PyString_AsString(PyObject_Str(PyObject_Type(data))));
goto wrapup;
}
diff --git a/monetdb5/extras/pyapi/pytypes.h b/monetdb5/extras/pyapi/pytypes.h
--- a/monetdb5/extras/pyapi/pytypes.h
+++ b/monetdb5/extras/pyapi/pytypes.h
@@ -88,7 +88,8 @@ pyapi_export PyObject *PyNullMask_FromBA
pyapi_export PyObject *PyArrayObject_FromScalar(PyInput* input_scalar, char
**return_message);
//! Creates a Numpy Masked Array from an PyInput structure containing a BAT
(essentially just combines PyArrayObject_FromBAT and PyNullMask_FromBAT)
pyapi_export PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start,
size_t t_end, char **return_message);
-
+//! Test if a PyDict object can be converted to the expected set of return
columns, by checking if the correct keys are in the dictionary and if the
values in the keys can be converted to single columns (to a single numpy array)
+pyapi_export PyObject *PyDict_CheckForConversion(PyObject *pResult, int
expected_columns, char **retcol_names, char **return_message);
//! Test if a specific PyObject can be converted to a set of
<expected_columns> BATs (or just check if they can be converted to any number
of BATs if expected_columns is smaller than 0)
pyapi_export PyObject *PyObject_CheckForConversion(PyObject *pResult, int
expected_columns, int *actual_columns, char **return_message);
//! Preprocess a PyObject (that is the result of PyObject_CheckForConversion),
pyreturn_values must be an array of PyReturn structs of size column_count
diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -14,6 +14,8 @@ HAVE_LIBPY?pyapi11
HAVE_LIBPY?pyapi12
HAVE_LIBPY?pyapi13
HAVE_LIBPY?pyapi14
+HAVE_LIBPY?pyapi16
+HAVE_LIBPY?pyapi17
#HAVE_LIBR?rapi00
#HAVE_LIBR?rapi01
diff --git a/sql/backends/monet5/Tests/pyapi17.sql
b/sql/backends/monet5/Tests/pyapi17.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi17.sql
@@ -0,0 +1,89 @@
+START TRANSACTION;
+
+# Return dictionary testing
+
+# Standard case
+CREATE FUNCTION pyapi17() returns TABLE (a STRING, b STRING, c INTEGER, d
BOOLEAN)
+language P
+{
+ retval = dict()
+ retval['a'] = ['foo']
+ retval['b'] = 'bar'
+ retval['c'] = numpy.zeros(1, dtype=numpy.int32)
+ retval['d'] = True
+ return retval
+};
+SELECT * FROM pyapi17();
+DROP FUNCTION pyapi17;
+
+# Too many keys (prints warning if warnings are enabled)
+CREATE FUNCTION pyapi17() returns TABLE (a STRING, b STRING, c INTEGER, d
BOOLEAN)
+language P
+{
+ retval = dict()
+ retval['a'] = ['foo']
+ retval['b'] = 'bar'
+ retval['c'] = numpy.zeros(1, dtype=numpy.int32)
+ retval['d'] = True
+ retval['e'] = "Unused value"
+ return retval
+};
+SELECT * FROM pyapi17();
+DROP FUNCTION pyapi17;
+
+# Keys are multidimensional arrays (cannot convert to single column, throws
error)
+CREATE FUNCTION pyapi17() returns TABLE (a STRING, b STRING, c INTEGER, d
BOOLEAN)
+language P
+{
+ retval = dict()
+ retval['a'] = [['foo'], ['hello']]
+ retval['b'] = 'bar'
+ retval['c'] = numpy.zeros(1, dtype=numpy.int32)
+ retval['d'] = True
+ return retval
+};
+SELECT * FROM pyapi17();
+ROLLBACK;
+
+START TRANSACTION;
+# Missing return value (throws error)
+CREATE FUNCTION pyapi17() returns TABLE (a STRING, b STRING, c INTEGER, d
BOOLEAN)
+language P
+{
+ retval = dict()
+ retval['a'] = ['foo']
+ retval['b'] = 'bar'
+ retval['d'] = True
+ return retval
+};
+SELECT * FROM pyapi17();
+ROLLBACK;
+
+START TRANSACTION;
+# Doesn't return a table, this means the return key must be 'result', so this
throws an error
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (3);
+CREATE FUNCTION pyapi17(i integer) returns integer
+language P
+{
+ retval = dict()
+ retval['a'] = 33
+ return retval
+};
+SELECT pyapi17(i) FROM integers;
+ROLLBACK;
+
+START TRANSACTION;
+# Doesn't return a table, now with return key as 'result'
+CREATE TABLE integers(i INTEGER);
+INSERT INTO integers VALUES (3);
+CREATE FUNCTION pyapi17(i integer) returns integer
+language P
+{
+ retval = dict()
+ retval['result'] = 33
+ return retval
+};
+SELECT pyapi17(i) FROM integers;
+DROP FUNCTION pyapi17;
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyapi17.stable.err
b/sql/backends/monet5/Tests/pyapi17.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi17.stable.err
@@ -0,0 +1,48 @@
+stderr of test 'pyapi17` in directory 'sql/backends/monet5` itself:
+
+
+# 18:29:27 >
+# 18:29:27 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=37688" "--set"
"mapi_usock=/var/tmp/mtest-4853/.s.monetdb.37688" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5" "--set"
"mal_listing=0" "--set" "embedded_r=true" "--set" "embedded_py=true"
+# 18:29:27 >
+
+# builtin opt gdk_dbpath = /home/mytherin/opt/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 37688
+# cmdline opt mapi_usock = /var/tmp/mtest-4853/.s.monetdb.37688
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbpath =
/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt mal_listing = 0
+# cmdline opt embedded_r = true
+# cmdline opt embedded_py = true
+# cmdline opt gdk_debug = 536870922
+
+# 18:29:27 >
+# 18:29:27 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-4853" "--port=37688"
+# 18:29:27 >
+
+MAPI = (monetdb) /var/tmp/mtest-5177/.s.monetdb.38806
+QUERY = SELECT * FROM pyapi17();
+ERROR = !An array of size 2 was returned, yet we expect a list of 1 columns.
The result is invalid.
+MAPI = (monetdb) /var/tmp/mtest-5177/.s.monetdb.38806
+QUERY = SELECT * FROM pyapi17();
+ERROR = !Expected a return value with name "c", but this key was not present
in the dictionary.
+MAPI = (monetdb) /var/tmp/mtest-5177/.s.monetdb.38806
+QUERY = SELECT pyapi17(i) FROM integers;
+ERROR = !Expected a return value with name "result", but this key was not
present in the dictionary.
+
+
+# 18:29:28 >
+# 18:29:28 > "Done."
+# 18:29:28 >
+
diff --git a/sql/backends/monet5/Tests/pyapi17.stable.out
b/sql/backends/monet5/Tests/pyapi17.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi17.stable.out
@@ -0,0 +1,111 @@
+stdout of test 'pyapi17` in directory 'sql/backends/monet5` itself:
+
+
+# 18:29:27 >
+# 18:29:27 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=37688" "--set"
"mapi_usock=/var/tmp/mtest-4853/.s.monetdb.37688" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5" "--set"
"mal_listing=0" "--set" "embedded_r=true" "--set" "embedded_py=true"
+# 18:29:27 >
+
+# MonetDB 5 server v11.22.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit
integers dynamically linked
+# Found 7.684 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://mytherin-N750JV:37688/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-4853/.s.monetdb.37688
+# Start processing logs sql/sql_logs version 52200
+# Finished processing logs sql/sql_logs
+# MonetDB/SQL module loaded
+# MonetDB/Python module loaded
+# MonetDB/R module loaded
+
+Ready.
+
+# 18:29:27 >
+# 18:29:27 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-4853" "--port=37688"
+# 18:29:27 >
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
