Changeset: ee3981264f46 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ee3981264f46
Added Files:
sql/backends/monet5/Tests/pyapi16.sql
sql/backends/monet5/Tests/pyapi16.stable.err
sql/backends/monet5/Tests/pyapi16.stable.out
Modified Files:
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pytypes.c
Branch: pyapi
Log Message:
Added _columns and _column_types parameters to PyAPI functions.
diffs (278 lines):
diff --git a/monetdb5/extras/pyapi/formatinput.c
b/monetdb5/extras/pyapi/formatinput.c
--- a/monetdb5/extras/pyapi/formatinput.c
+++ b/monetdb5/extras/pyapi/formatinput.c
@@ -251,6 +251,8 @@ char* FormatCode(char* code, char **args
bool multiline_statement = false;
int multiline_quotes = 0;
+ size_t additional_argcount = 2;
+ const char * additional_args[] = {"_columns", "_column_types"};
char base_start[] = "def pyfun(";
char base_end[] = "):\n";
*msg = NULL;
@@ -274,6 +276,9 @@ char* FormatCode(char* code, char **args
size += strlen(args[i]) + 1;
}
}
+ // Additional parameters
+ for(i = 0; i < additional_argcount; i++) size +=
strlen(additional_args[i]) + 1;
+
// First remove the "{" at the start and the "};" at the end of the
function, this is added when we have a function created through SQL and python
doesn't like them
// We need to be careful to only remove ones at the start/end, otherwise
we might invalidate some otherwise valid python code containing them
for(i = length - 1, j = 0; i > 0; i--)
@@ -426,12 +431,24 @@ char* FormatCode(char* code, char **args
for(i = 0; i < strlen(base_start); i++) {
newcode[code_location++] = base_start[i];
}
+ // Add user-defined parameters
for(i = 0; i < argcount; i++) {
if (args[i] != NULL) {
for(j = 0; j < strlen(args[i]); j++) {
newcode[code_location++] = args[i][j];
}
- if (i != argcount - 1) {
+ if (i != argcount - 1 || additional_argcount > 0) {
+ newcode[code_location++] = ',';
+ }
+ }
+ }
+ // Add additional parameters
+ for(i = 0; i < additional_argcount; i++) {
+ if (additional_args[i] != NULL) {
+ for(j = 0; j < strlen(additional_args[i]); j++) {
+ newcode[code_location++] = additional_args[i][j];
+ }
+ if (i != additional_argcount - 1) {
newcode[code_location++] = ',';
}
}
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -372,7 +372,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
BAT *b = NULL;
node * argnode;
int seengrp = FALSE;
- PyObject *pArgs = NULL, *pResult = NULL; // this is going to be the
parameter tuple
+ PyObject *pArgs = NULL, *pColumns = NULL, *pColumnTypes = NULL, *pResult =
NULL; // this is going to be the parameter tuple
PyObject *code_object = NULL;
PyReturn *pyreturn_values = NULL;
PyInput *pyinput_values = NULL;
@@ -426,18 +426,12 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
VERBOSE_MESSAGE("PyAPI Start\n");
args = (str*) GDKzalloc(pci->argc * sizeof(str));
- if (args == NULL) {
- throw(MAL, "pyapi.eval", MAL_MALLOC_FAIL);
- }
pyreturn_values = GDKzalloc(pci->retc * sizeof(PyReturn));
-
- if (pyreturn_values == NULL) {
- GDKfree(args);
+ if (args == NULL || pyreturn_values == NULL) {
throw(MAL, "pyapi.eval", MAL_MALLOC_FAIL);
}
- if ((pci->argc - (pci->retc + 2)) * sizeof(PyInput) > 0)
- {
+ if ((pci->argc - (pci->retc + 2)) * sizeof(PyInput) > 0) {
pyinput_values = GDKzalloc((pci->argc - (pci->retc + 2)) *
sizeof(PyInput));
if (pyinput_values == NULL) {
@@ -740,11 +734,13 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
// Now we will do the input handling (aka converting the input BATs to
numpy arrays)
// We will put the python arrays in a PyTuple object, we will use this
PyTuple object as the set of arguments to call the Python function
- pArgs = PyTuple_New(pci->argc - (pci->retc + 2));
+ pArgs = PyTuple_New(pci->argc - (pci->retc + 2) + 2);
+ pColumns = PyDict_New();
+ pColumnTypes = PyDict_New();
// Now we will loop over the input BATs and convert them to python objects
for (i = pci->retc + 2; i < pci->argc; i++) {
- PyObject *result_array;
+ PyObject *result_array, *arg_name, *arg_type;
// t_start and t_end hold the part of the BAT we will convert to a
Numpy array, by default these hold the entire BAT [0 - BATcount(b)]
size_t t_start = 0, t_end = pyinput_values[i - (pci->retc + 2)].count;
#ifndef WIN32
@@ -763,7 +759,6 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
}
}
#endif
-
// There are two possibilities, either the input is a BAT, or the
input is a scalar
// If the input is a scalar we will convert it to a python scalar
// If the input is a BAT, we will convert it to a numpy array
@@ -785,9 +780,16 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
}
goto wrapup;
}
+ arg_name = PyString_FromString(args[i]);
+ arg_type = PyString_FromString(BatType_Format(pyinput_values[i -
(pci->retc + 2)].bat_type));
+ PyDict_SetItem(pColumns, arg_name, result_array);
+ PyDict_SetItem(pColumnTypes, arg_name, arg_type);
+ Py_DECREF(arg_name); Py_DECREF(arg_type);
PyTuple_SetItem(pArgs, ai++, result_array);
}
+ PyTuple_SetItem(pArgs, ai++, pColumns);
+ PyTuple_SetItem(pArgs, ai++, pColumnTypes);
/*[EXECUTE_CODE]*/
VERBOSE_MESSAGE("Executing python code.\n");
@@ -833,6 +835,8 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
Py_DECREF(pFunc);
Py_DECREF(pArgs);
+ Py_DECREF(pColumns);
+ Py_DECREF(pColumnTypes);
if (PyErr_Occurred()) {
msg = PyError_CreateException("Python exception", pycall);
diff --git a/monetdb5/extras/pyapi/pytypes.c b/monetdb5/extras/pyapi/pytypes.c
--- a/monetdb5/extras/pyapi/pytypes.c
+++ b/monetdb5/extras/pyapi/pytypes.c
@@ -94,7 +94,7 @@ char *BatType_Format(int type)
case TYPE_bit: return "BIT";
case TYPE_bte: return "BYTE";
case TYPE_sht: return "SHORT";
- case TYPE_int: return "INT";
+ case TYPE_int: return "INTEGER";
case TYPE_lng: return "LONG";
case TYPE_flt: return "FLOAT";
case TYPE_dbl: return "DOUBLE";
diff --git a/sql/backends/monet5/Tests/pyapi16.sql
b/sql/backends/monet5/Tests/pyapi16.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi16.sql
@@ -0,0 +1,20 @@
+START TRANSACTION;
+
+CREATE TABLE vals(a STRING, b STRING, c STRING, d INTEGER);
+INSERT INTO vals VALUES ('foo', 'bar', '123', 33), ('t', 'e', 's', 7), ('f',
'o', 'u', 4), ('i', 'k', 'r', 149);
+
+CREATE FUNCTION pyapi16(a STRING, b string, c STRING, d INTEGER) returns table
(d boolean)
+language P
+{
+ print(_columns['a'])
+ print(_columns['b'])
+ print(_columns['c'])
+ print(_columns['d'])
+ print _column_types
+ return True
+};
+SELECT * FROM pyapi16( (SELECT * FROM vals) );
+DROP FUNCTION pyapi16;
+DROP TABLE vals;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyapi16.stable.err
b/sql/backends/monet5/Tests/pyapi16.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi16.stable.err
@@ -0,0 +1,38 @@
+stderr of test 'pyapi16` in directory 'sql/backends/monet5` itself:
+
+
+# 16:45:16 >
+# 16:45:16 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=38263" "--set"
"mapi_usock=/var/tmp/mtest-20609/.s.monetdb.38263" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5" "--set"
"mal_listing=0" "--set" "embedded_r=true" "--set" "embedded_py=true"
+# 16:45:16 >
+
+# builtin opt gdk_dbpath = /home/mytherin/opt/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 38263
+# cmdline opt mapi_usock = /var/tmp/mtest-20609/.s.monetdb.38263
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbpath =
/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt mal_listing = 0
+# cmdline opt embedded_r = true
+# cmdline opt embedded_py = true
+# cmdline opt gdk_debug = 536870922
+
+# 16:45:17 >
+# 16:45:17 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-20609" "--port=38263"
+# 16:45:17 >
+
+
+# 16:45:17 >
+# 16:45:17 > "Done."
+# 16:45:17 >
+
diff --git a/sql/backends/monet5/Tests/pyapi16.stable.out
b/sql/backends/monet5/Tests/pyapi16.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi16.stable.out
@@ -0,0 +1,62 @@
+stdout of test 'pyapi16` in directory 'sql/backends/monet5` itself:
+
+
+# 16:45:16 >
+# 16:45:16 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=38263" "--set"
"mapi_usock=/var/tmp/mtest-20609/.s.monetdb.38263" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/home/mytherin/opt/var/MonetDB/mTests_sql_backends_monet5" "--set"
"mal_listing=0" "--set" "embedded_r=true" "--set" "embedded_py=true"
+# 16:45:16 >
+
+# MonetDB 5 server v11.22.0
+# This is an unreleased version
+# Serving database 'mTests_sql_backends_monet5', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit
integers dynamically linked
+# Found 7.684 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://mytherin-N750JV:38263/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-20609/.s.monetdb.38263
+# Start processing logs sql/sql_logs version 52200
+# Finished processing logs sql/sql_logs
+# MonetDB/SQL module loaded
+# MonetDB/Python module loaded
+# MonetDB/R module loaded
+
+Ready.
+['foo' 't' 'f' 'i']
+['bar' 'e' 'o' 'k']
+['123' 's' 'u' 'r']
+[ 33 7 4 149]
+{'a': 'STRING', 'c': 'STRING', 'b': 'STRING', 'd': 'INTEGER'}
+
+# 16:45:17 >
+# 16:45:17 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-20609" "--port=38263"
+# 16:45:17 >
+
+#START TRANSACTION;
+#CREATE TABLE vals(a STRING, b STRING, c STRING, d INTEGER);
+#INSERT INTO vals VALUES ('foo', 'bar', '123', 33), ('t', 'e', 's', 7), ('f',
'o', 'u', 4), ('i', 'k', 'r', 149);
+[ 4 ]
+#CREATE FUNCTION pyapi16(a STRING, b string, c STRING, d INTEGER) returns
table (d boolean)
+#language P
+#{
+# print(_columns['a'])
+# print(_columns['b'])
+# print(_columns['c'])
+# print(_columns['d'])
+# print _column_types
+# return True
+#};
+#SELECT * FROM pyapi16( (SELECT * FROM vals) );
+% . # table_name
+% d # name
+% boolean # type
+% 5 # length
+[ true ]
+#DROP FUNCTION pyapi16;
+#DROP TABLE vals;
+#ROLLBACK;
+
+# 16:45:17 >
+# 16:45:17 > "Done."
+# 16:45:17 >
+
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
