Changeset: 02f7e275d499 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=02f7e275d499
Modified Files:
monetdb5/extras/pyapi/Tests/pyapi_modify_input.stable.err
monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/type_conversion.c
monetdb5/extras/pyapi/type_conversion.h
monetdb5/extras/pyapi/unicode.c
monetdb5/extras/pyapi/unicode.h
sql/backends/monet5/Tests/pyapi10.stable.err
sql/backends/monet5/Tests/pyapi10.stable.out
sql/backends/monet5/Tests/pyapi11.stable.out
Branch: pyapi
Log Message:
Fix for different compilers.
diffs (truncated from 468 to 300 lines):
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_modify_input.stable.err
b/monetdb5/extras/pyapi/Tests/pyapi_modify_input.stable.err
--- a/monetdb5/extras/pyapi/Tests/pyapi_modify_input.stable.err
+++ b/monetdb5/extras/pyapi/Tests/pyapi_modify_input.stable.err
@@ -31,23 +31,23 @@ stderr of test 'pyapi_modify_input` in d
# 12:27:29 > "mclient" "-lmal" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-30203" "--port=31030"
# 12:27:29 >
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = rstr:bat[:oid,:str] :=
pyapi.eval(nil:ptr,"arg1[0][0]='d'\nreturn(arg1)",bstr);
ERROR = !MALException:pyapi.eval:Python exception
!This ByteArray references to a BAT in the database, you may not
assign to it.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = rstr:bat[:oid,:str] :=
pyapi.eval(nil:ptr,"arg1[0].reverse()\nreturn(arg1)",bstr);
ERROR = !MALException:pyapi.eval:Python exception
!This ByteArray references to a BAT in the database, you may not
reverse it.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = rstr:bat[:oid,:str] := pyapi.eval(nil:ptr,"a =
arg1[0].remove('a')\nreturn(arg1)",bstr);
ERROR = !MALException:pyapi.eval:Python exception
!This ByteArray references to a BAT in the database, you may not
remove anything from it.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = rstr:bat[:oid,:str] := pyapi.eval(nil:ptr,"a =
arg1[0].pop()\nreturn(arg1)",bstr);
ERROR = !MALException:pyapi.eval:Python exception
!This ByteArray references to a BAT in the database, you may not
remove anything from it.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = result:bat[:oid,:int] := pyapi.eval(nil:ptr,
"arg1[0]=4\nreturn(arg1)", a);
ERROR = !MALException:pyapi.eval:Python exception
!assignment destination is read-only
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
b/monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
--- a/monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
+++ b/monetdb5/extras/pyapi/Tests/pyapi_numpy_numeric_nested.stable.out
@@ -30,27 +30,27 @@ Ready.
# h t # name
# void dbl # type
#--------------------------#
-[ 0@0, 1 ]
-[ 1@0, 11 ]
-[ 2@0, 4 ]
-[ 3@0, 67 ]
-[ 4@0, 92 ]
+[ 0@0, 1.96 ]
+[ 1@0, 11.56 ]
+[ 2@0, 4.41 ]
+[ 3@0, 67.24 ]
+[ 4@0, 92.16 ]
#io.print(a);
#--------------------------#
# h t # name
# void dbl # type
#--------------------------#
-[ 0@0, 1 ]
-[ 1@0, 3 ]
-[ 2@0, 2 ]
-[ 3@0, 8 ]
-[ 4@0, 9 ]
+[ 0@0, 1.4 ]
+[ 1@0, 3.4 ]
+[ 2@0, 2.1 ]
+[ 3@0, 8.2 ]
+[ 4@0, 9.6 ]
#io.print(x);
#--------------------------#
# h t # name
# void dbl # type
#--------------------------#
-[ 0@0, 546183377 ]
+[ 0@0, 546183377.8 ]
#io.print(s);
#--------------------------#
# h t # name
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
--- a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
+++ b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.stable.err
@@ -31,34 +31,34 @@ stderr of test 'pyapi_returntypes` in di
# 23:33:07 > "mclient" "-lmal" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-20340" "--port=36739"
# 23:33:07 >
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = r:bat[:oid,:int] := pyapi.eval(nil:ptr,"return(\"Test\")");
ERROR = !MALException:pyapi.eval:Could not convert from type STRING to type int
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) := pyapi.eval(nil:ptr,"class
NewClass:\n x = 5\n\nreturn(NewClass())");
ERROR = !MALException:pyapi.eval:Unsupported result object. Expected either an
array, a numpy array, a numpy masked array or a pandas data frame, but received
an object of type "<type 'instance'>"
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) :=
pyapi.eval(nil:ptr,"return(12)");
ERROR = !MALException:pyapi.eval:A single scalar was returned, yet we expect a
list of 2 columns. We can only convert a single scalar into a single column,
thus the result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) :=
pyapi.eval(nil:ptr,"return(numpy.array([12]))");
ERROR = !MALException:pyapi.eval:A single array was returned, yet we expect a
list of 2 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = (r:bat[:oid,:int], s:bat[:oid,:int]) :=
pyapi.eval(nil:ptr,"return([12])");
ERROR = !MALException:pyapi.eval:A single array was returned, yet we expect a
list of 2 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = r:bat[:oid,:int] := pyapi.eval(nil:ptr,"return([[33,24,55],
[44,66,345]])");
ERROR = !MALException:pyapi.eval:An array of size 2 was returned, yet we
expect a list of 1 columns. The result is invalid.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = a:bat[:oid,:str] := pyapi.eval(nil:ptr,"x =
unicode(\"hello\")\nreturn(x.encode(\"utf32\"))");
ERROR = !MALException:pyapi.eval:Invalid string encoding used. Please return a
regular ASCII string, or a Numpy_Unicode object.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = bb:bat[:oid,:int] := pyapi.eval(nil:ptr,"return (1");
ERROR = !MALException:pyapi.eval:Could not parse Python code
! 1. def pyfun():
!> 2. return (1
!invalid syntax (<string>, line 2)
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = cc:bat[:oid,:int] := pyapi.eval(nil:ptr,"x = 4\n x++\n\treturn (x)");
ERROR = !MALException:pyapi.eval:Could not parse Python code
! 1. def pyfun():
@@ -66,10 +66,10 @@ ERROR = !MALException:pyapi.eval:Could n
!> 3. x++
! 4. return (x)
!unexpected indent (<string>, line 3)
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = (str1:bat[:oid,:str], str2:bat[:oid,:str]) := pyapi.eval(nil:ptr,"x =
unicode(\"hello\")\nreturn(numpy.array([[x.encode(\"utf32\")],
[x.encode(\"utf32\")]]))");
ERROR = !MALException:pyapi.eval:Invalid string encoding used. Please return a
regular ASCII string, or a Numpy_Unicode object.
-MAPI = (monetdb) /var/tmp/mtest-30203/.s.monetdb.31030
+MAPI = (monetdb) /var/tmp/mtest-10930/.s.monetdb.36894
QUERY = (str1:bat[:oid,:str], str2:bat[:oid,:str]) :=
pyapi.eval(nil:ptr,"return(numpy.array([[\"Hëllo\", \"Hello Again\"], [\"Hello
Again Again\",\"That's quite enough.\"]]))");
ERROR = !MALException:pyapi.eval:Invalid string encoding used. Please return a
regular ASCII string, or a Numpy_Unicode object.
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
b/monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
--- a/monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
+++ b/monetdb5/extras/pyapi/Tests/pyapi_types_huge.stable.err
@@ -26,14 +26,6 @@ stderr of test 'pyapi_types_huge` in dir
# cmdline opt embedded_r = true
# cmdline opt embedded_py = true
# cmdline opt gdk_debug = 536870922
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
# 00:29:47 >
# 00:29:47 > "mclient" "-lmal" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-30800" "--port=32717"
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
b/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
--- a/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
+++ b/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.err
@@ -26,7 +26,6 @@ stderr of test 'pyapi_types_numeric` in
# cmdline opt embedded_r = true
# cmdline opt embedded_py = true
# cmdline opt gdk_debug = 536870922
-!PERFORMANCE WARNING: Type "hge" (128 bit) is unsupported by Numpy. The
numbers are instead converted to python objects of type "PyLong". This means a
python object is constructed for every huge integer and the entire column is
copied.
# 19:41:50 >
# 19:41:50 > "mclient" "-lmal" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-7261" "--port=31934"
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
b/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
--- a/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
+++ b/monetdb5/extras/pyapi/Tests/pyapi_types_numeric.stable.out
@@ -116,10 +116,10 @@ Ready.
# h t # name
# void dbl # type
#--------------------------#
-[ 0@0, 180428 ]
-[ 1@0, 84 ]
-[ 2@0, 168169 ]
-[ 3@0, 17146369 ]
+[ 0@0, 180428.9383 ]
+[ 1@0, 84.6930886 ]
+[ 2@0, 168169.2777 ]
+[ 3@0, 17146369.15 ]
[ 4@0, nil ]
#io.print(rint);
#--------------------------#
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -30,6 +30,7 @@
#include "type_conversion.h"
//#define _PYAPI_VERBOSE_
+//#define _PYAPI_WARNINGS_
#define _PYAPI_DEBUG_
#include <stdint.h>
@@ -59,6 +60,16 @@ const char* debug_enableflag = "enable_p
#define VERBOSE_MESSAGE(...) ((void) 0)
#endif
+#ifdef _PYAPI_WARNINGS_
+#define WARNING_MESSAGE(...) { \
+ fprintf(stderr, __VA_ARGS__); \
+ fflush(stdout); \
+}
+#else
+#define WARNING_MESSAGE(...) ((void) 0)
+#endif
+
+
#define GDK_Alloc(var, size) { \
var = GDKzalloc(size); \
if (var == NULL) { \
@@ -274,13 +285,13 @@ static int pyapiInitialized = FALSE;
if (ret->numpy_array == NULL)
\
{
\
/*shared memory return*/
\
- VERBOSE_MESSAGE("- Shared memory map!\n");
\
+ VERBOSE_MESSAGE("- Zero copy (shared memory)!\n");
\
BAT_MMAP(bat, mtpe, STORE_SHARED);
\
ret->array_data = NULL;
\
}
\
else
\
{
\
- VERBOSE_MESSAGE("- Memory map!\n");
\
+ VERBOSE_MESSAGE("- Zero copy!\n");
\
BAT_MMAP(bat, mtpe, STORE_CMEM);
\
}
\
}
\
@@ -288,6 +299,8 @@ static int pyapiInitialized = FALSE;
{
\
bat = BATnew(TYPE_void, TYPE_##mtpe, ret->count, TRANSIENT);
\
BATseqbase(bat, seqbase); bat->T->nil = 0; bat->T->nonil = 1;
\
+ if (TYPE_##mtpe != TYPE_hge) WARNING_MESSAGE("!PERFORMANCE
WARNING: You are returning a Numpy Array of type %s, which has to be converted
to a BAT of type %s. If you return a Numpy\
+Array of type %s no copying will be needed.\n",
PyType_Format(ret->result_type), BatType_Format(TYPE_##mtpe),
PyType_Format(BatType_ToPyType(TYPE_##mtpe))); \
bat->tkey = 0; bat->tsorted = 0; bat->trevsorted = 0;
\
switch(ret->result_type)
\
{
\
@@ -948,8 +961,6 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
goto wrapup;
}
PyArray_SETITEM((PyArrayObject*)vararray,
PyArray_GETPTR1((PyArrayObject*)vararray, j), obj);
- //PyObject *obj2 =
PyArray_GETITEM((PyArrayObject*)vararray,
PyArray_GETPTR1((PyArrayObject*)vararray, j));
- //printf("%s\n", (PyStringObject*)obj2)
}
if (j == t_end) break;
j++;
@@ -975,7 +986,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
NULL);
j = 0;
- fprintf(stderr, "!PERFORMANCE WARNING: Type \"hge\" (128 bit)
is unsupported by Numpy. The numbers are instead converted to python objects of
type \"PyLong\". This means a python object is constructed for every huge
integer and the entire column is copied.\n");
+ WARNING_MESSAGE("!PERFORMANCE WARNING: Type \"hge\" (128 bit)
is unsupported by Numpy. The numbers are instead converted to python objects of
type \"PyLong\". This means a python object is constructed for every huge
integer and the entire column is copied.\n");
BATloop(b, p, q) {
PyObject *obj;
const hge *t = (const hge *) BUNtail(li, p);
diff --git a/monetdb5/extras/pyapi/type_conversion.c
b/monetdb5/extras/pyapi/type_conversion.c
--- a/monetdb5/extras/pyapi/type_conversion.c
+++ b/monetdb5/extras/pyapi/type_conversion.c
@@ -205,7 +205,7 @@ bool s_to_dbl(char *ptr, size_t size, db
}
-bool utf32_to_lng(uint32_t *utf32, lng *value)
+bool utf32_to_lng(Py_UNICODE *utf32, lng *value)
{
size_t length = utf32_strlen(utf32);
int i = length;
@@ -236,7 +236,7 @@ bool utf32_to_lng(uint32_t *utf32, lng *
return true;
}
-bool utf32_to_dbl(uint32_t *utf32, dbl *value)
+bool utf32_to_dbl(Py_UNICODE *utf32, dbl *value)
{
size_t length = utf32_strlen(utf32);
int i = length;
@@ -268,7 +268,7 @@ bool utf32_to_dbl(uint32_t *utf32, dbl *
}
#ifdef HAVE_HGE
-bool utf32_to_hge(uint32_t *utf32, hge *value)
+bool utf32_to_hge(Py_UNICODE *utf32, hge *value)
{
size_t length = utf32_strlen(utf32);
int i = length;
@@ -399,7 +399,7 @@ bool py_to_dbl(PyObject *ptr, dbl *value
{ \
strval val; \
(void) size; \
- if (!utf32_to_##strval((uint32_t*)ptr, &val)) return false; \
+ if (!utf32_to_##strval((Py_UNICODE*)ptr, &val)) return false; \
*value = (tpe)val; \
return true; \
} \
diff --git a/monetdb5/extras/pyapi/type_conversion.h
b/monetdb5/extras/pyapi/type_conversion.h
--- a/monetdb5/extras/pyapi/type_conversion.h
+++ b/monetdb5/extras/pyapi/type_conversion.h
@@ -43,11 +43,11 @@ bool s_to_dbl(char *ptr, size_t size, db
//! Converts a base-10 string to a lng value
bool s_to_lng(char *ptr, size_t size, lng *value);
//! Converts a base-10 utf32-encoded string to a lng value
-bool utf32_to_lng(uint32_t *utf32, lng *value);
+bool utf32_to_lng(Py_UNICODE *utf32, lng *value);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
