Changeset: 3bfa5b88a360 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3bfa5b88a360
Added Files:
monetdb5/extras/pyapi/lazyarray.c
monetdb5/extras/pyapi/lazyarray.h
Modified Files:
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pytypes.c
monetdb5/extras/pyapi/pytypes.h
Branch: pyapi
Log Message:
Added PyLazyArrayObject (custom array type that defers loading from BAT until
necessary). [WIP, testing only right now]
diffs (truncated from 1511 to 300 lines):
diff --git a/monetdb5/extras/pyapi/Makefile.ag
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -17,7 +17,7 @@ MTSAFE
lib__pyapi = {
MODULE
DIR = libdir/monetdb5
- SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h bytearray.c bytearray.h formatinput.c
formatinput.h benchmark.c benchmark.h
+ SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h bytearray.c bytearray.h formatinput.c
formatinput.h benchmark.c benchmark.h lazyarray.c lazyarray.h
XDEPS = $(libpy_LIBDEP)
LIBS = ../../tools/libmonetdb5 \
../../../gdk/libbat \
diff --git a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi_returntypes.malC
@@ -76,8 +76,8 @@ bat.append(bstr,"héllo":str);
bat.append(bstr,"éáú üüäãö":str);
bat.append(bstr,"幺巾乡阜阝测试一些中国符号":str);
bat.append(bstr,"いくつかの日本のシンボルをテストします":str);
-(r:bat[:oid,:str], s:bat[:oid,:str]) :=
pyapi.eval(nil:ptr,"return(numpy.array([arg1, arg1]))",bstr);
-io.print(r,s);
+(z123:bat[:oid,:str], z234:bat[:oid,:str]) :=
pyapi.eval(nil:ptr,"return(numpy.array([arg1, arg1]))",bstr);
+io.print(z123,z234);
# return a multi-dimensional numpy array containing integers, but store them
in a string BAT
(r:bat[:oid,:str], s:bat[:oid,:str]) :=
pyapi.eval(nil:ptr,"return(numpy.array([[33, 24, 55], [44, 66,345]]))");
diff --git a/monetdb5/extras/pyapi/formatinput.c
b/monetdb5/extras/pyapi/formatinput.c
--- a/monetdb5/extras/pyapi/formatinput.c
+++ b/monetdb5/extras/pyapi/formatinput.c
@@ -177,13 +177,13 @@ PyObject *PyCodeObject_ParseString(char
int argcount, nlocals, stacksize, flags, firstlineno;
PyObject *code, *name, *filename, *lnotab;
PyObject *consts, *names, *varnames, *freevars, *cellvars;
+ size_t size;
char *temp_string = GDKmalloc(strlen(string));
char *temp_string2 = GDKmalloc(strlen(string));
if (temp_string == NULL || temp_string2 == NULL) {
*msg = createException(MAL, "pyapi.eval", MAL_MALLOC_FAIL);
return NULL;
}
- size_t size;
//argcount is a single int
argcount = atoi(GetArg(string, temp_string, 0));
diff --git a/monetdb5/extras/pyapi/lazyarray.c
b/monetdb5/extras/pyapi/lazyarray.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/pyapi/lazyarray.c
@@ -0,0 +1,553 @@
+
+#include "lazyarray.h"
+#include "pytypes.h"
+#include "type_conversion.h"
+
+// Numpy Library
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+#ifdef __INTEL_COMPILER
+// Intel compiler complains about trailing comma's in numpy source code, so
hopefully this works
+#pragma warning(disable:271)
+#endif
+#include <numpy/arrayobject.h>
+
+
+void lazyarray_init(void)
+{
+ import_array();
+}
+
+#define LAZYARRAY_BAT(op) ((BAT*)((PyLazyArrayObject*)op)->bat)
+#define LAZYARRAY_GETNUMERICITEM(op, tpe, index)
((tpe*)LAZYARRAY_BAT(op)->T->heap.base)[index]
+#define LAZYARRAY_GETITEMPTR(op, index)
((LAZYARRAY_BAT(op)->T->heap.base)[index * LAZYARRAY_BAT(op)->T->width])
+
+char* PyLazyArray_GetString(PyLazyArrayObject *a, Py_ssize_t index)
+{
+ return LAZYARRAY_BAT(a)->T->vheap->base +
VarHeapVal(LAZYARRAY_BAT(a)->T->heap.base, index, LAZYARRAY_BAT(a)->T->width);
+}
+
+hge PyLazyArray_GetHuge(PyLazyArrayObject *a, Py_ssize_t index)
+{
+ return ((hge*)LAZYARRAY_BAT(a)->T->heap.base)[index];
+}
+
+bool PyLazyArray_IsNil(PyLazyArrayObject *a, Py_ssize_t index)
+{
+ if (LAZYARRAY_BAT(a)->T->nonil) return false;
+
+ if (PyLazyArray_GET_TYPE(a) == TYPE_str) {
+ return strcmp(PyLazyArray_GetString(a, index), str_nil) == 0;
+ }
+ else {
+ const void *nil = ATOMnilptr(LAZYARRAY_BAT(a)->ttype);
+ int (*atomcmp)(const void *, const void *) =
ATOMcompare(LAZYARRAY_BAT(a)->ttype);
+ return (*atomcmp)(&LAZYARRAY_GETITEMPTR(a, index), nil) == 0;
+ }
+}
+
+PyObject* PyLazyArray_GetItem(PyLazyArrayObject *a, Py_ssize_t index)
+{
+ if (a->array != NULL && a->array[index] != NULL) {
+ Py_INCREF(a->array[index]);
+ return a->array[index];
+ }
+ if (a->array == NULL) {
+ a->array = GDKzalloc(sizeof(PyObject*) * Py_SIZE(a));
+ }
+
+ if (PyLazyArray_IsNil(a, index)) {
+ a->array[index] = Py_None;
+ Py_RETURN_NONE;
+ }
+
+ if (PyLazyArray_GET_TYPE(a) == TYPE_str) {
+ PyObject *obj = PyUnicode_FromString(PyLazyArray_GetString(a, index));
+ a->array[index] = obj;
+ Py_INCREF(obj);
+ return obj;
+ }
+ {
+ PyObject *obj = NULL;
+ switch (PyLazyArray_GET_TYPE(a))
+ {
+ case TYPE_bte: obj = SCALAR_TO_PYSCALAR(bte,
LAZYARRAY_GETNUMERICITEM(a, bte, index)); break;
+ case TYPE_bit: obj = SCALAR_TO_PYSCALAR(bit,
LAZYARRAY_GETNUMERICITEM(a, bit, index)); break;
+ case TYPE_sht: obj = SCALAR_TO_PYSCALAR(sht,
LAZYARRAY_GETNUMERICITEM(a, sht, index)); break;
+ case TYPE_int: obj = SCALAR_TO_PYSCALAR(int,
LAZYARRAY_GETNUMERICITEM(a, int, index)); break;
+ case TYPE_lng: obj = SCALAR_TO_PYSCALAR(lng,
LAZYARRAY_GETNUMERICITEM(a, lng, index)); break;
+ case TYPE_flt: obj = SCALAR_TO_PYSCALAR(flt,
LAZYARRAY_GETNUMERICITEM(a, flt, index)); break;
+ case TYPE_dbl: obj = SCALAR_TO_PYSCALAR(dbl,
LAZYARRAY_GETNUMERICITEM(a, dbl, index)); break;
+ case TYPE_hge: obj = PyLong_FromHge(PyLazyArray_GetHuge(a,
index)); break;
+ }
+ a->array[index] = obj;
+ Py_INCREF(obj);
+ return obj;
+ }
+}
+
+
+static void
+lazyarray_dealloc(PyLazyArrayObject *op)
+{
+ Py_ssize_t i;
+ if (op->array != NULL)
+ {
+ i = Py_SIZE(op);
+ while (--i >= 0)
+ {
+ Py_XDECREF(op->array[i]);
+ }
+ GDKfree(op->array);
+ }
+ Py_XDECREF(op->numpy_array);
+ Py_TYPE(op)->tp_free((PyObject *)op);
+}
+
+
+static int
+lazyarray_print(PyLazyArrayObject *op, FILE *fp, int flags)
+{
+ (void) op; (void) flags;
+ fprintf(fp, "[");
+ //todo
+ fprintf(fp, "]");
+ return 0;
+}
+
+
+static PyObject *
+lazyarray_repr(PyLazyArrayObject *v)
+{
+ PyObject *s, *temp;
+ (void) v;
+ s = PyString_FromString("[");
+ //todo
+ temp = PyString_FromString("]");
+ PyString_ConcatAndDel(&s, temp);
+ return s;
+}
+
+static Py_ssize_t
+lazyarray_length(PyLazyArrayObject *a)
+{
+ return Py_SIZE(a);
+}
+
+static PyObject *
+lazyarray_slice(PyLazyArrayObject *a, Py_ssize_t ilow, Py_ssize_t ihigh)
+{
+ PyListObject *np;
+ PyObject **dest;
+ Py_ssize_t i, len;
+ if (ilow < 0)
+ ilow = 0;
+ else if (ilow > Py_SIZE(a))
+ ilow = Py_SIZE(a);
+ if (ihigh < ilow)
+ ihigh = ilow;
+ else if (ihigh > Py_SIZE(a))
+ ihigh = Py_SIZE(a);
+ len = ihigh - ilow;
+ np = (PyListObject *) PyList_New(len);
+ if (np == NULL)
+ return NULL;
+
+ dest = np->ob_item;
+ for (i = 0; i < len; i++) {
+ PyObject *v = PyLazyArray_GetItem(a, i);
+ dest[i] = v;
+ }
+ return (PyObject *)np;
+}
+
+static int
+lazyarray_contains(PyLazyArrayObject *a, PyObject *sub_obj)
+{
+ (void) sub_obj;
+ //todo
+ if (PyLazyArray_GET_TYPE(a) == TYPE_str) {
+ return 0;
+ }
+ if (PyLazyArray_GET_TYPE(a) == TYPE_hge) {
+ return 0;
+ }
+ return -1;
+}
+
+
+#define COPY_BINARY_NUMPY_FUNCTION(name) static PyObject
*lazyarray_##name(PyLazyArrayObject *m1, PyObject *m2) { return
PyArray_Type.tp_as_number->nb_##name(PyLazyArray_AsNumpyArray(m1, 0,
Py_SIZE(m1)), m2); }
+
+COPY_BINARY_NUMPY_FUNCTION(add);
+COPY_BINARY_NUMPY_FUNCTION(subtract);
+COPY_BINARY_NUMPY_FUNCTION(multiply);
+COPY_BINARY_NUMPY_FUNCTION(divide);
+COPY_BINARY_NUMPY_FUNCTION(remainder);
+COPY_BINARY_NUMPY_FUNCTION(divmod);
+COPY_BINARY_NUMPY_FUNCTION(lshift);
+COPY_BINARY_NUMPY_FUNCTION(rshift);
+COPY_BINARY_NUMPY_FUNCTION(and);
+COPY_BINARY_NUMPY_FUNCTION(xor);
+COPY_BINARY_NUMPY_FUNCTION(or);
+COPY_BINARY_NUMPY_FUNCTION(inplace_add);
+COPY_BINARY_NUMPY_FUNCTION(inplace_subtract);
+COPY_BINARY_NUMPY_FUNCTION(inplace_multiply);
+COPY_BINARY_NUMPY_FUNCTION(inplace_divide);
+
+
+#define COPY_UNARY_NUMPY_FUNCTION(name) static PyObject
*lazyarray_##name(PyLazyArrayObject *v) { return
PyArray_Type.tp_as_number->nb_##name(PyLazyArray_AsNumpyArray(v, 0,
Py_SIZE(v))); }
+
+COPY_UNARY_NUMPY_FUNCTION(int);
+COPY_UNARY_NUMPY_FUNCTION(long);
+COPY_UNARY_NUMPY_FUNCTION(float);
+COPY_UNARY_NUMPY_FUNCTION(oct);
+COPY_UNARY_NUMPY_FUNCTION(hex);
+COPY_UNARY_NUMPY_FUNCTION(negative);
+COPY_UNARY_NUMPY_FUNCTION(positive);
+COPY_UNARY_NUMPY_FUNCTION(absolute);
+COPY_UNARY_NUMPY_FUNCTION(invert);
+// static PyObject *
+// lazyarray_multiply(PyLazyArrayObject *m1, PyObject *m2)
+// {
+// return
PyArray_Type.tp_as_number->nb_multiply(PyLazyArray_AsNumpyArray(m1, 0,
Py_SIZE(m1)), m2);
+// }
+
+// static PyObject *
+// lazyarray_inplace_multiply(PyLazyArrayObject *m1, PyObject *m2)
+// {
+// return
PyArray_Type.tp_as_number->nb_inplace_multiply(PyLazyArray_AsNumpyArray(m1, 0,
Py_SIZE(m1)), m2);
+// }
+
+static PyNumberMethods lazyarray_as_number = {
+ (binaryfunc)lazyarray_add, /*nb_add*/
+ (binaryfunc)lazyarray_subtract,
/*nb_subtract*/
+ (binaryfunc)lazyarray_multiply, /*nb_multiply*/
+#if !defined(NPY_PY3K)
+ (binaryfunc)lazyarray_divide, /*nb_divide*/
+#endif
+ (binaryfunc)lazyarray_remainder,
/*nb_remainder*/
+ (binaryfunc)lazyarray_divmod, /*nb_divmod*/
+ 0, /*nb_power*/
+ (unaryfunc)lazyarray_negative, /*nb_neg*/
+ (unaryfunc)lazyarray_positive, /*nb_pos*/
+ (unaryfunc)lazyarray_absolute,
/*(unaryfunc)array_abs,*/
+ 0, /*nb_nonzero*/
+ (unaryfunc)lazyarray_invert, /*nb_invert*/
+ (binaryfunc)lazyarray_lshift, /*nb_lshift*/
+ (binaryfunc)lazyarray_rshift, /*nb_rshift*/
+ (binaryfunc)lazyarray_and, /*nb_and*/
+ (binaryfunc)lazyarray_xor, /*nb_xor*/
+ (binaryfunc)lazyarray_or, /*nb_or*/
+#if !defined(NPY_PY3K)
+ 0, /*nb_coerce*/
+#endif
+ (unaryfunc)lazyarray_int, /*nb_int*/
+#if defined(NPY_PY3K)
+ 0, /*nb_reserved*/
+#else
+ (unaryfunc)lazyarray_long, /*nb_long*/
+#endif
+ (unaryfunc)lazyarray_float, /*nb_float*/
+#if !defined(NPY_PY3K)
+ (unaryfunc)lazyarray_oct, /*nb_oct*/
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
