Changeset: d724159ab6c7 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d724159ab6c7
Modified Files:
monetdb5/extras/pyapi/emit.c
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/type_conversion.c
monetdb5/extras/pyapi/type_conversion.h
sql/backends/monet5/Tests/pyapi30.sql
sql/backends/monet5/Tests/pyapi30.stable.out
Branch: pythonloader
Log Message:
Add support for strings in pyloader functions.
diffs (truncated from 653 to 300 lines):
diff --git a/monetdb5/extras/pyapi/emit.c b/monetdb5/extras/pyapi/emit.c
--- a/monetdb5/extras/pyapi/emit.c
+++ b/monetdb5/extras/pyapi/emit.c
@@ -13,7 +13,7 @@
tpe val = (tpe) tpe##_nil; msg = pyobject_to_##tpe(&dictEntry, 42,
&val); \
BUNappend(self->cols[i].b, &val, 0); \
if (msg != MAL_SUCCEED) { \
- PyErr_SetString(PyExc_TypeError, "conversion failed"); /* TODO:
better error message */ \
+ PyErr_Format(PyExc_TypeError, "Conversion Failed: %s", msg); \
return NULL; \
}}
@@ -91,7 +91,18 @@ static PyObject *
break;
#endif
case TYPE_str:
- // FIXME scalar_convert(str);
+ {
+ str val = NULL;
+ msg = pyobject_to_str(&dictEntry, 42, &val);
+ BUNappend(self->cols[i].b, val, 0);
+ if (val) {
+ free(val);
+ }
+ if (msg != MAL_SUCCEED) {
+ PyErr_Format(PyExc_TypeError, "Conversion
Failed: %s", msg);
+ return NULL;
+ }
+ }
break;
default:
break;
@@ -162,7 +173,7 @@ PyTypeObject Py_EmitType = {
0,
0,
0,
- 0,
+ 0,
0,
0
#ifdef IS_PY3K
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -399,7 +399,7 @@ Array of type %s no copying will be need
ptr[group][i][temp_indices[group]++] = batcontent[element_it];
\
}
\
GDKfree(temp_indices);
\
-}
+}
str
PyAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit
grouped, bit mapped);
@@ -627,7 +627,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
//create initial shared memory
MT_lock_set(&pyapiLock);
- mmap_id = get_unique_id(mmap_count);
+ mmap_id = get_unique_id(mmap_count);
MT_lock_unset(&pyapiLock);
mmap_ptrs = GDKzalloc(mmap_count * sizeof(void*));
@@ -712,11 +712,11 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
//wait for the child to finish
//note that we use a timeout here in case the child crashes
for some reason
//in this case the semaphore value is never increased, so we
would be stuck otherwise
- msg = change_semaphore_value_timeout(query_sem, 0, -1, 100,
&sem_success);
+ msg = change_semaphore_value_timeout(query_sem, 0, -1, 100,
&sem_success);
if (msg != MAL_SUCCEED){
goto wrapup;
}
- if (sem_success)
+ if (sem_success)
{
if (query_ptr->pending_query) {
// we have to handle a query for the forked process
@@ -737,7 +737,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
query_ptr->nr_cols = 0;
query_ptr->mmapid = -1;
- if (output != NULL && output->nr_cols > 0)
+ if (output != NULL && output->nr_cols > 0)
{
// copy the return values into shared memory if
there are any
size_t size = 0;
@@ -776,7 +776,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
size += sizeof(COLrec);
//[COLrec]
size += sizeof(BATrec);
//[BATrec]
size += batsize;
//[DATA]
-
+
if (b->T->vheap != NULL) {
size += sizeof(Heap);
//[VHEAP]
size += b->T->vheap->size;
//[VHEAPDATA]
@@ -789,7 +789,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
// create the actual shared memory region
MT_lock_set(&pyapiLock);
- query_ptr->mmapid = get_unique_id(1);
+ query_ptr->mmapid = get_unique_id(1);
MT_lock_unset(&pyapiLock);
msg = init_mmap_memory(query_ptr->mmapid, 0, size,
NULL, NULL, &result_ptr);
@@ -808,16 +808,16 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
size_t batsize = b->T->width * BATcount(b);
//[COLNAME]
- memcpy(result_ptr + position, colname,
strlen(colname) + 1);
+ memcpy(result_ptr + position, colname,
strlen(colname) + 1);
position += strlen(colname) + 1;
//[BAT]
- memcpy(result_ptr + position, b, sizeof(BAT));
+ memcpy(result_ptr + position, b, sizeof(BAT));
position += sizeof(BAT);
//[COLREC]
- memcpy(result_ptr + position, b->T,
sizeof(COLrec));
+ memcpy(result_ptr + position, b->T,
sizeof(COLrec));
position += sizeof(COLrec);
//[BATREC]
- memcpy(result_ptr + position, b->S,
sizeof(BATrec));
+ memcpy(result_ptr + position, b->S,
sizeof(BATrec));
position += sizeof(BATrec);
//[DATA]
memcpy(result_ptr + position, Tloc(b,
BUNfirst(b)), batsize);
@@ -935,7 +935,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
#endif
//After this point we will execute Python Code, so we need to acquire the
GIL
- if (!mapped) {
+ if (!mapped) {
gstate = Python_ObtainGIL();
}
@@ -949,14 +949,14 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
char address[1000];
struct stat buffer;
size_t length;
- if (exprStr[0] == '/') {
+ if (exprStr[0] == '/') {
// absolute path
snprintf(address, 1000, "%s", exprStr);
} else {
// relative path
snprintf(address, 1000, "%s/%s", FunctionBasePath(), exprStr);
}
- if (stat(address, &buffer) < 0) {
+ if (stat(address, &buffer) < 0) {
msg = createException(MAL, "pyapi.eval", "Could not find
Python source file \"%s\".", address);
goto wrapup;
}
@@ -1111,7 +1111,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
}
//now perform the actual splitting of the data, first construct
room for splits for every group
- // elements are structured as follows:
+ // elements are structured as follows:
// split_bats [groupnr] [columnnr] [elementnr]
split_bats = GDKzalloc(group_count * sizeof(void*));
for(group_it = 0; group_it < group_count; group_it++) {
@@ -1168,7 +1168,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
temp_indices = GDKzalloc(sizeof(PyObject*) *
group_count);
for(element_it = 0; element_it < elements;
element_it++) {
//group of current element
- oid group = aggr_group_arr[element_it];
+ oid group = aggr_group_arr[element_it];
//append current element to proper group
ptr[group][i][temp_indices[group]++] =
batcontent[element_it];
}
@@ -1194,7 +1194,7 @@ str PyAPIeval(Client cntxt, MalBlkPtr mb
double increment;
// if there are less groups than threads, limit threads to
amount of groups
- threads = group_count < threads ? group_count : threads;
+ threads = group_count < threads ? group_count : threads;
increment = (double) group_count / (double) threads;
// start running the threads
@@ -1698,7 +1698,7 @@ bool PyType_IsPyScalar(PyObject *object)
{
if (object == NULL) return false;
return (PyArray_CheckScalar(object) || PyInt_Check(object) ||
PyFloat_Check(object) || PyLong_Check(object) || PyString_Check(object) ||
PyBool_Check(object) || PyUnicode_Check(object) || PyByteArray_Check(object)
-#ifdef IS_PY3K
+#ifdef IS_PY3K
|| PyBytes_Check(object)
#endif
);
@@ -1961,7 +1961,7 @@ PyObject *PyArrayObject_FromBAT(PyInput
PyObject **data =
((PyObject**)PyArray_DATA((PyArrayObject*)vararray));
PyObject *obj;
j = 0;
- if (unicode) {
+ if (unicode) {
if (GDK_ELIMDOUBLES(b->T->vheap)) {
PyObject** pyptrs = GDKzalloc(b->T->vheap->free *
sizeof(PyObject*));
if (!pyptrs) {
@@ -2408,7 +2408,7 @@ BAT *PyObject_ConvertToBAT(PyReturn *ret
case EC_DEC:
bat_type = TYPE_dbl;
break;
- default:
+ default:
break;
}
@@ -2522,11 +2522,7 @@ BAT *PyObject_ConvertToBAT(PyReturn *ret
PyObject *obj;
if (mask != NULL && (mask[index_offset * ret->count +
iu]) == TRUE) continue;
obj = *((PyObject**) &data[(index_offset * ret->count
+ iu) * ret->memory_size]);
- if (PyString_CheckExact(obj) ||
PyByteArray_CheckExact(obj)) {
- size = Py_SIZE(obj); //Normal strings are 1
string per character
- } else if (PyUnicode_CheckExact(obj)) {
- size = Py_SIZE(obj) * 4; //UTF32 is 4 bytes per
character
- }
+ size = pyobject_get_size(obj);
if (size > utf8_size) utf8_size = size;
}
utf8_string = GDKzalloc(utf8_size);
@@ -2536,51 +2532,7 @@ BAT *PyObject_ConvertToBAT(PyReturn *ret
BUNappend(b, str_nil, FALSE);
} else {
//we try to handle as many types as possible
- PyObject *obj = *((PyObject**) &data[(index_offset
* ret->count + iu) * ret->memory_size]);
-#ifndef IS_PY3K
- if (PyString_CheckExact(obj)) {
- char *str = ((PyStringObject*)obj)->ob_sval;
- if (!string_copy(str, utf8_string, strlen(str)
+ 1, false)) {
- msg = createException(MAL, "pyapi.eval",
"Invalid string encoding used. Please return a regular ASCII string, or a
Numpy_Unicode object.\n");
- goto wrapup;
- }
- } else
-#endif
- if (PyByteArray_CheckExact(obj)) {
- char *str =
((PyByteArrayObject*)obj)->ob_bytes;
- if (!string_copy(str, utf8_string, strlen(str)
+ 1, false)) {
- msg = createException(MAL, "pyapi.eval",
"Invalid string encoding used. Please return a regular ASCII string, or a
Numpy_Unicode object.\n");
- goto wrapup;
- }
- } else if (PyUnicode_CheckExact(obj)) {
-#ifndef IS_PY3K
- Py_UNICODE *str =
(Py_UNICODE*)((PyUnicodeObject*)obj)->str;
-#if Py_UNICODE_SIZE >= 4
- utf32_to_utf8(0,
((PyUnicodeObject*)obj)->length, utf8_string, str);
-#else
- ucs2_to_utf8(0,
((PyUnicodeObject*)obj)->length, utf8_string, str);
-#endif
-#else
- char *str = PyUnicode_AsUTF8(obj);
- if (!string_copy(str, utf8_string, strlen(str)
+ 1, true)) {
- msg = createException(MAL, "pyapi.eval",
"Invalid string encoding used. Please return a regular ASCII string, or a
Numpy_Unicode object.\n");
- goto wrapup;
- }
-#endif
- } else if (PyBool_Check(obj) || PyLong_Check(obj)
|| PyInt_Check(obj) || PyFloat_Check(obj)) {
-#ifdef HAVE_HGE
- hge h;
- pyobject_to_hge(&obj, 0, &h);
- hge_to_string(utf8_string, h);
-#else
- lng h;
- pyobject_to_lng(&obj, 0, &h);
- snprintf(utf8_string, utf8string_minlength,
LLFMT, h);
-#endif
- } else {
- msg = createException(MAL, "pyapi.eval",
"Unrecognized Python object. Could not convert to NPY_UNICODE.\n");
- goto wrapup;
- }
+ pyobject_to_str(((PyObject**) &data[(index_offset
* ret->count + iu) * ret->memory_size]), utf8_size, &utf8_string);
BUNappend(b, utf8_string, FALSE);
}
}
@@ -2605,7 +2557,7 @@ BAT *PyObject_ConvertToBAT(PyReturn *ret
if (ConvertableSQLType(type)) {
BAT *result;
msg = ConvertToSQLType(NULL, b, type, &result, &bat_type);
- if (msg != MAL_SUCCEED) {
+ if (msg != MAL_SUCCEED) {
goto wrapup;
}
b = result;
@@ -2637,7 +2589,7 @@ int GetSQLType(sql_subtype *sql_subtype)
str ConvertFromSQLType(Client cntxt, BAT *b, sql_subtype *sql_subtype, BAT
**ret_bat, int *ret_type)
{
str res = MAL_SUCCEED;
- int conv_type;
+ int conv_type;
assert(sql_subtype);
assert(sql_subtype->type);
@@ -2652,7 +2604,7 @@ str ConvertFromSQLType(Client cntxt, BAT
case EC_DEC:
conv_type = TYPE_dbl;
break;
- default:
+ default:
return createException(MAL, "pyapi.eval", "Convert From SQL Type:
Unrecognized SQL type %s (%d).", sql_subtype->type->sqlname,
sql_subtype->type->eclass);
}
@@ -2683,7 +2635,7 @@ str ConvertFromSQLType(Client cntxt, BAT
case EC_TIMESTAMP:
d1 = 7;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
