Changeset: 2d8848e66425 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2d8848e66425
Modified Files:
sql/backends/monet5/UDF/pyapi/conversion.c
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/UDF/pyapi/pyloader.c
sql/backends/monet5/UDF/pyapi/pytypes.c
Branch: Nov2019
Log Message:
Fix for bug 6726 failing test. Look for TYPE_void bats in pyapi and generate
dense sequences from them. (I have to check for 32-bit architectures yet)
diffs (216 lines):
diff --git a/sql/backends/monet5/UDF/pyapi/conversion.c
b/sql/backends/monet5/UDF/pyapi/conversion.c
--- a/sql/backends/monet5/UDF/pyapi/conversion.c
+++ b/sql/backends/monet5/UDF/pyapi/conversion.c
@@ -37,6 +37,11 @@ static bool IsBlobType(int type)
return type == TYPE_blob;
}
+static bool IsVoidType(int type)
+{
+ return type == TYPE_void;
+}
+
PyObject *PyArrayObject_FromScalar(PyInput *inp, char **return_message)
{
PyObject *vararray = NULL;
@@ -44,6 +49,12 @@ PyObject *PyArrayObject_FromScalar(PyInp
assert(inp->scalar); // input has to be a scalar
switch (inp->bat_type) {
+ case TYPE_void:
+ vararray = PyArray_Arange(0, 1, 1, NPY_ULONGLONG);
+ break;
+ case TYPE_oid:
+ vararray = PyInt_FromLong((long)(*(oid *)inp->dataptr));
+ break;
case TYPE_bit:
vararray = PyInt_FromLong((long)(*(bit *)inp->dataptr));
break;
@@ -165,7 +176,7 @@ PyObject *PyArrayObject_FromBAT(PyInput
goto wrapup;
}
- if (!IsBlobType(inp->bat_type) &&
+ if (!IsVoidType(inp->bat_type) && !IsBlobType(inp->bat_type) &&
(!IsStandardBATType(inp->bat_type) ||
ConvertableSQLType(inp->sql_subtype))) { // if the sql type is
set, we
// have to do some conversion
@@ -205,6 +216,12 @@ PyObject *PyArrayObject_FromBAT(PyInput
}
} else {
switch (inp->bat_type) {
+ case TYPE_void:
+ BAT_TO_NP_CREATE_ALWAYS(b, NPY_ULONGLONG);
+ break;
+ case TYPE_oid:
+ BAT_TO_NP(b, oid, NPY_ULONG);
+ break;
case TYPE_bit:
BAT_TO_NP(b, bit, NPY_INT8);
break;
@@ -927,6 +944,9 @@ BAT *PyObject_ConvertToBAT(PyReturn *ret
BATsettrivprop(b);
} else {
switch (bat_type) {
+ case TYPE_void:
+ NP_CREATE_EMPTY_BAT(b, oid);
+ break;
case TYPE_bit:
NP_CREATE_BAT(b, bit);
break;
diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -12,6 +12,9 @@
* these are in a separate header because they are used in multiple places
*/
+#define BAT_TO_NP_CREATE_ALWAYS(bat, nptpe)
\
+ vararray = PyArray_Arange(0, (double)bat->batCount, 1, nptpe);
+
#define BAT_TO_NP(bat, mtpe, nptpe)
\
if (copy) {
\
vararray = PyArray_EMPTY(1, elements, nptpe, 0);
\
@@ -561,6 +564,22 @@ convert_and_append(BAT* b, const char* t
#define NOT_HGE(mtpe) true
#endif
+#define NP_CREATE_EMPTY_BAT(bat, mtpe) \
+ { \
+ bat = COLnew(seqbase, TYPE_##mtpe, (BUN)ret->count, TRANSIENT);
\
+ if (bat == NULL) {
\
+ msg = createException(MAL, "pyapi.eval",
SQLSTATE(PY000) "Cannot create column"); \
+ goto wrapup;
\
+ }
\
+ bat->tkey = false;
\
+ bat->tsorted = false;
\
+ bat->trevsorted = false;
\
+ bat->tnil = false;
\
+ bat->tnonil = true;
\
+ BATsetcount(bat, (BUN)ret->count);
\
+ BATsettrivprop(bat);
\
+ }
+
// This very big #define combines all the previous #defines for one big #define
// that is responsible for converting a Numpy array (described in the PyReturn
// object 'ret')
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -281,7 +281,7 @@ static str PyAPIeval(Client cntxt, MalBl
}
seqbase = b->hseqbase;
inp->count = BATcount(b);
- inp->bat_type = ATOMstorage(getBatType(getArgType(mb,
pci, i)));
+ inp->bat_type = b->ttype;
inp->bat = b;
if (inp->count == 0) {
// one of the input BATs is empty, don't
execute the function at
@@ -635,9 +635,10 @@ static str PyAPIeval(Client cntxt, MalBl
result_array = PyArrayObject_FromScalar(
&pyinput_values[i - (pci->retc + 2)], &msg);
} else {
+ int type = pyinput_values[i - (pci->retc + 2)].bat_type;
result_array = PyMaskedArray_FromBAT(
&pyinput_values[i - (pci->retc + 2)], t_start,
t_end, &msg,
- !enable_zerocopy_input);
+ !enable_zerocopy_input && type != TYPE_void);
}
if (result_array == NULL) {
if (msg == MAL_SUCCEED) {
@@ -756,6 +757,9 @@ static str PyAPIeval(Client cntxt, MalBl
if (!input.scalar) {
switch (input.bat_type) {
+ case TYPE_void:
+ NP_SPLIT_BAT(oid);
+ break;
case TYPE_bit:
NP_SPLIT_BAT(bit);
break;
@@ -1072,7 +1076,8 @@ static str PyAPIeval(Client cntxt, MalBl
// if someone has some problem with memory
size exploding when
// using PYTHON_MAP but it being fine in
regular PYTHON this
// is probably the issue
- int bat_type = getBatType(getArgType(mb, pci,
i));
+ PyInput *inp = &pyinput_values[i - (pci->retc +
2)];
+ int bat_type = inp->bat_type;
PyObject *new_array = PyArray_FromAny(
ret->numpy_array,
PyArray_DescrFromType(BatType_ToPyType(bat_type)), 1, 1,
@@ -1502,6 +1507,19 @@ static void ComputeParallelAggregation(A
} else {
npy_intp elements[1] = {group_elements};
switch (input.bat_type) {
+ case TYPE_void:
+ case TYPE_oid:
+ vararray = PyArray_New(
+ &PyArray_Type, 1,
elements, NPY_INT64, NULL,
+ ((oid
***)(*p->split_bats))[group_it][i], 0,
+ NPY_ARRAY_CARRAY ||
!NPY_ARRAY_WRITEABLE, NULL);
+ break;
+ case TYPE_bit:
+ vararray = PyArray_New(
+ &PyArray_Type, 1,
elements, NPY_BOOL, NULL,
+ ((bit
***)(*p->split_bats))[group_it][i], 0,
+ NPY_ARRAY_CARRAY ||
!NPY_ARRAY_WRITEABLE, NULL);
+ break;
case TYPE_bte:
vararray = PyArray_New(
&PyArray_Type, 1,
elements, NPY_INT8, NULL,
diff --git a/sql/backends/monet5/UDF/pyapi/pyloader.c
b/sql/backends/monet5/UDF/pyapi/pyloader.c
--- a/sql/backends/monet5/UDF/pyapi/pyloader.c
+++ b/sql/backends/monet5/UDF/pyapi/pyloader.c
@@ -138,7 +138,7 @@ PYFUNCNAME(PyAPIevalLoader)(Client cntxt
}
inp.scalar = false;
inp.count = BATcount(b);
- inp.bat_type = ATOMstorage(getBatType(getArgType(mb,
pci, i)));
+ inp.bat_type = getBatType(getArgType(mb, pci, i));
inp.bat = b;
val = PyMaskedArray_FromBAT(
diff --git a/sql/backends/monet5/UDF/pyapi/pytypes.c
b/sql/backends/monet5/UDF/pyapi/pytypes.c
--- a/sql/backends/monet5/UDF/pyapi/pytypes.c
+++ b/sql/backends/monet5/UDF/pyapi/pytypes.c
@@ -113,6 +113,8 @@ char *BatType_Format(int type)
return "BLOB";
}
switch (type) {
+ case TYPE_void:
+ return "VOID";
case TYPE_bit:
return "BOOL";
case TYPE_bte:
@@ -157,12 +159,8 @@ int PyType_ToBat(int type)
#endif
case NPY_LONGLONG:
return TYPE_lng;
- case NPY_UBYTE:
- case NPY_USHORT:
- case NPY_UINT:
case NPY_ULONG:
- case NPY_ULONGLONG:
- return TYPE_void;
+ return TYPE_oid;
case NPY_FLOAT16:
case NPY_FLOAT:
return TYPE_flt;
@@ -184,6 +182,8 @@ int BatType_ToPyType(int type)
return NPY_OBJECT;
}
switch (type) {
+ case TYPE_void:
+ return NPY_ULONGLONG;
case TYPE_bit:
return NPY_BOOL;
case TYPE_bte:
@@ -201,7 +201,7 @@ int BatType_ToPyType(int type)
case TYPE_str:
return NPY_UNICODE;
case TYPE_oid:
- return NPY_INT32;
+ return NPY_ULONG;
#ifdef HAVE_HGE
case TYPE_hge:
return NPY_FLOAT64;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
