Changeset: 6a9a1c00ce30 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6a9a1c00ce30
Modified Files:
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pytypes.h
tools/embeddedpy/Makefile.ag
tools/embeddedpy/embeddedpy.c
Branch: pyapi
Log Message:
Removed duplicated code.
diffs (truncated from 1916 to 300 lines):
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -1221,15 +1221,16 @@ returnvalues:
str
PyAPIprelude(void *ret) {
(void) ret;
+#ifndef _EMBEDDED_MONETDB_MONETDB_LIB_
MT_lock_init(&pyapiLock, "pyapi_lock");
if (PyAPIEnabled()) {
MT_lock_set(&pyapiLock, "pyapi.evaluate");
if (!pyapiInitialized) {
char* iar = NULL;
Py_Initialize();
+ PyRun_SimpleString("import numpy");
PyByteArray_Override();
import_array1(iar);
- PyRun_SimpleString("import numpy");
initialize_shared_memory();
lazyarray_init();
pyapiInitialized++;
@@ -1259,6 +1260,23 @@ str
fprintf(stdout, "# MonetDB/Python testing enabled.\n");
#endif
}
+#else
+ if (!pyapiInitialized) {
+ char* iar = NULL;
+ import_array1(iar);
+ pyapiInitialized++;
+ }
+#ifdef _PYAPI_VERBOSE_
+ option_verbose = GDKgetenv_isyes(verbose_enableflag) ||
GDKgetenv_istrue(verbose_enableflag);
+#endif
+#ifdef _PYAPI_DEBUG_
+ option_debug = GDKgetenv_isyes(debug_enableflag) ||
GDKgetenv_istrue(debug_enableflag);
+ (void) option_debug;
+#endif
+#ifdef _PYAPI_WARNINGS_
+ option_warning = GDKgetenv_isyes(warning_enableflag) ||
GDKgetenv_istrue(warning_enableflag);
+#endif
+#endif
return MAL_SUCCEED;
}
diff --git a/monetdb5/extras/pyapi/pytypes.h b/monetdb5/extras/pyapi/pytypes.h
--- a/monetdb5/extras/pyapi/pytypes.h
+++ b/monetdb5/extras/pyapi/pytypes.h
@@ -63,43 +63,43 @@ struct _PyInput{
#define PyInput struct _PyInput
//! Returns true if a NPY_#type is an integral type, and false otherwise
-bool PyType_IsInteger(int);
+pyapi_export bool PyType_IsInteger(int);
//! Returns true if a NPY_#type is a float type, and false otherwise
-bool PyType_IsFloat(int);
+pyapi_export bool PyType_IsFloat(int);
//! Returns true if a NPY_#type is a double type, and false otherwise
-bool PyType_IsDouble(int);
+pyapi_export bool PyType_IsDouble(int);
//! Formats NPY_#type as a String (so NPY_INT => "INT"), for usage in error
reporting and warnings
-char *PyType_Format(int);
+pyapi_export char *PyType_Format(int);
//! Returns true if a PyObject is a scalar type ('scalars' in this context
means numeric or string types)
-bool PyType_IsPyScalar(PyObject *object);
+pyapi_export bool PyType_IsPyScalar(PyObject *object);
//! Returns true if the PyObject is of type numpy.ndarray, and false otherwise
-bool PyType_IsNumpyArray(PyObject *object);
+pyapi_export bool PyType_IsNumpyArray(PyObject *object);
//! Returns true if the PyObject is of type numpy.ma.core.MaskedArray, and
false otherwise
-bool PyType_IsNumpyMaskedArray(PyObject *object);
+pyapi_export bool PyType_IsNumpyMaskedArray(PyObject *object);
//! Returns true if the PyObject is of type pandas.core.frame.DataFrame, and
false otherwise
-bool PyType_IsPandasDataFrame(PyObject *object);
+pyapi_export bool PyType_IsPandasDataFrame(PyObject *object);
//! Returns true if the PyObject is of type lazyarray, and false otherwise
-bool PyType_IsLazyArray(PyObject *object);
+pyapi_export bool PyType_IsLazyArray(PyObject *object);
//! Create a Numpy Array Object from a PyInput structure containing a BAT
-PyObject *PyArrayObject_FromBAT(PyInput *input_bat, size_t start, size_t end,
char **return_message);
+pyapi_export PyObject *PyArrayObject_FromBAT(PyInput *input_bat, size_t start,
size_t end, char **return_message);
//! Creates a Null Mask from a BAT (a Numpy Boolean Array of equal length to
the BAT, where NULLMASK[i] = True if BAT[i] is NULL, and False otherwise)
-PyObject *PyNullMask_FromBAT(BAT *b, size_t start, size_t end);
+pyapi_export PyObject *PyNullMask_FromBAT(BAT *b, size_t start, size_t end);
//! Creates a Numpy Array object from an PyInput structure containing a scalar
-PyObject *PyArrayObject_FromScalar(PyInput* input_scalar, char
**return_message);
+pyapi_export PyObject *PyArrayObject_FromScalar(PyInput* input_scalar, char
**return_message);
//! Creates a Numpy Masked Array from an PyInput structure containing a BAT
(essentially just combines PyArrayObject_FromBAT and PyNullMask_FromBAT)
-PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
char **return_message);
+pyapi_export PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start,
size_t t_end, char **return_message);
//! Test if a specific PyObject can be converted to a set of
<expected_columns> BATs (or just check if they can be converted to any number
of BATs if expected_columns is smaller than 0)
-PyObject *PyObject_CheckForConversion(PyObject *pResult, int expected_columns,
int *actual_columns, char **return_message);
+pyapi_export PyObject *PyObject_CheckForConversion(PyObject *pResult, int
expected_columns, int *actual_columns, char **return_message);
//! Preprocess a PyObject (that is the result of PyObject_CheckForConversion),
pyreturn_values must be an array of PyReturn structs of size column_count
-bool PyObject_PreprocessObject(PyObject *pResult, PyReturn *pyreturn_values,
int column_count, char **return_message);
+pyapi_export bool PyObject_PreprocessObject(PyObject *pResult, PyReturn
*pyreturn_values, int column_count, char **return_message);
//! Create a BAT from the i'th PyReturn struct (filled by
PyObject_PreprocessObject), with bat_type set to the expected BAT Type (set
this to PyType_ToBat(ret->result_type) if there is no expected type), seqbase
should be set to 0 unless you know what you're doing
-BAT *PyObject_ConvertToBAT(PyReturn *ret, int bat_type, int index, int
seqbase, char **return_message);
+pyapi_export BAT *PyObject_ConvertToBAT(PyReturn *ret, int bat_type, int
index, int seqbase, char **return_message);
-char *BatType_Format(int);
+pyapi_export char *BatType_Format(int);
-int PyType_ToBat(int);
-int BatType_ToPyType(int);
+pyapi_export int PyType_ToBat(int);
+pyapi_export int BatType_ToPyType(int);
#define bte_TO_PYSCALAR(value) PyInt_FromLong(value)
#define bit_TO_PYSCALAR(value) PyInt_FromLong(value)
diff --git a/tools/embeddedpy/Makefile.ag b/tools/embeddedpy/Makefile.ag
--- a/tools/embeddedpy/Makefile.ag
+++ b/tools/embeddedpy/Makefile.ag
@@ -17,7 +17,8 @@
MTSAFE
-INCLUDES = ../../monetdb5/mal \
+INCLUDES = ../../monetdb5/extras/pyapi \
+ ../../monetdb5/mal \
../../monetdb5/modules/atoms \
../../monetdb5/modules/mal \
../../gdk \
@@ -39,6 +40,7 @@ lib_pyembedded = {
LIBS = ../../monetdb5/tools/libmonetdb5 \
../../gdk/libbat \
../../sql/backends/monet5/lib_sql \
+ ../../monetdb5/extras/pyapi/lib_pyapi \
$(pcre_LIBS) $(PSAPILIB) \
$(SOCKET_LIBS) $(MALLOC_LIBS) $(DL_LIBS) $(PTHREAD_LIBS)
}
diff --git a/tools/embeddedpy/embeddedpy.c b/tools/embeddedpy/embeddedpy.c
--- a/tools/embeddedpy/embeddedpy.c
+++ b/tools/embeddedpy/embeddedpy.c
@@ -28,75 +28,8 @@
#include "msabaoth.h"
#include "sql_scenario.h"
-/////////////////////////////////////////////////////////////////////////////////////////////////
-// copy paste from pyapi.c
-#include <longintrepr.h>
-
-struct _PyInput{
- void *dataptr; //pointer to input data
- BAT *bat; //pointer to input BAT
- int bat_type; //BAT type as TYPE_<type>
- size_t count; //amount of elements in BAT
- bool scalar; //True if the input is a scalar (in
this case, BAT* is NULL)
-};
-#define PyInput struct _PyInput
-
-struct _PyReturn{
- PyObject *numpy_array; //PyArrayObject* with data (can be
NULL, as long as array_data is set)
- PyObject *numpy_mask; //PyArrayObject* with mask (NULL if
there is no mask)
- void *array_data; //void* pointer to data
- bool *mask_data; //bool* pointer to mask data
- size_t count; //amount of return elements
- size_t memory_size; //memory size of each element
- int result_type; //result type as NPY_<TYPE>
- bool multidimensional; //whether or not the result is
multidimensional
-};
-#define PyReturn struct _PyReturn
-
-PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
char **return_message);
-PyObject *PyArrayObject_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
char **return_message);
-PyObject *PyNullMask_FromBAT(BAT *b, size_t t_start, size_t t_end);
-static char *PyError_CreateException(char *error_text, char *pycall);
-
-PyObject *PyObject_CheckForConversion(PyObject *pResult, int expected_columns,
int *actual_columns, char **return_message);
-bool PyObject_PreprocessObject(PyObject *pResult, PyReturn *pyreturn_values,
int column_count, char **return_message);
-BAT *PyObject_ConvertToBAT(PyReturn *ret, int bat_type, int index, int
seqbase, char **return_message);
-int PyType_ToBat(int);
-//! Returns true if a PyObject is a scalar type ('scalars' in this context
means numeric or string types)
-bool PyType_IsPyScalar(PyObject *object);
-//! Returns true if the PyObject is of type numpy.ndarray, and false otherwise
-bool PyType_IsNumpyArray(PyObject *object);
-//! Returns true if the PyObject is of type numpy.ma.core.MaskedArray, and
false otherwise
-bool PyType_IsNumpyMaskedArray(PyObject *object);
-//! Returns true if the PyObject is of type pandas.core.frame.DataFrame, and
false otherwise
-bool PyType_IsPandasDataFrame(PyObject *object);
-//! Returns true if the PyObject is of type lazyarray, and false otherwise
-bool PyType_IsLazyArray(PyObject *object);
-char *BatType_Format(int);
-//! Formats NPY_#type as a String (so NPY_INT => "INT"), for usage in error
reporting and warnings
-char *PyType_Format(int);
-//! Converts a base-10 utf32-encoded string to a lng value
-bool utf32_to_lng(Py_UNICODE *utf32, size_t maxsize, lng *value);
-//! Converts a base-10 utf32-encoded string to a dbl value
-bool utf32_to_dbl(Py_UNICODE *utf32, size_t maxsize, dbl *value);
-//! Converts a base-10 utf32-encoded string to a hge value
-bool utf32_to_hge(Py_UNICODE *utf32, size_t maxsize, hge *value);
-//! Converts a PyObject to a dbl value
-bool py_to_dbl(PyObject *ptr, dbl *value);
-//! Converts a PyObject to a lng value
-bool py_to_lng(PyObject *ptr, lng *value);
-//! Converts a PyObject to a hge value
-bool py_to_hge(PyObject *ptr, hge *value);
-//! Converts a double to a string and writes it into the string "str"
-void dbl_to_string(char* str, dbl value);
-//! Copies the string of size up to max_size from the source to the
destination, returns FALSE if "source" is not a legal ASCII string (i.e. a
character is >= 128)
-bool string_copy(char * source, char* dest, size_t max_size);
-//! Converts a hge to a string and writes it into the string "str"
-int hge_to_string(char *str, hge );
-int utf32_strlen(const Py_UNICODE *utf32_str);
-bool utf32_to_utf8(size_t offset, size_t size, char *utf8_storage, const
Py_UNICODE *utf32);
-int utf32_char_to_utf8_char(size_t position, char *utf8_storage, Py_UNICODE
utf32_char);
-/////////////////////////////////////////////////////////////////////////////////////////////////
+#include "pyapi.h"
+#include "pytypes.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
// copy paste from embedded.c
@@ -129,6 +62,7 @@ PyObject *monetdb_init(PyObject *self, P
PyErr_SetString(PyExc_Exception, "Failed to initialize
MonetDB with the specified directory.");
return NULL;
}
+ PyAPIprelude(NULL);
}
Py_RETURN_NONE;
}
@@ -545,1694 +479,3 @@ str monetdb_get_columns(char *table_name
return msg;
}
-
-/////////////////////////////////////////////////////////
-// This code is copy pasted from pyapi.c, should be changed to a shared
library but PyAPI is not in this branch
-/////////////////////////////////////////////////////////
-
-
-int utf8_length(unsigned char utf8_char);
-int utf8_strlen(const char *utf8_str, bool *ascii);
-PyObject *PyLong_FromHge(hge h);
-void lng_to_string(char* str, lng value);
-#ifdef HAVE_HGE
-bool s_to_hge(char *ptr, size_t size, hge *value);
-#endif
-bool s_to_dbl(char *ptr, size_t size, dbl *value);
-bool s_to_lng(char *ptr, size_t size, lng *value);
-//using macros, create a number of str_to_<type>, unicode_to_<type> and
pyobject_to_<type> functions (we are Java now)
-#define CONVERSION_FUNCTION_HEADER_FACTORY(tpe) \
- bool str_to_##tpe(void *ptr, size_t size, tpe *value); \
- bool unicode_to_##tpe(void *ptr, size_t size, tpe *value);
\
- bool pyobject_to_##tpe(void *ptr, size_t size, tpe *value);
\
-
-CONVERSION_FUNCTION_HEADER_FACTORY(bit)
-CONVERSION_FUNCTION_HEADER_FACTORY(sht)
-CONVERSION_FUNCTION_HEADER_FACTORY(int)
-CONVERSION_FUNCTION_HEADER_FACTORY(lng)
-CONVERSION_FUNCTION_HEADER_FACTORY(flt)
-CONVERSION_FUNCTION_HEADER_FACTORY(dbl)
-#ifdef HAVE_HGE
-CONVERSION_FUNCTION_HEADER_FACTORY(hge)
-#endif
-
-
-
-#define BAT_TO_NP(bat, mtpe, nptpe)
\
- vararray = PyArray_New(&PyArray_Type, 1, (npy_intp[1])
{(t_end-t_start)}, \
- nptpe, NULL, &((mtpe*) Tloc(bat, BUNfirst(bat)))[t_start], 0,
\
- NPY_ARRAY_CARRAY || !NPY_ARRAY_WRITEABLE, NULL);
-
-
-// This #define creates a new BAT with the internal data and mask from a Numpy
array, without copying the data
-// 'bat' is a BAT* pointer, which will contain the new BAT. TYPE_'mtpe' is the
BAT type, and 'batstore' is the heap storage type of the BAT (this should be
STORE_CMEM or STORE_SHARED)
-#define CREATE_BAT_ZEROCOPY(bat, mtpe, batstore) {
\
- bat = BATnew(TYPE_void, TYPE_##mtpe, 0, TRANSIENT);
\
- BATseqbase(bat, seqbase); bat->T->nil = 0; bat->T->nonil = 1;
\
- bat->tkey = 0; bat->tsorted = 0; bat->trevsorted = 0;
\
- /*Change nil values to the proper values, if they exist*/
\
- if (mask != NULL)
\
- {
\
- for (iu = 0; iu < ret->count; iu++)
\
- {
\
- if (mask[index_offset * ret->count + iu] == TRUE)
\
- {
\
- (*(mtpe*)(&data[(index_offset * ret->count + iu) *
ret->memory_size])) = mtpe##_nil; \
- bat->T->nil = 1;
\
- }
\
- }
\
- }
\
- bat->T->nonil = 1 - bat->T->nil;
\
- /*When we create a BAT a small part of memory is allocated, free it*/
\
- GDKfree(bat->T->heap.base);
\
-
\
- bat->T->heap.base = &data[(index_offset * ret->count) *
ret->memory_size]; \
- bat->T->heap.size = ret->count * ret->memory_size;
\
- bat->T->heap.free = bat->T->heap.size; /*There are no free places in
the array*/ \
- /*If index_offset > 0, we are mapping part of a multidimensional
array.*/ \
- /*The entire array will be cleared when the part with index_offset=0
is freed*/ \
- /*So we set this part of the mapping to 'NOWN'*/
\
- if (index_offset > 0) bat->T->heap.storage = STORE_NOWN;
\
- else bat->T->heap.storage = batstore;
\
- bat->T->heap.newstorage = STORE_MEM;
\
- bat->S->count = ret->count;
\
- bat->S->capacity = ret->count;
\
- bat->S->copiedtodisk = false;
\
-
\
- /*Take over the data from the numpy array*/
\
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
