Changeset: ddbd934ce8cf for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ddbd934ce8cf
Modified Files:
gdk/shared_memory.c
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pytypes.c
monetdb5/extras/pyapi/pytypes.h
monetdb5/extras/pyapi/type_conversion.c
Branch: pyapi
Log Message:
Removed excess whitespace.
diffs (truncated from 1249 to 300 lines):
diff --git a/gdk/shared_memory.c b/gdk/shared_memory.c
--- a/gdk/shared_memory.c
+++ b/gdk/shared_memory.c
@@ -10,13 +10,13 @@
#include <string.h>
#include <sys/types.h>
-#include <sys/ipc.h>
-#include <sys/shm.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
#include <sys/wait.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
-#include <fcntl.h>
+#include <fcntl.h>
#include <sched.h>
#include <errno.h>
#include <sys/sem.h>
@@ -42,7 +42,7 @@ str initialize_shared_memory(void)
{
if (shm_is_initialized) //maybe this should just return MAL_SUCCEED as
well
return createException(MAL, "shared_memory.init", "Attempting to
initialize shared memory when it was already initialized.");
-
+
//initialize the pointer to memory ID structure
shm_ptrs = malloc(shm_max_id * sizeof(void*));
shm_memory_ids = malloc(shm_max_id * sizeof(int));
@@ -310,7 +310,7 @@ int get_unique_shared_memory_id(int offs
(void) offset;
NOTIMPLEMENTED();
return -1;
-}
+}
str get_shared_memory(int id, size_t size, void **return_ptr)
{
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -93,7 +93,7 @@ static int pyapiInitialized = FALSE;
#define BAT_TO_NP(bat, mtpe, nptpe)
\
if (!option_zerocopyinput) {
\
mtpe *array;
\
- vararray = PyArray_Zeros(1, (npy_intp[1]) {(t_end - t_start)},
PyArray_DescrFromType(nptpe), 0); \
+ vararray = PyArray_Zeros(1, (npy_intp[1]) {(t_end - t_start)},
PyArray_DescrFromType(nptpe), 0); \
array = PyArray_DATA((PyArrayObject*)vararray);
\
for(j = t_start; j < t_end; j++) {
\
array[j - t_start] = ((mtpe*) Tloc(bat, BUNfirst(bat)))[j];
\
@@ -107,7 +107,7 @@ static int pyapiInitialized = FALSE;
#define BAT_TO_NP(bat, mtpe, nptpe)
\
vararray = PyArray_New(&PyArray_Type, 1, (npy_intp[1])
{(t_end-t_start)}, \
nptpe, NULL, &((mtpe*) Tloc(bat, BUNfirst(bat)))[t_start], 0,
\
- NPY_ARRAY_CARRAY || !NPY_ARRAY_WRITEABLE, NULL);
+ NPY_ARRAY_CARRAY || !NPY_ARRAY_WRITEABLE, NULL);
#endif
// This #define creates a new BAT with the internal data and mask from a Numpy
array, without copying the data
@@ -177,7 +177,7 @@ static int pyapiInitialized = FALSE;
} }
// This #define converts a Numpy Array to a BAT by copying the internal data
to the BAT. It converts the data from the Numpy Array to the BAT using a
function
-// This function has to have the prototype 'bool function(void *data, size_t
memory_size, mtpe_to *resulting_value)', and either return False (if conversion
fails)
+// This function has to have the prototype 'bool function(void *data, size_t
memory_size, mtpe_to *resulting_value)', and either return False (if conversion
fails)
// or write the value into the 'resulting_value' pointer. This is used
convertring strings/unicodes/python objects to numeric values.
#define NP_COL_BAT_LOOP_FUNC(bat, mtpe_to, func) {
\
mtpe_to value;
\
@@ -213,7 +213,7 @@ static int pyapiInitialized = FALSE;
}
\
}
\
} }
-
+
// This #define is for converting a numeric numpy array into a string BAT.
'conv' is a function that turns a numeric value of type 'mtpe' to a char* array.
#define NP_COL_BAT_STR_LOOP(bat, mtpe, conv)
\
@@ -246,7 +246,7 @@ static int pyapiInitialized = FALSE;
#ifdef _PYAPI_TESTING_
#define ZEROCOPY_OUTPUT option_zerocopyoutput &&
#else
-#define ZEROCOPY_OUTPUT
+#define ZEROCOPY_OUTPUT
#endif
// This very big #define combines all the previous #defines for one big
#define that is responsible for converting a Numpy array (described in the
PyReturn object 'ret')
@@ -320,35 +320,35 @@ Array of type %s no copying will be need
}
\
}
-str
-PyAPIeval(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped, bit mapped);
+str
+PyAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit
grouped, bit mapped);
-str
-PyAPIevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+str
+PyAPIevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
(void) cntxt;
- return PyAPIeval(mb, stk, pci, 0, 0);
+ return PyAPIeval(cntxt, mb, stk, pci, 0, 0);
}
-str
-PyAPIevalStdMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+str
+PyAPIevalStdMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
(void) cntxt;
- return PyAPIeval(mb, stk, pci, 0, 1);
+ return PyAPIeval(cntxt, mb, stk, pci, 0, 1);
}
-str
-PyAPIevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+str
+PyAPIevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
(void) cntxt;
- return PyAPIeval(mb, stk, pci, 1, 0);
+ return PyAPIeval(cntxt, mb, stk, pci, 1, 0);
}
-str
-PyAPIevalAggrMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+str
+PyAPIevalAggrMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
(void) cntxt;
- return PyAPIeval(mb, stk, pci, 1, 1);
+ return PyAPIeval(cntxt, mb, stk, pci, 1, 1);
}
static char *PyError_CreateException(char *error_text, char *pycall);
@@ -361,7 +361,7 @@ static char *PyError_CreateException(cha
//! [EXECUTE_CODE] Step 3: It executes the Python code using the Numpy arrays
as arguments
//! [RETURN_VALUES] Step 4: It collects the return values and converts them
back into BATs
//! If 'mapped' is set to True, it will fork a separate process at
[FORK_PROCESS] that executes Step 1-3, the process will then write the return
values into Shared memory [SHARED_MEMORY] and exit, then Step 4 is executed by
the main process
-str PyAPIeval(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped, bit
mapped) {
+str PyAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit
grouped, bit mapped) {
sql_func * sqlfun = *(sql_func**) getArgReference(stk, pci, pci->retc);
str exprStr = *getArgReference_str(stk, pci, pci->retc + 1);
@@ -394,6 +394,8 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
int j;
size_t iu;
+ (void) cntxt;
+
if (!PyAPIEnabled()) {
throw(MAL, "pyapi.eval",
"Embedded Python has not been enabled. Start server with --set
%s=true",
@@ -467,20 +469,17 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
}
// Construct PyInput objects, we do this before any multiprocessing
because there is some locking going on in there, and locking + forking = bad
idea (a thread can fork while another process is in the lock, which means we
can get stuck permanently)
- for (i = pci->retc + 2; i < pci->argc; i++)
+ for (i = pci->retc + 2; i < pci->argc; i++)
{
PyInput *inp = &pyinput_values[i - (pci->retc + 2)];
- if (!isaBatType(getArgType(mb,pci,i)))
- {
+ if (!isaBatType(getArgType(mb,pci,i))) {
inp->scalar = true;
inp->bat_type = getArgType(mb, pci, i);
inp->count = 1;
- if (inp->bat_type == TYPE_str)
- {
+ if (inp->bat_type == TYPE_str) {
inp->dataptr = getArgReference_str(stk, pci, i);
}
- else
- {
+ else {
inp->dataptr = getArgReference(stk, pci, i);
}
}
@@ -519,7 +518,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
//create initial shared memory
MT_lock_set(&pyapiLock, "pyapi.evaluate");
- shm_id = get_unique_shared_memory_id(1 + pci->retc * 2); //we need 1 +
pci->retc * 2 shared memory spaces, the first is for the header information,
the second pci->retc * 2 is one for each return BAT, and one for each return
mask array
+ shm_id = get_unique_shared_memory_id(1 + pci->retc * 2); //we need 1 +
pci->retc * 2 shared memory spaces, the first is for the header information,
the second pci->retc * 2 is one for each return BAT, and one for each return
mask array
MT_lock_unset(&pyapiLock, "pyapi.evaluate");
VERBOSE_MESSAGE("Creating multiple processes.\n");
@@ -533,9 +532,9 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
assert(memory_size > 0);
//create the shared memory for the header
MT_lock_set(&pyapiLock, "pyapi.evaluate");
- msg = create_shared_memory(shm_id, memory_size, (void**) &ptr);
+ msg = create_shared_memory(shm_id, memory_size, (void**) &ptr);
MT_lock_unset(&pyapiLock, "pyapi.evaluate");
- if (msg != MAL_SUCCEED)
+ if (msg != MAL_SUCCEED)
{
GDKfree(pids);
process_id = 0;
@@ -553,7 +552,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
//this means processes will only start returning values once all
the processes are finished, this is done because we want to have one big shared
memory block for each return value
//and we can only create that block when we know how many return
values there are, which we only know when all the processes have returned
-
+
sem_id = create_process_semaphore(shm_id, 2);
change_semaphore_value(sem_id, 0, process_count);
}
@@ -645,7 +644,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
ret->memory_size = 0;
ret->result_type = 0;
- //first get header information
+ //first get header information
#ifdef _PYAPI_TESTING_
peak_memory_usage = 0;
#endif
@@ -653,7 +652,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
{
ReturnBatDescr *descr = &(((ReturnBatDescr*)ptr)[j *
pci->retc + i]);
ret->count += descr->bat_count;
- total_size += descr->bat_size;
+ total_size += descr->bat_size;
#ifdef _PYAPI_TESTING_
peak_memory_usage += descr->peak_memory_usage;
#endif
@@ -707,8 +706,8 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
}
}
msg = MAL_SUCCEED;
-
- if (sem_id >= 0) release_process_semaphore(sem_id);
+
+ if (sem_id >= 0) release_process_semaphore(sem_id);
if (ptr != NULL) release_shared_memory(ptr);
if (pids != NULL) GDKfree(pids);
process_id = 0;
@@ -722,13 +721,13 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
gstate = PyGILState_Ensure();
/*[PARSE_CODE]*/
- VERBOSE_MESSAGE("Formatting python code.\n");
+ VERBOSE_MESSAGE("Formatting python code.\n");
pycall = FormatCode(exprStr, args, pci->argc, 4, &code_object, &msg);
if (pycall == NULL && code_object == NULL) {
if (msg == NULL) { msg = createException(MAL, "pyapi.eval", "Error
while parsing Python code."); }
goto wrapup;
}
-
+
/*[CONVERT_BAT]*/
VERBOSE_MESSAGE("Loading data from the database into Python.\n");
@@ -740,7 +739,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
// Now we will loop over the input BATs and convert them to python objects
for (i = pci->retc + 2; i < pci->argc; i++) {
- PyObject *result_array, *arg_name, *arg_type;
+ PyObject *result_array;
// t_start and t_end hold the part of the BAT we will convert to a
Numpy array, by default these hold the entire BAT [0 - BATcount(b)]
size_t t_start = 0, t_end = pyinput_values[i - (pci->retc + 2)].count;
#ifndef WIN32
@@ -768,7 +767,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
#ifdef _PYAPI_TESTING_
if (option_lazyarray) {
result_array = PyLazyArray_FromBAT(pyinput_values[i -
(pci->retc + 2)].bat);
- } else
+ } else
#endif
{
result_array = PyMaskedArray_FromBAT(&pyinput_values[i -
(pci->retc + 2)], t_start, t_end, &msg);
@@ -781,11 +780,10 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
goto wrapup;
}
if (code_object == NULL) {
- arg_name = PyString_FromString(args[i]);
- arg_type = PyString_FromString(BatType_Format(pyinput_values[i -
(pci->retc + 2)].bat_type));
- PyDict_SetItem(pColumns, arg_name, result_array);
- PyDict_SetItem(pColumnTypes, arg_name, arg_type);
- Py_DECREF(arg_name); Py_DECREF(arg_type);
+ PyObject *arg_type =
PyString_FromString(BatType_Format(pyinput_values[i - (pci->retc +
2)].bat_type));
+ PyDict_SetItemString(pColumns, args[i], result_array);
+ PyDict_SetItemString(pColumnTypes, args[i], arg_type);
+ Py_DECREF(arg_type);
}
PyTuple_SetItem(pArgs, ai++, result_array);
}
@@ -807,7 +805,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
msg = PyError_CreateException("Failed to load module", NULL);
goto wrapup;
}
-
+
// Now we will add the UDF to the main module
d = PyModule_GetDict(pModule);
if (code_object == NULL) {
@@ -831,7 +829,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
goto wrapup;
}
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
