Changeset: 71c0440590e1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=71c0440590e1
Added Files:
tools/embeddedpy/pyclient.c
tools/embeddedpy/pyclient.h
Modified Files:
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
tools/embeddedpy/embedded_module.c
tools/embeddedpy/embeddedpy.c
tools/embeddedpy/embeddedpy.h
Branch: pyapi
Log Message:
Added monetdblite.connect() support, allowing users to open multiple client
connections with monetdblite. Also free the GIL when a monetdblite query is
running.
diffs (truncated from 798 to 300 lines):
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -69,6 +69,7 @@ MCinit(void)
{
char *max_clients = GDKgetenv("max_clients");
int maxclients = 0;
+ Client c;
if (max_clients != NULL)
maxclients = atoi(max_clients);
@@ -85,6 +86,9 @@ MCinit(void)
showException(GDKout, MAL, "MCinit",MAL_MALLOC_FAIL);
mal_exit();
}
+ for (c = mal_clients; c < mal_clients + MAL_MAXCLIENTS; c++) {
+ MT_lock_init(&c->query_lock, "client.query_lock");
+ }
}
int
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -165,6 +165,15 @@ typedef struct CLIENT {
void *sqlcontext;
/*
+ Executing multiple SQL queries in parallel with the same client
+ does not work because of shared client state, causing
+ segfaults. In normal usage this is not possible, but
+ in MonetDBLite and MonetDB/Python this can happen. To
+ prevent segfualts, we have to lock the client when performing a query.
+ */
+ MT_Lock query_lock;
+
+ /*
* keep track of which instructions are currently being executed
*/
bit active; /* processing a query or not */
diff --git a/tools/embeddedpy/embedded_module.c
b/tools/embeddedpy/embedded_module.c
--- a/tools/embeddedpy/embedded_module.c
+++ b/tools/embeddedpy/embedded_module.c
@@ -22,12 +22,15 @@ static char create_docstring[] =
"monetdblite.create(tablename, dictionary), monetdblite.create(tablename,
column_names, values) => Create a SQL table from the given Python objects,
objects must either be a (column name, value) dictionary or a list of column
names and a list of values";
static char insert_docstring[] =
"monetdblite.insert(tablename, dictionary), monetdblite.insert(tablename,
column_names, values) => Insert a set of values into a SQL table";
+static char connect_docstring[] =
+ "monetdblite.connect() => Create a connection object to the monetdblite
database.";
static PyMethodDef module_methods[] = {
{"init", monetdb_init, METH_O, init_docstring},
- {"sql", monetdb_sql, METH_O, sql_docstring},
- {"create", monetdb_create, METH_VARARGS, create_docstring},
- {"insert", monetdb_insert, METH_VARARGS, insert_docstring},
+ {"sql", (PyCFunction)monetdb_sql, METH_VARARGS|METH_KEYWORDS,
sql_docstring},
+ {"create", (PyCFunction)monetdb_create, METH_VARARGS|METH_KEYWORDS,
create_docstring},
+ {"insert", (PyCFunction)monetdb_insert, METH_VARARGS|METH_KEYWORDS,
insert_docstring},
+ {"connect", (PyCFunction)monetdb_client, METH_NOARGS, connect_docstring},
{NULL, NULL, 0, NULL}
};
@@ -40,5 +43,5 @@ PyMODINIT_FUNC initmonetdblite(void)
return;
//import numpy stuff
- init_embedded_py();
+ monetdblite_init();
}
diff --git a/tools/embeddedpy/embeddedpy.c b/tools/embeddedpy/embeddedpy.c
--- a/tools/embeddedpy/embeddedpy.c
+++ b/tools/embeddedpy/embeddedpy.c
@@ -32,23 +32,8 @@
#include "pyapi.h"
#include "pytypes.h"
-/////////////////////////////////////////////////////////////////////////////////////////////////
-// copy paste from embedded.h
-typedef struct append_data {
- char* colname;
- bat batid;
-} append_data;
-
-str monetdb_startup(char* dir, char silent);
-str monetdb_query(char* query, void** result);
-str monetdb_create_table(char *schema, char *table_name, append_data *ad, int
ncols);
-str monetdb_append(const char* schema, const char* table, append_data *ad, int
ncols);
-void monetdb_cleanup_result(void* output);
-static str monetdb_get_columns(const char* schema_name, const char
*table_name, int *column_count, char ***column_names, int **column_types,
sql_subtype ***sql_subtypes);
-
-static bit monetdb_embedded_initialized = 0;
-static MT_Lock monetdb_embedded_lock;
-/////////////////////////////////////////////////////////////////////////////////////////////////
+#include "pyclient.h"
+#include "embedded.h"
PyObject *monetdb_init(PyObject *self, PyObject *args)
{
@@ -61,12 +46,14 @@ PyObject *monetdb_init(PyObject *self, P
{
char *msg;
char *directory = &(((PyStringObject*)args)->ob_sval[0]);
+ char installdir[1024];
printf("Making directory %s\n", directory);
if (GDKcreatedir(directory) != GDK_SUCCEED) {
PyErr_Format(PyExc_Exception, "Failed to create
directory %s.", directory);
return NULL;
}
- msg = monetdb_startup(directory, 1);
+ snprintf(installdir, 1024, "%s/../", BINDIR);
+ msg = monetdb_startup(installdir, directory, 1);
if (msg != MAL_SUCCEED) {
PyErr_Format(PyExc_Exception, "Failed to initialize
MonetDB. %s", msg);
return NULL;
@@ -76,26 +63,40 @@ PyObject *monetdb_init(PyObject *self, P
Py_RETURN_NONE;
}
-PyObject *monetdb_sql(PyObject *self, PyObject *args)
+PyObject *monetdb_sql(PyObject *self, PyObject *args, PyObject *keywds)
{
+ Client c = monetdb_default_client;
+ char *query;
+ PyObject *client = NULL;
+ static char *kwlist[] = {"query", "conn", NULL};
(void) self;
- if (!PyString_CheckExact(args)) {
- PyErr_SetString(PyExc_TypeError, "Expected a SQL query as a
single string argument.");
- return NULL;
- }
if (!monetdb_embedded_initialized) {
PyErr_SetString(PyExc_Exception, "monetdb has not been
initialized yet");
return NULL;
}
-
+ if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|O", kwlist, &query,
&client)) {
+ return NULL;
+ }
+ if (client != NULL) {
+ if (!PyClient_CheckExact(client)) {
+ PyErr_SetString(PyExc_Exception, "conn must be a
connection object created by monetdblite.connect().");
+ return NULL;
+ }
+ c = ((PyClientObject*)client)->cntxt;
+ }
{
PyObject *result;
res_table* output = NULL;
+ PyObject *querystring;
char* err;
// Append ';'' to the SQL query, just in case
- args = PyString_FromFormat("%s;",
&(((PyStringObject*)args)->ob_sval[0]));
+ querystring = PyString_FromFormat("%s;", query);
// Perform the SQL query
- err = monetdb_query(&(((PyStringObject*)args)->ob_sval[0]),
(void**)&output);
+Py_BEGIN_ALLOW_THREADS
+ MT_lock_set(&c->query_lock, "client.query_lock");
+ err = monetdb_query(c,
&(((PyStringObject*)querystring)->ob_sval[0]), (void**)&output);
+ MT_lock_unset(&c->query_lock, "client.query_lock");
+Py_END_ALLOW_THREADS
if (err != NULL) {
PyErr_Format(PyExc_Exception, "SQL Query Failed: %s",
(err ? err : "<no error>"));
return NULL;
@@ -117,15 +118,15 @@ PyObject *monetdb_sql(PyObject *self, Py
input.scalar = false;
input.sql_subtype = &col.type;
- numpy_array = PyMaskedArray_FromBAT(&mal_clients[0], &input, 0,
input.count, &msg, true);
+ numpy_array = PyMaskedArray_FromBAT(c, &input, 0, input.count,
&msg, true);
if (!numpy_array) {
- monetdb_cleanup_result(output);
+ monetdb_cleanup_result(c, output);
PyErr_Format(PyExc_Exception, "SQL
Query Failed: %s", (msg ? msg : "<no error>"));
return NULL;
}
PyDict_SetItem(result,
PyString_FromString(output->cols[i].name), numpy_array);
}
- monetdb_cleanup_result(output);
+ monetdb_cleanup_result(c, output);
return result;
} else {
Py_RETURN_NONE;
@@ -133,40 +134,49 @@ PyObject *monetdb_sql(PyObject *self, Py
}
}
-PyObject *monetdb_create(PyObject *self, PyObject *args)
+PyObject *monetdb_create(PyObject *self, PyObject *args, PyObject *keywds)
{
- char *schema = "sys";
+ char *schema_name = "sys";
char *table_name;
- PyObject *dict = NULL, *values = NULL;
+ PyObject *values = NULL, *client = NULL, *colnames = NULL;
+ Client c = monetdb_default_client;
+ static char *kwlist[] = {"name", "values", "colnames", "schema",
"conn", NULL};
int i;
(void) self;
if (!monetdb_embedded_initialized) {
PyErr_SetString(PyExc_Exception, "monetdb has not been
initialized yet");
return NULL;
}
- if (!PyArg_ParseTuple(args, "sO|O", &table_name, &dict, &values)) {
+ if (!PyArg_ParseTupleAndKeywords(args, keywds, "sO|OsO", kwlist,
&table_name, &values, &colnames, &schema_name, &client)) {
return NULL;
}
- if (values == NULL) {
- if (!PyDict_CheckExact(dict)) {
- PyErr_SetString(PyExc_TypeError, "the second argument
has to be a dict");
+ if (client != NULL) {
+ if (!PyClient_CheckExact(client)) {
+ PyErr_SetString(PyExc_Exception, "conn must be a
connection object created by monetdblite.connect().");
+ return NULL;
+ }
+ c = ((PyClientObject*)client)->cntxt;
+ }
+ if (colnames == NULL) {
+ if (!PyDict_CheckExact(values)) {
+ PyErr_SetString(PyExc_TypeError, "no colnames are
specified and values is not a dict");
return NULL;
}
PyErr_Format(PyExc_Exception, "dict is not implemented yet");
return NULL;
Py_RETURN_NONE;
} else {
- if (!PyList_CheckExact(dict)) {
- PyErr_SetString(PyExc_TypeError, "the second argument
(column names) must be a list");
+ if (!PyList_Check(colnames)) {
+ PyErr_SetString(PyExc_TypeError, "colnames must be a
list");
return NULL;
}
- if (PyList_Size(dict) == 0) {
- PyErr_SetString(PyExc_TypeError, "the list of column
names must be non-zero");
+ if (PyList_Size(colnames) == 0) {
+ PyErr_SetString(PyExc_TypeError, "colnames must have at
least one element");
return NULL;
}
- for(i = 0; i < PyList_Size(dict); i++) {
- PyObject *column_name = PyList_GetItem(dict, i);
+ for(i = 0; i < PyList_Size(colnames); i++) {
+ PyObject *column_name = PyList_GetItem(colnames, i);
if (!PyString_CheckExact(column_name)) {
PyErr_Format(PyExc_TypeError, "the entry %d in
the column names is not a string", i);
return NULL;
@@ -176,7 +186,7 @@ PyObject *monetdb_create(PyObject *self,
{
char *msg = NULL;
PyObject *pResult;
- int columns = PyList_Size(dict);
+ int columns = PyList_Size(colnames);
append_data *append_bats = NULL;
PyReturn *pyreturn_values = NULL;
@@ -195,9 +205,13 @@ PyObject *monetdb_create(PyObject *self,
BAT *b =
PyObject_ConvertToBAT(&pyreturn_values[i], NULL,
PyType_ToBat(pyreturn_values[i].result_type), i, 0, &msg, true);
if (b == NULL) goto cleanup;
append_bats[i].batid = b->batCacheid;
- append_bats[i].colname =
PyString_AS_STRING(PyList_GetItem(dict, i));
+ append_bats[i].colname =
PyString_AS_STRING(PyList_GetItem(colnames, i));
}
- msg = monetdb_create_table(schema, table_name,
append_bats, columns);
+Py_BEGIN_ALLOW_THREADS
+ MT_lock_set(&c->query_lock, "client.query_lock");
+ msg = monetdb_create_table(c, schema_name, table_name,
append_bats, columns);
+ MT_lock_unset(&c->query_lock, "client.query_lock");
+Py_END_ALLOW_THREADS
cleanup:
if (pyreturn_values) GDKfree(pyreturn_values);
if (append_bats) {
@@ -216,19 +230,29 @@ cleanup:
Py_RETURN_NONE;
}
-PyObject *monetdb_insert(PyObject *self, PyObject *args)
+PyObject *monetdb_insert(PyObject *self, PyObject *args, PyObject *keywds)
{
char *schema_name = "sys";
char *table_name;
- PyObject *values = NULL;
+ PyObject *values = NULL, *client = NULL;
+ Client c = monetdb_default_client;
+ static char *kwlist[] = {"name", "values", "schema", "conn", NULL};
(void) self;
if (!monetdb_embedded_initialized) {
PyErr_SetString(PyExc_Exception, "monetdb has not been
initialized yet");
return NULL;
}
- if (!PyArg_ParseTuple(args, "sO", &table_name, &values)) {
+
+ if (!PyArg_ParseTupleAndKeywords(args, keywds, "sO|sO", kwlist,
&table_name, &values, &schema_name, &client)) {
return NULL;
}
+ if (client != NULL) {
+ if (!PyClient_CheckExact(client)) {
+ PyErr_SetString(PyExc_Exception, "conn must be a
connection object created by monetdblite.connect().");
+ return NULL;
+ }
+ c = ((PyClientObject*)client)->cntxt;
+ }
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
